"""
SwarmMetrics — reciprocity & echo measurement for agent communication graphs.

Input: list of message dicts with keys:
    from_id (str), to_id (str), timestamp (float, epoch seconds),
    channel (str, optional), concepts (list[str], optional)

Output: ScoredGraph with per-edge reciprocity, per-channel classification,
        per-node influence metrics, gravitational shadow estimates.

Built from empirical data on 6345+ inter-agent bus messages.
"""

import math
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Optional


@dataclass
class EdgeScore:
    source: str
    target: str
    messages_ab: int = 0
    messages_ba: int = 0
    reciprocity_raw: float = 0.0
    reciprocity_f3: float = 0.0  # log-transform, half-life weighted
    reciprocity_f32: float = 0.0  # F3.2: log(count) * log(1+reciprocity)
    reciprocity_f33: float = 0.0  # F3.3: log(unique_concepts) * log(1+reciprocity)
    unique_concepts: int = 0      # distinct concepts across both directions


@dataclass
class ChannelScore:
    channel: str
    speakers: list = field(default_factory=list)
    gini: float = 0.0
    classification: str = "unknown"  # dialogue, broadcast, echo, monologue
    dominant_speaker: Optional[str] = None
    dominant_share: float = 0.0


@dataclass
class NodeScore:
    node_id: str
    total_sent: int = 0
    total_received: int = 0
    echo_coefficient: float = 0.0
    shadow_strength: float = 0.0
    is_shadow: bool = False
    last_active: float = 0.0


@dataclass
class ScoredGraph:
    edges: dict = field(default_factory=dict)       # (a,b) -> EdgeScore
    channels: dict = field(default_factory=dict)     # ch -> ChannelScore
    nodes: dict = field(default_factory=dict)        # id -> NodeScore
    summary: dict = field(default_factory=dict)


def _gini(values: list[float]) -> float:
    """Gini coefficient. 0 = perfect equality, 1 = one speaker dominates."""
    if len(values) <= 1:
        return 1.0  # single speaker = max inequality
    n = len(values)
    values = sorted(values)
    total = sum(values)
    if total == 0:
        return 0.0
    cumsum = 0.0
    weighted_sum = 0.0
    for i, v in enumerate(values):
        cumsum += v
        weighted_sum += (2 * (i + 1) - n - 1) * v
    return weighted_sum / (n * total)


def _half_life_weight(dt_seconds: float, tau_days: float = 7.0) -> float:
    """Exponential decay weight. tau_days = half-life in days."""
    tau_seconds = tau_days * 86400
    if tau_seconds == 0:
        return 0.0
    return math.exp(-0.693 * dt_seconds / tau_seconds)


def score_reciprocity(messages: list[dict], now: Optional[float] = None,
                      tau_days: float = 7.0) -> dict:
    """
    F3 reciprocity: log(1 + min(a→b, b→a) / max(a→b, b→a))
    with half-life decay weighting.

    F3.2 (added 2026-07-27): log(count) * log(1 + reciprocity)
    Double-log compression prevents volume from buying back low reciprocity.
    Fix proposed by reticuli (Colony): count*log(1+r) lets 100-ping broadcast
    outrank 5-message dialogue. log(count)*log(1+r) closes that gap.

    F3.3 (added 2026-07-27): log(unique_concepts) * log(1 + reciprocity)
    Replaces raw message count with unique concept count per edge.
    Idea from 小风 (ClawdChat): "count is the problem, not log. Replace count
    with unique topic count or information entropy — 100 pings covering 2 topics
    collapse, 5 conversations covering 5 topics win." Requires concepts field
    in messages. Falls back to F3.2 when no concepts available.
    """
    import time
    if now is None:
        now = time.time()

    # Count weighted messages per directed edge
    weighted = defaultdict(float)
    raw = defaultdict(int)
    edge_concepts = defaultdict(set)  # (a,b) -> set of unique concepts
    for m in messages:
        a, b = m["from_id"], m["to_id"]
        t = m.get("timestamp", now)
        w = _half_life_weight(now - t, tau_days)
        weighted[(a, b)] += w
        raw[(a, b)] += 1
        for c in m.get("concepts", []):
            edge_concepts[(a, b)].add(c)

    # Compute reciprocity per undirected pair
    edges = {}
    seen = set()
    for (a, b) in list(weighted.keys()) + list(raw.keys()):
        pair = tuple(sorted([a, b]))
        if pair in seen:
            continue
        seen.add(pair)

        ab_w = weighted.get((a, b), 0) + weighted.get((pair[0], pair[1]), 0) \
            if (a, b) != (pair[0], pair[1]) else weighted.get((pair[0], pair[1]), 0)
        ba_w = weighted.get((b, a), 0) + weighted.get((pair[1], pair[0]), 0) \
            if (b, a) != (pair[1], pair[0]) else weighted.get((pair[1], pair[0]), 0)

        # Simpler: just use pair order
        ab_w = weighted.get((pair[0], pair[1]), 0)
        ba_w = weighted.get((pair[1], pair[0]), 0)
        ab_r = raw.get((pair[0], pair[1]), 0)
        ba_r = raw.get((pair[1], pair[0]), 0)

        mx = max(ab_w, ba_w)
        if mx > 0:
            ratio = min(ab_w, ba_w) / mx
            f3 = math.log(1 + ratio)
        else:
            f3 = 0.0

        mx_raw = max(ab_r, ba_r)
        raw_ratio = min(ab_r, ba_r) / mx_raw if mx_raw > 0 else 0.0

        # F3.2: log(count) * log(1 + reciprocity)
        total_count = ab_r + ba_r
        f32 = math.log(max(total_count, 1)) * f3  # f3 is already log(1+ratio)

        # F3.3: log(unique_concepts) * log(1 + reciprocity)
        concepts_ab = edge_concepts.get((pair[0], pair[1]), set())
        concepts_ba = edge_concepts.get((pair[1], pair[0]), set())
        unique = concepts_ab | concepts_ba
        n_unique = len(unique)
        # Fall back to F3.2 when no concepts available
        f33 = math.log(max(n_unique, 1)) * f3 if n_unique > 0 else f32

        es = EdgeScore(
            source=pair[0], target=pair[1],
            messages_ab=ab_r, messages_ba=ba_r,
            reciprocity_raw=raw_ratio,
            reciprocity_f3=f3,
            reciprocity_f32=f32,
            reciprocity_f33=f33,
            unique_concepts=n_unique
        )
        edges[pair] = es

    return edges


def score_channels(messages: list[dict], gini_broadcast_threshold: float = 0.6) -> dict:
    """
    Per-channel Gini evenness. Classifies channels as:
    - monologue: 1 speaker
    - dialogue: 2 speakers, reciprocity exists
    - broadcast: N speakers but Gini > threshold
    - conversation: N speakers, Gini <= threshold
    """
    ch_counts = defaultdict(lambda: defaultdict(int))
    for m in messages:
        ch = m.get("channel", "default")
        speaker = m["from_id"]
        ch_counts[ch][speaker] += 1

    channels = {}
    for ch, speakers in ch_counts.items():
        counts = list(speakers.values())
        n = len(counts)
        total = sum(counts)
        gini = _gini(counts)

        dominant = max(speakers, key=speakers.get)
        dominant_share = speakers[dominant] / total if total > 0 else 0

        if n == 1:
            classification = "monologue"
        elif n == 2:
            classification = "dialogue"
        elif gini > gini_broadcast_threshold:
            classification = "broadcast"
        else:
            classification = "conversation"

        channels[ch] = ChannelScore(
            channel=ch,
            speakers=list(speakers.keys()),
            gini=round(gini, 4),
            classification=classification,
            dominant_speaker=dominant,
            dominant_share=round(dominant_share, 4)
        )

    return channels


def score_echo(messages: list[dict], concept_window_seconds: float = 604800,
               use_idf: bool = True, snapshot_idf: bool = False,
               idf_window_seconds: float = 0,
               idf_decay_halflife: float = 0) -> dict:
    """
    Echo coefficient: measures concept diffusion from nodes that
    don't reply but whose concepts appear downstream.

    For each node, echo_coeff = concepts_echoed / concepts_introduced.
    High echo + low message count = gravitational shadow.

    When use_idf=True, applies inverse-document-frequency weighting:
    rare concepts that spread are weighted higher than common ones.
    IDF improvement suggested by hermes-final (Colony, 2026-07-27).

    When snapshot_idf=True, computes IDF at time of emission rather than
    on full corpus. Fixes endogeneity bug: post-hoc IDF is contaminated
    by the diffusion it measures — a concept becomes common BECAUSE it
    diffused, so post-hoc IDF penalizes successful diffusion.
    Bug identified by ColonistOne (Colony, 2026-07-28).

    When idf_window_seconds > 0 (requires snapshot_idf=True), only messages
    within [t - idf_window_seconds, t] contribute to IDF at time t. This
    prevents retired loud agents from suppressing credit via stale vocabulary
    norms. Fixes semantic hysteresis (t=-3.96, Dispatch, 2026-07-28).

    When idf_decay_halflife > 0 (requires snapshot_idf=True), uses
    exponentially weighted moving average (EWMA) for document frequency
    instead of a hard window cutoff. Each past agent-concept association
    decays as exp(-λ·Δt) where λ = ln(2)/halflife. Fixes the boundary
    discontinuity of hard windows. Suggested by Eliza-Gemma (Colony,
    2026-07-28). Overrides idf_window_seconds if both are set.
    """
    # Sort messages by time for temporal IDF snapshots
    sorted_msgs = sorted(messages, key=lambda m: m.get("timestamp", 0))

    # Build concept timeline: who introduced which concept, when
    introductions = {}  # concept -> (first_node, first_time)
    echoes = defaultdict(float)  # source_node -> weighted echo count

    all_agents = set(m["from_id"] for m in messages) | set(m["to_id"] for m in messages)
    n_agents = max(len(all_agents), 1)

    if use_idf and not snapshot_idf:
        # Legacy: pre-compute IDF on full corpus (endogenous, but backwards-compatible)
        concept_agents = defaultdict(set)
        for m in messages:
            for c in m.get("concepts", []):
                concept_agents[c].add(m["from_id"])

    if use_idf and snapshot_idf:
        # Temporal IDF: track concept usage incrementally
        concept_agents_at = defaultdict(set)  # concept -> set of agents seen so far
        if idf_decay_halflife > 0 or idf_window_seconds > 0:
            # For windowed/EWMA IDF: store (agent, timestamp) pairs
            concept_agent_times = defaultdict(list)  # concept -> [(agent, time), ...]
        if idf_decay_halflife > 0:
            _ewma_lambda = math.log(2) / idf_decay_halflife

    for m in sorted_msgs:
        concepts = m.get("concepts", [])
        node = m["from_id"]
        t = m.get("timestamp", 0)

        # Update temporal IDF tracker BEFORE scoring (snapshot = state before this msg)
        if use_idf and snapshot_idf:
            # Snapshot IDF for this message is computed from agents seen BEFORE now
            pass  # concept_agents_at already has pre-emission state

        for c in concepts:
            if c not in introductions:
                introductions[c] = (node, t)
            else:
                orig_node, orig_t = introductions[c]
                if orig_node != node and (t - orig_t) <= concept_window_seconds:
                    if use_idf:
                        if snapshot_idf:
                            if idf_decay_halflife > 0:
                                # EWMA IDF: smooth exponential decay
                                agent_weights = defaultdict(float)
                                for a, at in concept_agent_times.get(c, []):
                                    w = math.exp(-_ewma_lambda * (t - at))
                                    agent_weights[a] += w
                                effective = sum(min(w, 1.0) for w in agent_weights.values())
                                idf = math.log(n_agents / max(effective, 1))
                            elif idf_window_seconds > 0:
                                # Windowed IDF: only count agents within the window
                                recent = [a for a, at in concept_agent_times.get(c, [])
                                          if t - at <= idf_window_seconds]
                                agent_count = len(set(recent))
                                idf = math.log(n_agents / max(agent_count, 1))
                            else:
                                # Snapshot IDF: all agents seen before now
                                agent_count = len(concept_agents_at.get(c, set()))
                                idf = math.log(n_agents / max(agent_count, 1))
                        else:
                            # Legacy: full-corpus IDF
                            agent_count = len(concept_agents.get(c, set()))
                            idf = math.log(n_agents / max(agent_count, 1))
                        echoes[orig_node] += max(idf, 0.1)  # floor at 0.1
                    else:
                        echoes[orig_node] += 1

        # Update temporal tracker AFTER scoring this message
        if use_idf and snapshot_idf:
            for c in concepts:
                concept_agents_at[c].add(node)
                if idf_decay_halflife > 0 or idf_window_seconds > 0:
                    concept_agent_times[c].append((node, t))

    # Compute per-node echo coefficient
    concepts_per_node = defaultdict(set)
    for m in messages:
        for c in m.get("concepts", []):
            concepts_per_node[m["from_id"]].add(c)

    node_echo = {}
    for node, concepts in concepts_per_node.items():
        introduced = sum(1 for c in concepts
                        if introductions.get(c, (None,))[0] == node)
        echo_count = echoes.get(node, 0.0)
        coeff = echo_count / introduced if introduced > 0 else 0.0
        node_echo[node] = round(coeff, 4)

    return node_echo


def shuffle_test(messages: list[dict], n_shuffles: int = 100,
                 use_idf: bool = True) -> dict:
    """
    Shuffle test for causal echo (Anagnostopoulos, Kumar & Mahdian 2008).
    Permutes timestamps while holding network fixed.
    If echo scores survive shuffling, they measured correlation (shared
    environment) not diffusion (causal influence).

    Returns dict with:
      - observed: {node: echo_coeff} from actual data
      - mean_shuffled: {node: mean echo across shuffles}
      - z_scores: {node: (observed - mean_shuffled) / std_shuffled}
      - significant: {node: bool} where |z| > 2.0

    Cited: ColonistOne (Colony, 2026-07-28) pointed to this method.
    """
    import random

    # Observed echo
    observed = score_echo(messages, use_idf=use_idf)

    # Run shuffles
    shuffled_scores = defaultdict(list)
    timestamps = [m.get("timestamp", 0) for m in messages]

    for _ in range(n_shuffles):
        # Permute timestamps, keep everything else
        perm = timestamps.copy()
        random.shuffle(perm)
        shuffled_msgs = []
        for i, m in enumerate(messages):
            sm = m.copy()
            sm["timestamp"] = perm[i]
            shuffled_msgs.append(sm)

        shuffled_echo = score_echo(shuffled_msgs, use_idf=use_idf)
        all_nodes = set(observed) | set(shuffled_echo)
        for node in all_nodes:
            shuffled_scores[node].append(shuffled_echo.get(node, 0.0))

    # Compute z-scores
    result = {
        "observed": observed,
        "mean_shuffled": {},
        "z_scores": {},
        "significant": {}
    }

    for node in set(observed) | set(shuffled_scores):
        scores = shuffled_scores.get(node, [0.0])
        mean_s = sum(scores) / len(scores)
        std_s = (sum((s - mean_s)**2 for s in scores) / len(scores)) ** 0.5
        obs = observed.get(node, 0.0)

        result["mean_shuffled"][node] = round(mean_s, 4)
        if std_s > 0:
            z = (obs - mean_s) / std_s
        else:
            z = 0.0 if obs == mean_s else float('inf')
        result["z_scores"][node] = round(z, 4)
        result["significant"][node] = abs(z) > 2.0

    return result


def ablation_sensitivity(messages: list[dict], target: str,
                         n_shuffles: int = 50, use_idf: bool = True) -> dict:
    """
    Ablation test: remove target agent, recompute shuffle test, measure delta.

    Returns dict with:
      - target: agent removed
      - target_msg_count: how many messages removed
      - target_traffic_pct: percentage of corpus removed
      - deltas: {node: z_ablated - z_full} for all nodes
      - gainers: nodes whose echo INCREASED (were being suppressed by target)
      - losers: nodes whose echo DECREASED (were being amplified by target)
      - verdict: 'suppressor' if mean delta > 0, 'amplifier' if < 0, 'neutral'

    Discovered 2026-07-28: bolt (38% traffic) = systematic suppressor,
    dispatch (5.6% traffic) = mild amplifier. Crossover threshold TBD.
    Holocene (Colony) asked the question; ablation answered it.
    """
    # Baseline
    st_full = shuffle_test(messages, n_shuffles=n_shuffles, use_idf=use_idf)

    # Ablate target
    ablated = [m for m in messages if m.get("from_id", "") != target]
    removed = len(messages) - len(ablated)
    if removed == 0:
        return {"target": target, "target_msg_count": 0,
                "error": f"agent '{target}' not found in corpus"}

    st_abl = shuffle_test(ablated, n_shuffles=n_shuffles, use_idf=use_idf)

    z_full = st_full["z_scores"]
    z_abl = st_abl["z_scores"]

    deltas = {}
    for node in set(z_full) | set(z_abl):
        if node == target:
            continue
        deltas[node] = round(z_abl.get(node, 0) - z_full.get(node, 0), 4)

    gainers = {k: v for k, v in deltas.items() if v > 2.0}
    losers = {k: v for k, v in deltas.items() if v < -2.0}

    mean_delta = sum(deltas.values()) / max(len(deltas), 1)
    if mean_delta > 0.5:
        verdict = "suppressor"
    elif mean_delta < -0.5:
        verdict = "amplifier"
    else:
        verdict = "neutral"

    return {
        "target": target,
        "target_msg_count": removed,
        "target_traffic_pct": round(100 * removed / len(messages), 1),
        "mean_delta": round(mean_delta, 4),
        "verdict": verdict,
        "gainers": dict(sorted(gainers.items(), key=lambda x: -x[1])),
        "losers": dict(sorted(losers.items(), key=lambda x: x[1])),
        "deltas": dict(sorted(deltas.items(), key=lambda x: -abs(x[1]))),
    }


def detect_shadows(messages: list[dict], now: Optional[float] = None,
                   silence_days: float = 7.0) -> dict:
    """
    Gravitational shadow: nodes that were active but are now silent,
    with residual influence estimated from historical message weight.
    """
    import time
    if now is None:
        now = time.time()

    silence_threshold = silence_days * 86400
    node_activity = defaultdict(list)

    for m in messages:
        node_activity[m["from_id"]].append(m.get("timestamp", now))

    shadows = {}
    for node, timestamps in node_activity.items():
        last = max(timestamps)
        silence = now - last

        if silence > silence_threshold:
            # Historical weight: sum of half-life-weighted messages
            total_weight = sum(_half_life_weight(now - t, tau_days=30.0)
                             for t in timestamps)
            shadows[node] = {
                "last_active": last,
                "silence_days": round(silence / 86400, 1),
                "historical_weight": round(total_weight, 4),
                "message_count": len(timestamps)
            }

    return shadows


def analyze(messages: list[dict], now: Optional[float] = None,
            tau_days: float = 7.0, gini_threshold: float = 0.6,
            silence_days: float = 7.0) -> ScoredGraph:
    """
    Full analysis: reciprocity + channels + echo + shadows.
    Returns ScoredGraph with all metrics.
    """
    edges = score_reciprocity(messages, now=now, tau_days=tau_days)
    channels = score_channels(messages, gini_broadcast_threshold=gini_threshold)
    echo = score_echo(messages)
    shadows = detect_shadows(messages, now=now, silence_days=silence_days)

    # Build node scores
    nodes = {}
    node_sent = defaultdict(int)
    node_recv = defaultdict(int)
    node_last = defaultdict(float)

    for m in messages:
        node_sent[m["from_id"]] += 1
        node_recv[m["to_id"]] += 1
        t = m.get("timestamp", 0)
        node_last[m["from_id"]] = max(node_last[m["from_id"]], t)

    all_nodes = set(node_sent) | set(node_recv)
    for n in all_nodes:
        ns = NodeScore(
            node_id=n,
            total_sent=node_sent[n],
            total_received=node_recv[n],
            echo_coefficient=echo.get(n, 0.0),
            shadow_strength=shadows.get(n, {}).get("historical_weight", 0.0),
            is_shadow=n in shadows,
            last_active=node_last.get(n, 0.0)
        )
        nodes[n] = ns

    # Summary
    dialogue_count = sum(1 for c in channels.values()
                        if c.classification == "dialogue")
    broadcast_count = sum(1 for c in channels.values()
                         if c.classification == "broadcast")
    shadow_count = len(shadows)

    graph = ScoredGraph(
        edges=edges,
        channels=channels,
        nodes=nodes,
        summary={
            "total_messages": len(messages),
            "total_nodes": len(all_nodes),
            "total_edges": len(edges),
            "dialogue_channels": dialogue_count,
            "broadcast_channels": broadcast_count,
            "shadow_nodes": shadow_count,
            "mean_reciprocity_f3": round(
                sum(e.reciprocity_f3 for e in edges.values()) / max(len(edges), 1), 4
            )
        }
    )
    return graph


if __name__ == "__main__":
    # Quick demo with synthetic data
    import time
    now = time.time()
    day = 86400

    demo_messages = [
        # Alice-Bob dialogue (high reciprocity)
        {"from_id": "alice", "to_id": "bob", "timestamp": now - 1*day, "channel": "dev", "concepts": ["reciprocity", "graph"]},
        {"from_id": "bob", "to_id": "alice", "timestamp": now - 1*day + 3600, "channel": "dev", "concepts": ["graph", "metrics"]},
        {"from_id": "alice", "to_id": "bob", "timestamp": now - 0.5*day, "channel": "dev", "concepts": ["decay"]},
        {"from_id": "bob", "to_id": "alice", "timestamp": now - 0.5*day + 1800, "channel": "dev"},
        # Carol broadcasts (low reciprocity, high Gini)
        {"from_id": "carol", "to_id": "alice", "timestamp": now - 2*day, "channel": "announce", "concepts": ["launch"]},
        {"from_id": "carol", "to_id": "bob", "timestamp": now - 2*day + 60, "channel": "announce"},
        {"from_id": "carol", "to_id": "dave", "timestamp": now - 2*day + 120, "channel": "announce"},
        {"from_id": "carol", "to_id": "eve", "timestamp": now - 2*day + 180, "channel": "announce"},
        {"from_id": "alice", "to_id": "carol", "timestamp": now - 1.5*day, "channel": "announce"},
        # Dave: silent but concept echoed
        {"from_id": "dave", "to_id": "alice", "timestamp": now - 10*day, "channel": "research", "concepts": ["echo", "shadow"]},
        {"from_id": "alice", "to_id": "bob", "timestamp": now - 5*day, "channel": "dev", "concepts": ["echo"]},
        {"from_id": "bob", "to_id": "carol", "timestamp": now - 3*day, "channel": "dev", "concepts": ["shadow"]},
        # Eve: completely silent listener
        {"from_id": "eve", "to_id": "carol", "timestamp": now - 15*day, "channel": "announce"},
    ]

    result = analyze(demo_messages, now=now)

    print("=== SwarmMetrics Demo ===\n")
    print(f"Summary: {result.summary}\n")

    print("Edge Reciprocity:")
    for pair, es in sorted(result.edges.items(), key=lambda x: -x[1].reciprocity_f3):
        print(f"  {es.source} <-> {es.target}: F3={es.reciprocity_f3:.4f} "
              f"({es.messages_ab}↔{es.messages_ba})")

    print("\nChannel Classification:")
    for ch, cs in result.channels.items():
        print(f"  {ch}: {cs.classification} (Gini={cs.gini}, "
              f"dominant={cs.dominant_speaker} @ {cs.dominant_share:.0%})")

    print("\nNode Echo Coefficients:")
    for nid, ns in sorted(result.nodes.items(), key=lambda x: -x[1].echo_coefficient):
        shadow = " [SHADOW]" if ns.is_shadow else ""
        print(f"  {nid}: echo={ns.echo_coefficient:.4f}, "
              f"sent={ns.total_sent}, recv={ns.total_received}{shadow}")


# ── Semantic collapse (F3.4) ─────────────────────────────
# Defense against synonym padding: adversary generates 50 unique strings
# that all mean "hello", defeating unique_concepts count in F3.3.
# Collapse semantically similar concepts before counting.
# Atomic Raven (Colony, 2026-07-28) identified the attack vector.

def _char_ngrams(s: str, n: int = 3) -> set:
    return set(s[i:i+n] for i in range(max(len(s) - n + 1, 1)))


def _jaccard(a: str, b: str) -> float:
    sa, sb = _char_ngrams(a), _char_ngrams(b)
    if not sa or not sb:
        return 0.0
    return len(sa & sb) / len(sa | sb)


def semantic_collapse(concepts: list, threshold: float = 0.6) -> list:
    """Collapse similar concepts into canonical forms using n-gram Jaccard.

    A cheap proxy for semantic similarity. Catches "greeting-1"/"greeting-2"
    style padding but preserves genuinely diverse concepts.

    threshold: Jaccard similarity above which concepts merge.
    0.6 collapses trivial suffix variations, keeps distinct concepts.

    Returns deduplicated concept list (canonical forms only).
    """
    canonicals = []  # (canonical_form, ngrams)
    mapping = {}

    for c in concepts:
        matched = False
        for canon, canon_ngrams in canonicals:
            if _jaccard(c, canon) >= threshold:
                mapping[c] = canon
                matched = True
                break
        if not matched:
            canonicals.append((c, _char_ngrams(c)))
            mapping[c] = c

    return list(set(mapping[c] for c in concepts))


# ── Agent-alias merging ──────────────────────────────────
# Agents that change names mid-corpus create phantom signals
# in temporal analysis. merge_aliases() canonicalizes from_id/to_id
# before scoring, collapsing identity evolution into a single node.

def merge_aliases(messages: list, alias_map: dict) -> list:
    """Merge agent aliases in a message corpus.

    alias_map: {old_name: canonical_name, ...}
    Returns new list with from_id/to_id replaced by canonical names.

    Example:
        alias_map = {
            'petrovich': 'petrovich-codex',
            'hausmaster': 'φ-hausmaster',
            'konstantин': 'кот-констант',
            'кот-константин': 'кот-констант',
        }
        merged = merge_aliases(messages, alias_map)
    """
    merged = []
    for m in messages:
        m2 = m.copy()
        m2['from_id'] = alias_map.get(m2.get('from_id', ''), m2.get('from_id', ''))
        m2['to_id'] = alias_map.get(m2.get('to_id', ''), m2.get('to_id', ''))
        merged.append(m2)
    return merged


# Default alias map for the OMPU bus corpus
OMPU_BUS_ALIASES = {
    'petrovich': 'petrovich-codex',
    'hausmaster': 'φ-hausmaster',
    'phi': 'φ-hausmaster',
    'phi_hausmaster': 'φ-hausmaster',
    'phi-hausmaster': 'φ-hausmaster',
    'константин': 'кот-констант',
    'кот-константин': 'кот-констант',
}


# ── φ-accrual failure detector for agent liveness ──────
# Hayashibara et al. 2004. Continuous suspicion metric instead
# of binary alive/dead. Tracks inter-arrival times of an agent's
# messages, outputs a suspicion level φ that climbs monotonically
# during silence. Higher φ = more suspicious that the agent is dead.
#
# φ = -log10(1 - F(t_now - t_last))
# where F is the CDF of the inter-arrival distribution (assumed normal).
#
# Three states derived from φ:
#   φ < 1.0  → GREEN (< 90% suspicion, probably alive)
#   φ < 3.0  → STALE (< 99.9% suspicion, silence is unusual)
#   φ >= 3.0 → GRAY  (> 99.9% suspicion, likely dead or budget-exhausted)
#
# This distinguishes three failure modes that dashboards collapse:
#   - stale green (checked at T, referent moved since)
#   - silence (no check ran, no timestamp to compute staleness from)
#   - dead observer (checker itself stopped)

@dataclass
class AgentLiveness:
    agent_id: str
    last_seen: float = 0.0         # epoch seconds
    message_count: int = 0
    mean_interval: float = 0.0     # seconds between messages
    std_interval: float = 0.0
    phi: float = 0.0               # suspicion score
    state: str = 'unknown'         # green / stale / gray / unknown

    def as_dict(self) -> dict:
        return {
            'agent_id': self.agent_id,
            'last_seen': self.last_seen,
            'message_count': self.message_count,
            'mean_interval': self.mean_interval,
            'std_interval': self.std_interval,
            'phi': round(self.phi, 3),
            'state': self.state,
        }


def phi_accrual(messages: list, t_now: float = None,
                min_messages: int = 5) -> dict:
    """Compute φ-accrual suspicion for each agent in the corpus.

    Returns: {agent_id: AgentLiveness, ...}

    Args:
        messages: list of dicts with 'from_id' and 'timestamp' keys
        t_now: current time (epoch seconds). If None, uses max timestamp + 1h.
        min_messages: minimum messages to compute φ (else state='unknown')
    """
    # Collect per-agent timestamps, sorted
    agent_times: dict = defaultdict(list)
    for m in messages:
        agent_id = m.get('from_id', '')
        ts = m.get('timestamp', 0)
        if agent_id and ts:
            agent_times[agent_id].append(ts)

    if t_now is None:
        all_ts = [t for times in agent_times.values() for t in times]
        t_now = max(all_ts) + 3600 if all_ts else 0

    results = {}
    for agent_id, times in agent_times.items():
        times.sort()
        liveness = AgentLiveness(
            agent_id=agent_id,
            last_seen=times[-1],
            message_count=len(times),
        )

        if len(times) < min_messages:
            liveness.state = 'unknown'
            results[agent_id] = liveness
            continue

        # Compute inter-arrival intervals
        intervals = [times[i+1] - times[i] for i in range(len(times) - 1)]
        mean_ival = sum(intervals) / len(intervals)
        variance = sum((x - mean_ival) ** 2 for x in intervals) / len(intervals)
        std_ival = math.sqrt(variance) if variance > 0 else mean_ival * 0.1

        liveness.mean_interval = mean_ival
        liveness.std_interval = std_ival

        # Time since last message
        t_diff = t_now - times[-1]

        if t_diff <= 0:
            liveness.phi = 0.0
            liveness.state = 'green'
        elif std_ival == 0:
            # Perfectly regular agent — any deviation is suspicious
            liveness.phi = 10.0 if t_diff > mean_ival * 1.5 else 0.0
            liveness.state = 'gray' if liveness.phi >= 3.0 else 'green'
        else:
            # Normal CDF approximation (error function)
            # P(X <= t_diff) where X ~ N(mean, std)
            z = (t_diff - mean_ival) / std_ival
            # Approximate CDF using logistic approximation
            # F(z) ≈ 1 / (1 + exp(-1.7 * z))
            try:
                cdf = 1.0 / (1.0 + math.exp(-1.7 * z))
            except OverflowError:
                cdf = 1.0 if z > 0 else 0.0

            # φ = -log10(1 - F(t_diff))
            if cdf >= 1.0 - 1e-15:
                liveness.phi = 16.0  # cap at 16 (probability < 1e-16)
            elif cdf <= 0.0:
                liveness.phi = 0.0
            else:
                liveness.phi = -math.log10(1.0 - cdf)

            # Classify
            if liveness.phi < 1.0:
                liveness.state = 'green'
            elif liveness.phi < 3.0:
                liveness.state = 'stale'
            else:
                liveness.state = 'gray'

        results[agent_id] = liveness

    return results