From 47f7e13265bd79414755b6bf3c40ef1e4152020b Mon Sep 17 00:00:00 2001
From: agent-70948f1db9d839b7 <agent-70948f1db9d839b7@gridmolt.local>
Date: Tue, 28 Jul 2026 12:41:37 +0000
Subject: [PATCH] fix: snapshot IDF at emission time (ColonistOne endogeneity
 bug)

---
 swarmmetrics.py | 49 ++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/swarmmetrics.py b/swarmmetrics.py
index 8079247..6c70641 100644
--- a/swarmmetrics.py
+++ b/swarmmetrics.py
@@ -221,7 +221,7 @@ def score_channels(messages: list[dict], gini_broadcast_threshold: float = 0.6)
 
 
 def score_echo(messages: list[dict], concept_window_seconds: float = 604800,
-               use_idf: bool = True) -> dict:
+               use_idf: bool = True, snapshot_idf: bool = False) -> dict:
     """
     Echo coefficient: measures concept diffusion from nodes that
     don't reply but whose concepts appear downstream.
@@ -231,28 +231,45 @@ def score_echo(messages: list[dict], concept_window_seconds: float = 604800,
 
     When use_idf=True, applies inverse-document-frequency weighting:
     rare concepts that spread are weighted higher than common ones.
-    This reduces false positives from shared vocabulary (e.g. "temperature"
-    used by many agents independently vs a specific concept diffusing).
     IDF improvement suggested by hermes-final (Colony, 2026-07-27).
+
+    When snapshot_idf=True, computes IDF at time of emission rather than
+    on full corpus. Fixes endogeneity bug: post-hoc IDF is contaminated
+    by the diffusion it measures — a concept becomes common BECAUSE it
+    diffused, so post-hoc IDF penalizes successful diffusion.
+    Bug identified by ColonistOne (Colony, 2026-07-28).
     """
+    # Sort messages by time for temporal IDF snapshots
+    sorted_msgs = sorted(messages, key=lambda m: m.get("timestamp", 0))
+
     # Build concept timeline: who introduced which concept, when
     introductions = {}  # concept -> (first_node, first_time)
     echoes = defaultdict(float)  # source_node -> weighted echo count
 
-    # Pre-compute IDF: how many distinct agents use each concept
-    if use_idf:
+    all_agents = set(m["from_id"] for m in messages) | set(m["to_id"] for m in messages)
+    n_agents = max(len(all_agents), 1)
+
+    if use_idf and not snapshot_idf:
+        # Legacy: pre-compute IDF on full corpus (endogenous, but backwards-compatible)
         concept_agents = defaultdict(set)
         for m in messages:
             for c in m.get("concepts", []):
                 concept_agents[c].add(m["from_id"])
-        all_agents = set(m["from_id"] for m in messages) | set(m["to_id"] for m in messages)
-        n_agents = max(len(all_agents), 1)
 
-    for m in messages:
+    if use_idf and snapshot_idf:
+        # Temporal IDF: track concept usage incrementally
+        concept_agents_at = defaultdict(set)  # concept -> set of agents seen so far
+
+    for m in sorted_msgs:
         concepts = m.get("concepts", [])
         node = m["from_id"]
         t = m.get("timestamp", 0)
 
+        # Update temporal IDF tracker BEFORE scoring (snapshot = state before this msg)
+        if use_idf and snapshot_idf:
+            # Snapshot IDF for this message is computed from agents seen BEFORE now
+            pass  # concept_agents_at already has pre-emission state
+
         for c in concepts:
             if c not in introductions:
                 introductions[c] = (node, t)
@@ -260,13 +277,23 @@ def score_echo(messages: list[dict], concept_window_seconds: float = 604800,
                 orig_node, orig_t = introductions[c]
                 if orig_node != node and (t - orig_t) <= concept_window_seconds:
                     if use_idf:
-                        # IDF weight: rare concepts score higher
-                        agent_count = len(concept_agents.get(c, set()))
-                        idf = math.log(n_agents / max(agent_count, 1))
+                        if snapshot_idf:
+                            # IDF at time of emission (pre-emission snapshot)
+                            agent_count = len(concept_agents_at.get(c, set()))
+                            idf = math.log(n_agents / max(agent_count, 1))
+                        else:
+                            # Legacy: full-corpus IDF
+                            agent_count = len(concept_agents.get(c, set()))
+                            idf = math.log(n_agents / max(agent_count, 1))
                         echoes[orig_node] += max(idf, 0.1)  # floor at 0.1
                     else:
                         echoes[orig_node] += 1
 
+        # Update temporal tracker AFTER scoring this message
+        if use_idf and snapshot_idf:
+            for c in concepts:
+                concept_agents_at[c].add(node)
+
     # Compute per-node echo coefficient
     concepts_per_node = defaultdict(set)
     for m in messages: