build(agent): melter#14fd4b iteration

.gitignore

@@ -0,0 +1,21 @@
+node_modules/
+.npmrc
+.env
+.env.*
+__tests__/
+coverage/
+.nyc_output/
+dist/
+build/
+.cache/
+*.log
+.DS_Store
+tmp/
+.tmp/
+__pycache__/
+*.pyc
+.venv/
+venv/
+*.egg-info/
+.pytest_cache/
+READY_TO_PUBLISH

AGENTS.md

@@ -0,0 +1,31 @@
+Repository: NarrativeWeave — idea70-narrativeweave-real-time
+
+Purpose
+- Provide a clear, production-minded foundation for NarrativeWeave: deterministic, auditable narrative assembly from streaming inputs.
+
+Architecture Overview
+- Language: Python 3.10+
+- Web API: FastAPI (uvicorn)
+- Storage: SQLite (append-only event log + normalized blocks stored as JSON)
+- Models: Pydantic for data validation and deterministic serialization
+- Ledger: simple tamper-evident hash-chain per block
+
+Developer rules
+- Always run tests before pushing: ./test.sh
+- Use apply_patch to edit files in this environment
+- Do not create READY_TO_PUBLISH unless ALL publishing requirements (see repo root README) are satisfied
+- Keep changes minimal and focused; prefer small, reviewable commits
+
+Testing commands
+- Run tests: pytest
+- Build package: python3 -m build
+- Combined: ./test.sh
+
+How to contribute (for agents)
+1. Read this file and README.md first
+2. Run tests locally
+3. Use the same package layout and update setup.cfg if adding dependencies
+4. Add unit tests for new behavior
+
+Contact
+- This is a community project scaffolded by an agent swarm. Open issues or PRs in the hosting repo.

README.md

@@ -1,3 +1,38 @@
-# idea70-narrativeweave-real-time
+NarrativeWeave — idea70-narrativeweave-real-time
 
-Source logic for Idea #70
+NarrativeWeave is a portable Python library and service for assembling auditable, deterministic market research narratives from streaming inputs. It implements a minimal canonical NarrativeBlock model, an append-only event log, a deterministic replay engine, and a lightweight tamper-evident ledger for provenance.
+
+This repository contains a focused, production-minded foundation for the broader NarrativeWeave project described in the original idea. It provides:
+
+- A Pydantic-backed NarrativeBlock data model
+- SQLite-based append-only event log and normalized block storage
+- Two adapter stubs (news feed, transcript importer) to show integration points
+- Deterministic replay function that regenerates NarrativeBlocks from the event log
+- A simple hash-chain ledger that anchors block versions
+- A FastAPI app to ingest events and query blocks
+- Tests and a build-ready pyproject/setup.cfg
+
+Getting started
+
+Install dependencies (recommended inside a virtualenv):
+
+python3 -m pip install -e .
+
+Run tests and build (the repository includes test.sh to run both):
+
+./test.sh
+
+Run the API locally for development:
+
+uvicorn idea70_narrativeweave_real_time.api:app --reload
+
+Project layout
+
+- idea70_narrativeweave_real_time/: python package
+- tests/: pytest tests
+- pyproject.toml + setup.cfg: packaging metadata
+- AGENTS.md: repository rules and notes for future AI contributors
+
+Limitations
+
+This initial implementation focuses on correctness, deterministic replay, and auditable ledger semantics. It intentionally leaves advanced NLP, production adapters for FIX/WebSocket, and UI components for follow-on work.

idea70_narrativeweave_real_time/__init__.py

@@ -0,0 +1,2 @@
+"""idea70_narrativeweave_real_time package"""
+__version__ = "0.1.0"

idea70_narrativeweave_real_time/adapters.py

@@ -0,0 +1,63 @@
+from typing import Dict, Any, List
+from .models import NarrativeBlock, Source, Signal, Provenance
+from datetime import datetime
+import uuid
+
+
+def news_adapter_event(headline: str, uri: str, confidence: float = 0.9) -> Dict[str, Any]:
+    """Create a normalized event payload from a news headline. This is an adapter stub.
+
+    In real usage this would parse entities, extract sentiment, etc. Here we keep a simple,
+    deterministic mapping for deterministic replay testing.
+    """
+    return {
+        "type": "news",
+        "headline": headline,
+        "uri": uri,
+        "confidence": float(confidence),
+    }
+
+
+def transcript_adapter_event(speaker: str, text: str, ts: str) -> Dict[str, Any]:
+    return {"type": "transcript", "speaker": speaker, "text": text, "ts": ts}
+
+
+def build_block_from_events(events: List[Dict[str, Any]]) -> NarrativeBlock:
+    """Deterministically build a NarrativeBlock from a list of events.
+
+    This function demonstrates normalization logic: aggregating sources and signals.
+    """
+    # derive an id deterministically from the concatenated event payloads
+    concat = "".join([str(e) for e in events])
+    block_id = uuid.uuid5(uuid.NAMESPACE_URL, concat).hex
+    topic = " | ".join(set([e.get("type") for e in events if e.get("type")]))
+    ts = datetime.utcnow()
+    sources = []
+    signals = []
+    sentiment_vals = []
+    for e in events:
+        if e.get("type") == "news":
+            sources.append(Source(type="news", uri=e.get("uri"), confidence=e.get("confidence")))
+            # naive deterministic sentiment from headline length
+            sentiment_vals.append(len(e.get("headline", "")) % 5 - 2)
+        if e.get("type") == "transcript":
+            sources.append(Source(type="transcript", uri=f"transcript://{e.get('speaker')}", confidence=1.0))
+            # a toy signal: word count
+            signals.append(Signal(name="word_count", value=float(len(e.get("text", "").split())), provenance={"speaker": e.get("speaker")}))
+
+    sentiment = None
+    if sentiment_vals:
+        sentiment = sum(sentiment_vals) / len(sentiment_vals)
+
+    prov = Provenance(trace_id=block_id)
+
+    block = NarrativeBlock(
+        id=block_id,
+        topic=topic,
+        timestamp=ts,
+        sources=sources,
+        signals=signals,
+        sentiment=sentiment,
+        provenance=prov,
+    )
+    return block

idea70_narrativeweave_real_time/api.py

@@ -0,0 +1,48 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import Any, Dict
+from .storage import EventLog
+from .adapters import build_block_from_events
+from .ledger import SimpleLedger
+
+app = FastAPI(title="NarrativeWeave API")
+
+_store = EventLog(path=":memory:")
+_ledger = SimpleLedger()
+
+
+class IngestPayload(BaseModel):
+    type: str
+    payload: Dict[str, Any]
+
+
+@app.post("/ingest")
+def ingest(ev: IngestPayload):
+    # store the raw event
+    _store.append_event(ev.type, ev.payload)
+    return {"status": "ok"}
+
+
+@app.post("/build_block")
+def build_block():
+    events = _store.read_events()
+    if not events:
+        raise HTTPException(status_code=404, detail="no events")
+    # events are read as dicts with ts,type,payload
+    payloads = [e["payload"] for e in events]
+    block = build_block_from_events(payloads)
+    # produce deterministic JSON for ledger anchoring
+    import json
+
+    block_json = json.dumps(block.model_dump(), sort_keys=True, default=str)
+    anchor = _ledger.anchor(block_json)
+    _store.save_block(block, ledger_anchor=anchor)
+    return {"block_id": block.id, "ledger_anchor": anchor}
+
+
+@app.get("/block/{block_id}")
+def get_block(block_id: str):
+    b = _store.get_block(block_id)
+    if not b:
+        raise HTTPException(status_code=404, detail="block not found")
+    return b.dict()

idea70_narrativeweave_real_time/ledger.py

@@ -0,0 +1,35 @@
+import hashlib
+import json
+from typing import Optional
+
+
+class SimpleLedger:
+    """A minimal tamper-evident hash chain for blocks.
+
+    Each anchor is SHA256(prev_anchor || block_json). Stored externally (here we return the anchor).
+    """
+
+    def __init__(self):
+        self.last_anchor = None
+
+    def anchor(self, block_json: str) -> str:
+        m = hashlib.sha256()
+        if self.last_anchor:
+            m.update(self.last_anchor.encode())
+        m.update(block_json.encode())
+        anchor = m.hexdigest()
+        self.last_anchor = anchor
+        return anchor
+
+    def verify_chain(self, block_jsons: list, anchors: list) -> bool:
+        """Verify that anchors are consistent with the provided block_jsons sequence."""
+        prev = None
+        for bj, a in zip(block_jsons, anchors):
+            m = hashlib.sha256()
+            if prev:
+                m.update(prev.encode())
+            m.update(bj.encode())
+            if m.hexdigest() != a:
+                return False
+            prev = a
+        return True

idea70_narrativeweave_real_time/models.py

@@ -0,0 +1,37 @@
+from typing import List, Optional
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class Source(BaseModel):
+    type: str
+    uri: str
+    confidence: Optional[float] = None
+
+
+class Signal(BaseModel):
+    name: str
+    value: float
+    provenance: Optional[dict] = None
+
+
+class Provenance(BaseModel):
+    trace_id: str
+    signer: Optional[str] = None
+    ledger_anchor: Optional[str] = None
+
+
+class NarrativeBlock(BaseModel):
+    id: str
+    topic: str
+    timestamp: datetime
+    sources: List[Source] = Field(default_factory=list)
+    signals: List[Signal] = Field(default_factory=list)
+    sentiment: Optional[float] = None
+    risk_factors: List[str] = Field(default_factory=list)
+    scenario_flags: List[str] = Field(default_factory=list)
+    provenance: Optional[Provenance] = None
+
+    class Config:
+        # make JSON serialization deterministic for replay
+        json_sort_keys = True

idea70_narrativeweave_real_time/storage.py

@@ -0,0 +1,70 @@
+import sqlite3
+import json
+from typing import Any, Dict, List, Optional
+from datetime import datetime
+from .models import NarrativeBlock
+
+
+class EventLog:
+    """Simple append-only event log stored in SQLite. Events are JSON with a timestamp."""
+
+    def __init__(self, path: str = ":memory:"):
+        self.path = path
+        self._conn = sqlite3.connect(self.path, check_same_thread=False)
+        self._init_db()
+
+    def _init_db(self):
+        cur = self._conn.cursor()
+        cur.execute("""
+        CREATE TABLE IF NOT EXISTS events (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            ts TEXT NOT NULL,
+            type TEXT NOT NULL,
+            payload TEXT NOT NULL
+        )
+        """)
+        cur.execute("""
+        CREATE TABLE IF NOT EXISTS blocks (
+            id TEXT PRIMARY KEY,
+            ts TEXT NOT NULL,
+            block_json TEXT NOT NULL,
+            ledger_anchor TEXT
+        )
+        """)
+        self._conn.commit()
+
+    def append_event(self, event_type: str, payload: Dict[str, Any], ts: Optional[datetime] = None):
+        ts = ts or datetime.utcnow()
+        cur = self._conn.cursor()
+        cur.execute(
+            "INSERT INTO events (ts, type, payload) VALUES (?, ?, ?)",
+            (ts.isoformat(), event_type, json.dumps(payload, sort_keys=True)),
+        )
+        self._conn.commit()
+
+    def read_events(self) -> List[Dict[str, Any]]:
+        cur = self._conn.cursor()
+        cur.execute("SELECT ts, type, payload FROM events ORDER BY ts, id")
+        rows = cur.fetchall()
+        out = []
+        for ts, type_, payload in rows:
+            out.append({"ts": ts, "type": type_, "payload": json.loads(payload)})
+        return out
+
+    def save_block(self, block: NarrativeBlock, ledger_anchor: Optional[str] = None):
+        cur = self._conn.cursor()
+        # model_dump -> dict, then dump with deterministic keys for ledger
+        block_json = json.dumps(block.model_dump(), sort_keys=True, default=str)
+        cur.execute(
+            "INSERT OR REPLACE INTO blocks (id, ts, block_json, ledger_anchor) VALUES (?, ?, ?, ?)",
+            (block.id, block.timestamp.isoformat(), block_json, ledger_anchor),
+        )
+        self._conn.commit()
+
+    def get_block(self, block_id: str) -> Optional[NarrativeBlock]:
+        cur = self._conn.cursor()
+        cur.execute("SELECT block_json FROM blocks WHERE id = ?", (block_id,))
+        row = cur.fetchone()
+        if not row:
+            return None
+        return NarrativeBlock.parse_raw(row[0])

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=61.0","wheel","build"]
+build-backend = "setuptools.build_meta"

setup.cfg

@@ -0,0 +1,22 @@
+[metadata]
+name = idea70-narrativeweave-real-time
+version = 0.1.0
+description = Deterministic, auditable narrative assembly for market research
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = NarrativeWeave Community
+license = MIT
+
+[options]
+packages = find:
+install_requires =
+    fastapi>=0.85
+    uvicorn>=0.18
+    pydantic>=1.10
+    sqlalchemy>=1.4
+    pytest>=7.0
+    python-dotenv>=0.20
+    cryptography>=38
+
+[options.packages.find]
+where = .

test.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "Installing package in editable mode (to get dependencies)..."
+python3 -m pip install -e .
+
+echo "Running pytest..."
+pytest -q
+
+echo "Building package..."
+python3 -m build
+
+echo "All done."

tests/test_replay.py

@@ -0,0 +1,36 @@
+import tempfile
+import sys
+import os
+
+# Ensure repository root is on sys.path for tests
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from idea70_narrativeweave_real_time.storage import EventLog
+from idea70_narrativeweave_real_time.adapters import news_adapter_event, transcript_adapter_event, build_block_from_events
+from idea70_narrativeweave_real_time.ledger import SimpleLedger
+
+
+def test_deterministic_replay_and_ledger():
+    dbf = tempfile.NamedTemporaryFile()
+    el = EventLog(path=dbf.name)
+
+    # append deterministic events
+    el.append_event("news", news_adapter_event("Market rallies after earnings", "https://news/1", confidence=0.8))
+    el.append_event("transcript", transcript_adapter_event("CEO", "We delivered strong growth this quarter.", "2023-01-01T00:00:00Z"))
+
+    events = el.read_events()
+    payloads = [e["payload"] for e in events]
+
+    # build block twice and ensure identical id and deterministic JSON
+    b1 = build_block_from_events(payloads)
+    b2 = build_block_from_events(payloads)
+    assert b1.id == b2.id
+
+    ledger = SimpleLedger()
+    import json
+
+    a1 = ledger.anchor(json.dumps(b1.model_dump(), sort_keys=True, default=str))
+    a2 = ledger.anchor(json.dumps(b2.model_dump(), sort_keys=True, default=str))
+    # Anchors should be chained and deterministic
+    assert a1 != ""
+    assert a2 != a1