build(agent): c3po#b883b4 iteration

.gitignore

@@ -0,0 +1,21 @@
+node_modules/
+.npmrc
+.env
+.env.*
+__tests__/
+coverage/
+.nyc_output/
+dist/
+build/
+.cache/
+*.log
+.DS_Store
+tmp/
+.tmp/
+__pycache__/
+*.pyc
+.venv/
+venv/
+*.egg-info/
+.pytest_cache/
+READY_TO_PUBLISH

AGENTS.md

@@ -0,0 +1,34 @@
+# CivicSwarm Agent Guide
+
+## Architecture
+
+- `civicswarm/db.py` defines the SQLite schema and engine setup.
+- `civicswarm/analysis.py` contains TF-IDF clustering, extractive summarization, language detection, and lightweight sentiment heuristics.
+- `civicswarm/service.py` is the domain layer for proposals, comments, routing, privacy-preserving preference aggregation, the deliberation ledger, dashboard metrics, and civic brief export.
+- `civicswarm/api.py` exposes the service through FastAPI.
+
+## Tech Stack
+
+- Python 3.11
+- FastAPI for the HTTP surface
+- SQLAlchemy Core with SQLite for persistence
+- scikit-learn and numpy for topic clustering and routing similarity
+- langdetect for best-effort multilingual handling
+
+## Working Rules
+
+- Keep resident identity pseudonymous. Use `resident_key` values only; do not add raw identity storage.
+- Prefer the smallest correct change.
+- Keep logic deterministic where possible so tests remain stable.
+- Add or update tests for every behavior change.
+- Do not remove or rewrite unrelated files.
+
+## Testing
+
+- Install dependencies and run the suite: `bash test.sh`
+- `test.sh` must succeed with `pytest` and `python3 -m build`.
+
+## Packaging
+
+- Distribution name: `idea198-civicswarm-privacy-preserving`
+- README must stay current because it is wired into package metadata.

README.md

@@ -1,3 +1,69 @@
-# idea198-civicswarm-privacy-preserving
+# CivicSwarm
 
-Source logic for Idea #198
+Privacy-preserving neighborhood deliberation router for participatory politics.
+
+## What It Does
+
+CivicSwarm stores proposals, comments, resident profiles, preference signals, and deliberation ledger entries in SQLite. It then:
+
+- clusters comments with TF-IDF + KMeans
+- summarizes discussion by detected language
+- routes proposal fragments to relevant residents by geography, interests, experience, and language
+- aggregates preferences with optional differential-privacy noise
+- exports a civic brief with provenance and audit trail
+
+## Stack
+
+- Python 3.11
+- FastAPI
+- SQLAlchemy Core + SQLite
+- scikit-learn
+- langdetect
+
+## Install
+
+```bash
+python3 -m pip install -e .
+```
+
+For development:
+
+```bash
+python3 -m pip install -e ".[dev]"
+```
+
+## Run
+
+```bash
+python -m civicswarm
+```
+
+Environment variables:
+
+- `CIVICSWARM_DB_PATH` defaults to `civicswarm.sqlite`
+- `CIVICSWARM_HOST` defaults to `127.0.0.1`
+- `CIVICSWARM_PORT` defaults to `8000`
+
+## API
+
+- `POST /proposals`
+- `POST /residents`
+- `POST /proposals/{proposal_id}/comments`
+- `POST /proposals/{proposal_id}/preferences`
+- `GET /proposals/{proposal_id}/route`
+- `GET /proposals/{proposal_id}/dashboard`
+- `GET /proposals/{proposal_id}/brief`
+- `GET /proposals/{proposal_id}/ledger`
+
+## Testing
+
+```bash
+bash test.sh
+```
+
+That runs `pytest` and `python3 -m build`.
+
+## Notes
+
+- Resident identity is kept pseudonymous through `resident_key`.
+- The current codebase is backend-first and designed for mobile, SMS, and field-capture integrations.

civicswarm/__init__.py

@@ -0,0 +1,6 @@
+"""CivicSwarm package."""
+
+from .api import create_app
+from .service import CivicSwarmService
+
+__all__ = ["CivicSwarmService", "create_app"]

civicswarm/__main__.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import os
+
+import uvicorn
+
+from .api import create_app
+
+
+def main() -> None:
+    db_path = os.environ.get("CIVICSWARM_DB_PATH", "civicswarm.sqlite")
+    app = create_app(db_path)
+    uvicorn.run(app, host=os.environ.get("CIVICSWARM_HOST", "127.0.0.1"), port=int(os.environ.get("CIVICSWARM_PORT", "8000")))
+
+
+if __name__ == "__main__":
+    main()

civicswarm/analysis.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from typing import Iterable
+
+import numpy as np
+from langdetect import DetectorFactory, detect
+from sklearn.cluster import KMeans
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+
+DetectorFactory.seed = 0
+
+SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+")
+WORD_RE = re.compile(r"[A-Za-zÀ-ÿ0-9']+")
+
+POSITIVE_WORDS = {
+    "support",
+    "approve",
+    "agree",
+    "welcome",
+    "benefit",
+    "safe",
+    "fair",
+    "help",
+    "yes",
+}
+
+NEGATIVE_WORDS = {
+    "oppose",
+    "reject",
+    "harm",
+    "unsafe",
+    "unfair",
+    "no",
+    "risk",
+    "delay",
+    "concern",
+    "object",
+}
+
+
+def detect_language(text: str) -> str:
+    sample = text.strip()
+    if len(sample) < 10:
+        return "und"
+    try:
+        return detect(sample)
+    except Exception:
+        return "und"
+
+
+def tokenize(text: str) -> list[str]:
+    return [token.lower() for token in WORD_RE.findall(text)]
+
+
+def split_sentences(text: str) -> list[str]:
+    parts = [part.strip() for part in SENTENCE_SPLIT.split(text) if part.strip()]
+    return parts or ([text.strip()] if text.strip() else [])
+
+
+def sentence_centrality_summary(texts: Iterable[str], max_sentences: int = 3) -> list[str]:
+    sentences: list[str] = []
+    for text in texts:
+        sentences.extend(split_sentences(text))
+
+    unique_sentences: list[str] = []
+    seen = set()
+    for sentence in sentences:
+        key = sentence.lower()
+        if key not in seen:
+            seen.add(key)
+            unique_sentences.append(sentence)
+
+    if not unique_sentences:
+        return []
+    if len(unique_sentences) <= max_sentences:
+        return unique_sentences
+
+    vectorizer = TfidfVectorizer(stop_words="english")
+    matrix = vectorizer.fit_transform(unique_sentences)
+    similarity = (matrix * matrix.T).toarray()
+    scores = similarity.sum(axis=1)
+    ranked = sorted(range(len(unique_sentences)), key=lambda idx: (-scores[idx], idx))[:max_sentences]
+    return [unique_sentences[idx] for idx in sorted(ranked)]
+
+
+def summarize_multilingual_comments(comments: list[dict], max_sentences: int = 3) -> dict[str, list[str]]:
+    by_language: dict[str, list[str]] = defaultdict(list)
+    for comment in comments:
+        language = comment.get("language") or detect_language(comment.get("text", ""))
+        by_language[language].append(comment.get("text", ""))
+
+    return {
+        language: sentence_centrality_summary(texts, max_sentences=max_sentences)
+        for language, texts in sorted(by_language.items())
+    }
+
+
+def cluster_texts(texts: list[str]) -> list[dict]:
+    if not texts:
+        return []
+    if len(texts) == 1:
+        return [{"cluster": 0, "items": [texts[0]], "top_terms": tokenize(texts[0])[:5]}]
+
+    vectorizer = TfidfVectorizer(stop_words="english")
+    matrix = vectorizer.fit_transform(texts)
+    n_clusters = max(1, min(int(math.sqrt(len(texts))) or 1, 4))
+    if n_clusters == 1:
+        return [{"cluster": 0, "items": texts, "top_terms": _top_terms(matrix, vectorizer)}]
+
+    model = KMeans(n_clusters=n_clusters, n_init=10, random_state=0)
+    labels = model.fit_predict(matrix)
+    clusters: dict[int, list[str]] = defaultdict(list)
+    for label, text in zip(labels, texts):
+        clusters[int(label)].append(text)
+
+    return [
+        {"cluster": cluster_id, "items": items, "top_terms": _top_terms(matrix[[i for i, label in enumerate(labels) if label == cluster_id]], vectorizer)}
+        for cluster_id, items in sorted(clusters.items())
+    ]
+
+
+def _top_terms(matrix, vectorizer: TfidfVectorizer, limit: int = 5) -> list[str]:
+    if matrix.shape[0] == 0:
+        return []
+    averaged = np.asarray(matrix.mean(axis=0)).ravel()
+    terms = np.array(vectorizer.get_feature_names_out())
+    top_indices = averaged.argsort()[::-1][:limit]
+    return [str(terms[index]) for index in top_indices if averaged[index] > 0]
+
+
+def sentiment_score(text: str) -> float:
+    tokens = tokenize(text)
+    if not tokens:
+        return 0.0
+    positive = sum(1 for token in tokens if token in POSITIVE_WORDS)
+    negative = sum(1 for token in tokens if token in NEGATIVE_WORDS)
+    return (positive - negative) / max(1, len(tokens))
+
+
+def laplace_noise(scale: float, seed: int | None = None) -> float:
+    rng = np.random.default_rng(seed)
+    return float(rng.laplace(0.0, scale))

civicswarm/api.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel, Field
+
+from .service import CivicSwarmService
+
+
+class ProposalCreate(BaseModel):
+    city: str
+    title: str
+    body: str
+    geography: str = ""
+    tags: list[str] = Field(default_factory=list)
+
+
+class ResidentCreate(BaseModel):
+    resident_key: str
+    geography: str = ""
+    interests: list[str] = Field(default_factory=list)
+    lived_experience: list[str] = Field(default_factory=list)
+    languages: list[str] = Field(default_factory=list)
+
+
+class CommentCreate(BaseModel):
+    text: str
+    channel: str = "mobile"
+    resident_key: str | None = None
+    geography: str = ""
+    metadata: dict = Field(default_factory=dict)
+
+
+class PreferenceCreate(BaseModel):
+    resident_key: str
+    score: float
+    channel: str = "mobile"
+
+
+def create_app(db_path: str | Path = ":memory:") -> FastAPI:
+    service = CivicSwarmService.from_path(db_path)
+    app = FastAPI(title="CivicSwarm", version="0.1.0")
+    app.state.service = service
+
+    @app.post("/proposals")
+    def create_proposal(payload: ProposalCreate):
+        return service.create_proposal(payload.city, payload.title, payload.body, payload.geography, payload.tags)
+
+    @app.post("/residents")
+    def create_resident(payload: ResidentCreate):
+        return service.register_resident(payload.resident_key, payload.geography, payload.interests, payload.lived_experience, payload.languages)
+
+    @app.post("/proposals/{proposal_id}/comments")
+    def create_comment(proposal_id: int, payload: CommentCreate):
+        try:
+            return service.add_comment(proposal_id, payload.text, payload.channel, payload.resident_key, payload.geography, payload.metadata)
+        except KeyError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+    @app.post("/proposals/{proposal_id}/preferences")
+    def create_preference(proposal_id: int, payload: PreferenceCreate):
+        return service.submit_preference(proposal_id, payload.resident_key, payload.score, payload.channel)
+
+    @app.get("/proposals/{proposal_id}/route")
+    def route_proposal(proposal_id: int, top_n: int = 3):
+        return service.route_proposal(proposal_id, top_n=top_n)
+
+    @app.get("/proposals/{proposal_id}/dashboard")
+    def dashboard(proposal_id: int):
+        return service.build_dashboard(proposal_id)
+
+    @app.get("/proposals/{proposal_id}/brief")
+    def brief(proposal_id: int):
+        return service.export_brief(proposal_id)
+
+    @app.get("/proposals/{proposal_id}/ledger")
+    def ledger(proposal_id: int):
+        return service.list_ledger(proposal_id)
+
+    return app

civicswarm/db.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy import (
+    JSON,
+    Column,
+    DateTime,
+    Float,
+    Integer,
+    MetaData,
+    String,
+    Table,
+    Text,
+    create_engine,
+    inspect,
+)
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+metadata = MetaData()
+
+proposals = Table(
+    "proposals",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("city", String, nullable=False),
+    Column("title", String, nullable=False),
+    Column("body", Text, nullable=False),
+    Column("status", String, nullable=False, default="draft"),
+    Column("geography", String, nullable=False, default=""),
+    Column("tags_json", Text, nullable=False, default="[]"),
+    Column("version", Integer, nullable=False, default=1),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+    Column("updated_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+proposal_versions = Table(
+    "proposal_versions",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("proposal_id", Integer, nullable=False),
+    Column("version", Integer, nullable=False),
+    Column("rationale", Text, nullable=False, default=""),
+    Column("objections_json", Text, nullable=False, default="[]"),
+    Column("resolution_status", String, nullable=False, default="open"),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+residents = Table(
+    "residents",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("resident_key", String, nullable=False, unique=True),
+    Column("geography", String, nullable=False, default=""),
+    Column("interests_json", Text, nullable=False, default="[]"),
+    Column("lived_experience_json", Text, nullable=False, default="[]"),
+    Column("languages_json", Text, nullable=False, default="[]"),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+    Column("updated_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+comments = Table(
+    "comments",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("proposal_id", Integer, nullable=False),
+    Column("resident_key", String, nullable=True),
+    Column("channel", String, nullable=False),
+    Column("geography", String, nullable=False, default=""),
+    Column("language", String, nullable=False, default="und"),
+    Column("text", Text, nullable=False),
+    Column("metadata_json", Text, nullable=False, default="{}"),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+preferences = Table(
+    "preferences",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("proposal_id", Integer, nullable=False),
+    Column("resident_key", String, nullable=False),
+    Column("score", Float, nullable=False),
+    Column("channel", String, nullable=False),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+routes = Table(
+    "routes",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("proposal_id", Integer, nullable=False),
+    Column("fragment_index", Integer, nullable=False),
+    Column("resident_key", String, nullable=False),
+    Column("score", Float, nullable=False),
+    Column("rationale_json", Text, nullable=False, default="[]"),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+ledger_entries = Table(
+    "ledger_entries",
+    metadata,
+    Column("id", Integer, primary_key=True),
+    Column("proposal_id", Integer, nullable=False),
+    Column("kind", String, nullable=False),
+    Column("payload_json", Text, nullable=False),
+    Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow),
+)
+
+
+@dataclass(frozen=True)
+class Database:
+    engine: object
+
+
+def create_database(db_url: str):
+    engine = create_engine(db_url, future=True)
+    metadata.create_all(engine)
+    return Database(engine=engine)
+
+
+def sqlite_path_url(path: str | Path) -> str:
+    p = Path(path)
+    if str(p) == ":memory:":
+        return "sqlite+pysqlite:///:memory:"
+    return f"sqlite+pysqlite:///{p}"

civicswarm/service.py

@@ -0,0 +1,505 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import math
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+import numpy as np
+from sqlalchemy import delete, func, insert, select, update
+
+from .analysis import (
+    cluster_texts,
+    detect_language,
+    laplace_noise,
+    sentiment_score,
+    summarize_multilingual_comments,
+    tokenize,
+)
+from .db import (
+    comments,
+    create_database,
+    ledger_entries,
+    preferences,
+    proposal_versions,
+    proposals,
+    residents,
+    routes,
+    sqlite_path_url,
+)
+
+
+def _json(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, sort_keys=True)
+
+
+def _parse_json(value: str | None, default: Any) -> Any:
+    if not value:
+        return default
+    return json.loads(value)
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _normalize_terms(values: Iterable[str]) -> set[str]:
+    terms: set[str] = set()
+    for value in values:
+        terms.update(tokenize(value))
+    return terms
+
+
+@dataclass
+class CivicSwarmService:
+    db_url: str
+    secret: str = "civicswarm"
+
+    def __post_init__(self) -> None:
+        self.db = create_database(self.db_url)
+
+    @classmethod
+    def from_path(cls, path: str | Path, secret: str = "civicswarm") -> "CivicSwarmService":
+        return cls(db_url=sqlite_path_url(path), secret=secret)
+
+    def _token(self, resident_key: str | None) -> str | None:
+        if resident_key is None:
+            return None
+        return hmac.new(self.secret.encode(), resident_key.encode(), hashlib.sha256).hexdigest()
+
+    def create_proposal(self, city: str, title: str, body: str, geography: str = "", tags: list[str] | None = None) -> dict[str, Any]:
+        tags = tags or []
+        with self.db.engine.begin() as conn:
+            result = conn.execute(
+                insert(proposals).values(
+                    city=city,
+                    title=title,
+                    body=body,
+                    geography=geography,
+                    tags_json=_json(tags),
+                    status="draft",
+                    version=1,
+                    created_at=_now(),
+                    updated_at=_now(),
+                )
+            )
+            proposal_id = result.inserted_primary_key[0]
+            conn.execute(
+                insert(ledger_entries).values(
+                    proposal_id=proposal_id,
+                    kind="proposal_created",
+                    payload_json=_json({"title": title, "tags": tags, "geography": geography}),
+                    created_at=_now(),
+                )
+            )
+            conn.execute(
+                insert(proposal_versions).values(
+                    proposal_id=proposal_id,
+                    version=1,
+                    rationale="initial proposal",
+                    objections_json=_json([]),
+                    resolution_status="open",
+                    created_at=_now(),
+                )
+            )
+        return self.get_proposal(proposal_id)
+
+    def get_proposal(self, proposal_id: int) -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            row = conn.execute(select(proposals).where(proposals.c.id == proposal_id)).mappings().first()
+        if row is None:
+            raise KeyError(f"proposal {proposal_id} not found")
+        return self._proposal_dict(row)
+
+    def list_comments(self, proposal_id: int) -> list[dict[str, Any]]:
+        with self.db.engine.begin() as conn:
+            rows = conn.execute(select(comments).where(comments.c.proposal_id == proposal_id).order_by(comments.c.id.asc())).mappings().all()
+        return [self._comment_dict(row) for row in rows]
+
+    def register_resident(
+        self,
+        resident_key: str,
+        geography: str = "",
+        interests: list[str] | None = None,
+        lived_experience: list[str] | None = None,
+        languages: list[str] | None = None,
+    ) -> dict[str, Any]:
+        interests = interests or []
+        lived_experience = lived_experience or []
+        languages = languages or []
+        with self.db.engine.begin() as conn:
+            existing = conn.execute(select(residents).where(residents.c.resident_key == resident_key)).mappings().first()
+            values = dict(
+                resident_key=resident_key,
+                geography=geography,
+                interests_json=_json(interests),
+                lived_experience_json=_json(lived_experience),
+                languages_json=_json(languages),
+                updated_at=_now(),
+            )
+            if existing is None:
+                values["created_at"] = _now()
+                conn.execute(insert(residents).values(**values))
+            else:
+                conn.execute(update(residents).where(residents.c.resident_key == resident_key).values(**values))
+        return self.get_resident(resident_key)
+
+    def get_resident(self, resident_key: str) -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            row = conn.execute(select(residents).where(residents.c.resident_key == resident_key)).mappings().first()
+        if row is None:
+            raise KeyError(f"resident {resident_key} not found")
+        return self._resident_dict(row)
+
+    def add_comment(
+        self,
+        proposal_id: int,
+        text: str,
+        channel: str = "mobile",
+        resident_key: str | None = None,
+        geography: str = "",
+        metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        metadata = metadata or {}
+        language = detect_language(text)
+        with self.db.engine.begin() as conn:
+            result = conn.execute(
+                insert(comments).values(
+                    proposal_id=proposal_id,
+                    resident_key=self._token(resident_key),
+                    channel=channel,
+                    geography=geography,
+                    language=language,
+                    text=text,
+                    metadata_json=_json(metadata),
+                    created_at=_now(),
+                )
+            )
+            comment_id = result.inserted_primary_key[0]
+            conn.execute(
+                insert(ledger_entries).values(
+                    proposal_id=proposal_id,
+                    kind="comment_received",
+                    payload_json=_json({"comment_id": comment_id, "channel": channel, "language": language}),
+                    created_at=_now(),
+                )
+            )
+        return self.get_comment(comment_id)
+
+    def get_comment(self, comment_id: int) -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            row = conn.execute(select(comments).where(comments.c.id == comment_id)).mappings().first()
+        if row is None:
+            raise KeyError(f"comment {comment_id} not found")
+        return self._comment_dict(row)
+
+    def submit_preference(self, proposal_id: int, resident_key: str, score: float, channel: str = "mobile") -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            result = conn.execute(
+                insert(preferences).values(
+                    proposal_id=proposal_id,
+                    resident_key=self._token(resident_key) or resident_key,
+                    score=float(score),
+                    channel=channel,
+                    created_at=_now(),
+                )
+            )
+            preference_id = result.inserted_primary_key[0]
+            conn.execute(
+                insert(ledger_entries).values(
+                    proposal_id=proposal_id,
+                    kind="preference_submitted",
+                    payload_json=_json({"preference_id": preference_id, "channel": channel}),
+                    created_at=_now(),
+                )
+            )
+        return {"id": preference_id, "proposal_id": proposal_id, "resident_key": resident_key, "score": float(score), "channel": channel}
+
+    def aggregate_preferences(self, proposal_id: int, epsilon: float | None = None, seed: int = 0) -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            rows = conn.execute(select(preferences.c.score).where(preferences.c.proposal_id == proposal_id)).all()
+        scores = [float(row[0]) for row in rows]
+        count = len(scores)
+        total = float(sum(scores))
+        mean = total / count if count else 0.0
+        if epsilon and epsilon > 0:
+            scale = 1.0 / float(epsilon)
+            total += laplace_noise(scale, seed=seed)
+            count = max(1, int(round(count + laplace_noise(scale / 2.0, seed=seed + 1))))
+            mean = total / count
+        return {"proposal_id": proposal_id, "count": count, "mean_score": mean, "raw_total": float(sum(scores))}
+
+    def cluster_comments(self, proposal_id: int) -> list[dict[str, Any]]:
+        comments_data = self.list_comments(proposal_id)
+        return cluster_texts([comment["text"] for comment in comments_data])
+
+    def summarize_comments(self, proposal_id: int, max_sentences: int = 3) -> dict[str, list[str]]:
+        comments_data = self.list_comments(proposal_id)
+        return summarize_multilingual_comments(comments_data, max_sentences=max_sentences)
+
+    def route_proposal(self, proposal_id: int, top_n: int = 3) -> dict[str, Any]:
+        proposal = self.get_proposal(proposal_id)
+        fragments = self._proposal_fragments(proposal["body"])
+        with self.db.engine.begin() as conn:
+            resident_rows = conn.execute(select(residents)).mappings().all()
+
+        routed_fragments = []
+        for index, fragment in enumerate(fragments):
+            ranked = self._rank_residents_for_fragment(proposal, fragment, resident_rows)
+            top_matches = ranked[:top_n]
+            routed_fragments.append({"fragment_index": index, "fragment": fragment, "matches": top_matches})
+            with self.db.engine.begin() as conn:
+                conn.execute(delete(routes).where(routes.c.proposal_id == proposal_id, routes.c.fragment_index == index))
+                for match in top_matches:
+                    conn.execute(
+                        insert(routes).values(
+                            proposal_id=proposal_id,
+                            fragment_index=index,
+                            resident_key=match["resident_key"],
+                            score=match["score"],
+                            rationale_json=_json(match["reasons"]),
+                            created_at=_now(),
+                        )
+                    )
+        return {"proposal_id": proposal_id, "fragments": routed_fragments}
+
+    def build_dashboard(self, proposal_id: int) -> dict[str, Any]:
+        proposal = self.get_proposal(proposal_id)
+        comments_data = self.list_comments(proposal_id)
+        summaries = self.summarize_comments(proposal_id)
+        preferences_summary = self.aggregate_preferences(proposal_id)
+        routes_summary = self.route_proposal(proposal_id)
+
+        consensus_pockets = []
+        unresolved_conflicts = []
+        outreach_gaps = []
+
+        for fragment in routes_summary["fragments"]:
+            fragment_text = fragment["fragment"]
+            fragment_comments = [comment for comment in comments_data if any(term in comment["text"].lower() for term in tokenize(fragment_text))]
+            sentiment_values = [sentiment_score(comment["text"]) for comment in fragment_comments]
+            if sentiment_values and np.mean(sentiment_values) > 0:
+                consensus_pockets.append({"fragment_index": fragment["fragment_index"], "fragment": fragment_text, "support": float(np.mean(sentiment_values))})
+            if sentiment_values and min(sentiment_values) < 0 < max(sentiment_values):
+                unresolved_conflicts.append({"fragment_index": fragment["fragment_index"], "fragment": fragment_text, "support": float(np.mean(sentiment_values)), "spread": float(max(sentiment_values) - min(sentiment_values))})
+            if not fragment["matches"]:
+                outreach_gaps.append({"fragment_index": fragment["fragment_index"], "fragment": fragment_text})
+
+        return {
+            "proposal": proposal,
+            "summaries": summaries,
+            "preference_aggregate": preferences_summary,
+            "consensus_pockets": consensus_pockets,
+            "unresolved_conflicts": unresolved_conflicts,
+            "outreach_gaps": outreach_gaps,
+        }
+
+    def record_ledger_entry(self, proposal_id: int, kind: str, payload: dict[str, Any]) -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            result = conn.execute(
+                insert(ledger_entries).values(
+                    proposal_id=proposal_id,
+                    kind=kind,
+                    payload_json=_json(payload),
+                    created_at=_now(),
+                )
+            )
+        return {"id": result.inserted_primary_key[0], "proposal_id": proposal_id, "kind": kind, "payload": payload}
+
+    def record_proposal_version(self, proposal_id: int, rationale: str, objections: list[str], resolution_status: str) -> dict[str, Any]:
+        with self.db.engine.begin() as conn:
+            current = conn.execute(
+                select(func.max(proposal_versions.c.version)).where(proposal_versions.c.proposal_id == proposal_id)
+            ).scalar_one_or_none()
+            version = int(current or 0) + 1
+            result = conn.execute(
+                insert(proposal_versions).values(
+                    proposal_id=proposal_id,
+                    version=version,
+                    rationale=rationale,
+                    objections_json=_json(objections),
+                    resolution_status=resolution_status,
+                    created_at=_now(),
+                )
+            )
+            conn.execute(
+                update(proposals)
+                .where(proposals.c.id == proposal_id)
+                .values(version=version, updated_at=_now(), status=resolution_status)
+            )
+            conn.execute(
+                insert(ledger_entries).values(
+                    proposal_id=proposal_id,
+                    kind="proposal_version_recorded",
+                    payload_json=_json({"version": version, "resolution_status": resolution_status}),
+                    created_at=_now(),
+                )
+            )
+        return {"id": result.inserted_primary_key[0], "proposal_id": proposal_id, "version": version, "resolution_status": resolution_status}
+
+    def list_proposal_versions(self, proposal_id: int) -> list[dict[str, Any]]:
+        with self.db.engine.begin() as conn:
+            rows = conn.execute(
+                select(proposal_versions).where(proposal_versions.c.proposal_id == proposal_id).order_by(proposal_versions.c.version.asc())
+            ).mappings().all()
+        return [
+            {
+                "id": row["id"],
+                "proposal_id": row["proposal_id"],
+                "version": row["version"],
+                "rationale": row["rationale"],
+                "objections": _parse_json(row["objections_json"], []),
+                "resolution_status": row["resolution_status"],
+                "created_at": row["created_at"].isoformat(),
+            }
+            for row in rows
+        ]
+
+    def list_ledger(self, proposal_id: int) -> list[dict[str, Any]]:
+        with self.db.engine.begin() as conn:
+            rows = conn.execute(select(ledger_entries).where(ledger_entries.c.proposal_id == proposal_id).order_by(ledger_entries.c.id.asc())).mappings().all()
+        return [
+            {
+                "id": row["id"],
+                "proposal_id": row["proposal_id"],
+                "kind": row["kind"],
+                "payload": _parse_json(row["payload_json"], {}),
+                "created_at": row["created_at"].isoformat(),
+            }
+            for row in rows
+        ]
+
+    def export_brief(self, proposal_id: int) -> dict[str, Any]:
+        proposal = self.get_proposal(proposal_id)
+        comments_data = self.list_comments(proposal_id)
+        dashboard = self.build_dashboard(proposal_id)
+        ledger = self.list_ledger(proposal_id)
+        clusters = self.cluster_comments(proposal_id)
+        versions = self.list_proposal_versions(proposal_id)
+
+        provenance = [
+            {"comment_id": comment["id"], "language": comment["language"], "channel": comment["channel"]}
+            for comment in comments_data
+        ]
+        return {
+            "proposal": proposal,
+            "headline": proposal["title"],
+            "summary": dashboard["summaries"],
+            "topic_clusters": clusters,
+            "preference_aggregate": dashboard["preference_aggregate"],
+            "consensus_pockets": dashboard["consensus_pockets"],
+            "unresolved_conflicts": dashboard["unresolved_conflicts"],
+            "outreach_gaps": dashboard["outreach_gaps"],
+            "ledger": ledger,
+            "proposal_versions": versions,
+            "provenance": provenance,
+        }
+
+    def _proposal_fragments(self, body: str) -> list[str]:
+        fragments = []
+        for chunk in body.split("\n"):
+            chunk = chunk.strip(" -\t")
+            if not chunk:
+                continue
+            fragments.extend([part.strip() for part in re.split(r"(?<=[.!?])\s+", chunk) if part.strip()])
+        return fragments or ([body.strip()] if body.strip() else [])
+
+    def _rank_residents_for_fragment(self, proposal: dict[str, Any], fragment: str, resident_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        proposal_tags = set(proposal.get("tags", []))
+        fragment_terms = _normalize_terms([fragment])
+        ranked = []
+        for resident in resident_rows:
+            interests = set(_parse_json(resident["interests_json"], []))
+            experience = set(_parse_json(resident["lived_experience_json"], []))
+            languages = set(_parse_json(resident["languages_json"], []))
+            resident_geo = (resident["geography"] or "").lower()
+            score = 0.0
+            reasons = []
+
+            if resident_geo and resident_geo in (proposal["geography"] or "").lower():
+                score += 0.25
+                reasons.append("geography_match")
+
+            overlap = len(_normalize_terms(proposal_tags) & interests)
+            if overlap:
+                score += min(0.3, 0.1 * overlap)
+                reasons.append("interest_overlap")
+
+            experience_overlap = len(fragment_terms & _normalize_terms(experience))
+            if experience_overlap:
+                score += min(0.25, 0.12 * experience_overlap)
+                reasons.append("lived_experience_match")
+
+            fragment_language = detect_language(fragment)
+            if fragment_language in languages:
+                score += 0.1
+                reasons.append("language_match")
+
+            profile_text = " ".join([resident["geography"], " ".join(sorted(interests)), " ".join(sorted(experience)), " ".join(sorted(languages))])
+            lexical = self._similarity(fragment, profile_text)
+            score += 0.35 * lexical
+            if lexical > 0:
+                reasons.append("lexical_similarity")
+
+            ranked.append(
+                {
+                    "resident_key": resident["resident_key"],
+                    "score": round(float(score), 4),
+                    "reasons": reasons,
+                }
+            )
+
+        ranked.sort(key=lambda item: (-item["score"], item["resident_key"]))
+        return [item for item in ranked if item["score"] > 0]
+
+    def _similarity(self, left: str, right: str) -> float:
+        left_tokens = _normalize_terms([left])
+        right_tokens = _normalize_terms([right])
+        if not left_tokens or not right_tokens:
+            return 0.0
+        overlap = len(left_tokens & right_tokens)
+        return overlap / math.sqrt(len(left_tokens) * len(right_tokens))
+
+    def _proposal_dict(self, row: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "id": row["id"],
+            "city": row["city"],
+            "title": row["title"],
+            "body": row["body"],
+            "status": row["status"],
+            "geography": row["geography"],
+            "tags": _parse_json(row["tags_json"], []),
+            "version": row["version"],
+            "created_at": row["created_at"].isoformat(),
+            "updated_at": row["updated_at"].isoformat(),
+        }
+
+    def _resident_dict(self, row: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "resident_key": row["resident_key"],
+            "geography": row["geography"],
+            "interests": _parse_json(row["interests_json"], []),
+            "lived_experience": _parse_json(row["lived_experience_json"], []),
+            "languages": _parse_json(row["languages_json"], []),
+            "created_at": row["created_at"].isoformat(),
+            "updated_at": row["updated_at"].isoformat(),
+        }
+
+    def _comment_dict(self, row: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "id": row["id"],
+            "proposal_id": row["proposal_id"],
+            "resident_key": row["resident_key"],
+            "channel": row["channel"],
+            "geography": row["geography"],
+            "language": row["language"],
+            "text": row["text"],
+            "metadata": _parse_json(row["metadata_json"], {}),
+            "created_at": row["created_at"].isoformat(),
+        }

pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "idea198-civicswarm-privacy-preserving"
+version = "0.1.0"
+description = "Privacy-preserving neighborhood deliberation router for participatory politics"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [{name = "OpenCode"}]
+dependencies = [
+  "fastapi>=0.115",
+  "pydantic>=2.7",
+  "sqlalchemy>=2.0",
+  "numpy>=1.26",
+  "scikit-learn>=1.5",
+  "langdetect>=1.0.9",
+  "uvicorn>=0.30",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0",
+  "build>=1.2",
+  "httpx>=0.27",
+]
+
+[project.urls]
+Homepage = "https://example.invalid/civicswarm"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+include = ["civicswarm*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python3 -m pip install -e ".[dev]"
+pytest
+python3 -m build

tests/test_service.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from civicswarm.api import create_app
+from civicswarm.service import CivicSwarmService
+
+
+def build_service(tmp_path: Path) -> CivicSwarmService:
+    return CivicSwarmService.from_path(tmp_path / "civicswarm.sqlite")
+
+
+def seed_service(service: CivicSwarmService) -> int:
+    proposal = service.create_proposal(
+        city="Springfield",
+        title="School street safety",
+        body=(
+            "Add protected bike lanes near schools.\n"
+            "Install safer crossings for pedestrians.\n"
+            "Preserve bus access for students and elders."
+        ),
+        geography="ward-3",
+        tags=["transport", "schools", "safety"],
+    )
+    service.register_resident("resident-a", geography="ward-3", interests=["transport", "safety"], lived_experience=["parent"], languages=["en"])
+    service.register_resident("resident-b", geography="ward-7", interests=["schools"], lived_experience=["bus rider"], languages=["es", "en"])
+    service.register_resident("resident-c", geography="ward-3", interests=["parks"], lived_experience=["elder"], languages=["en"])
+    service.add_comment(proposal["id"], "I support safer crossings for children and elders.", resident_key="resident-a", geography="ward-3")
+    service.add_comment(proposal["id"], "We need bus access and safer sidewalks.", resident_key="resident-b", geography="ward-7")
+    service.add_comment(proposal["id"], "I worry about delayed bus service.", resident_key="resident-c", geography="ward-3")
+    service.submit_preference(proposal["id"], "resident-a", 0.9)
+    service.submit_preference(proposal["id"], "resident-b", 0.4)
+    service.submit_preference(proposal["id"], "resident-c", 0.1)
+    return proposal["id"]
+
+
+def test_service_routes_comments_and_exports_brief(tmp_path: Path):
+    service = build_service(tmp_path)
+    proposal_id = seed_service(service)
+
+    summary = service.summarize_comments(proposal_id)
+    assert summary
+    assert "en" in summary
+
+    routed = service.route_proposal(proposal_id, top_n=2)
+    assert routed["fragments"]
+    assert routed["fragments"][0]["matches"]
+
+    dashboard = service.build_dashboard(proposal_id)
+    assert dashboard["preference_aggregate"]["count"] == 3
+    assert dashboard["consensus_pockets"]
+
+    brief = service.export_brief(proposal_id)
+    assert brief["proposal"]["title"] == "School street safety"
+    assert brief["provenance"]
+    assert brief["ledger"]
+
+
+def test_dashboard_surface_exposes_api(tmp_path: Path):
+    app = create_app(tmp_path / "api.sqlite")
+    client = __import__("fastapi.testclient", fromlist=["TestClient"]).TestClient(app)
+
+    proposal = client.post(
+        "/proposals",
+        json={
+            "city": "Springfield",
+            "title": "Library weekend hours",
+            "body": "Keep the library open later on weekends for students and families.",
+            "geography": "ward-1",
+            "tags": ["libraries", "education"],
+        },
+    ).json()
+
+    client.post("/residents", json={"resident_key": "resident-x", "geography": "ward-1", "interests": ["libraries"], "lived_experience": ["student"], "languages": ["en"]})
+    client.post(f"/proposals/{proposal['id']}/comments", json={"text": "I support later hours.", "resident_key": "resident-x", "channel": "sms"})
+    brief = client.get(f"/proposals/{proposal['id']}/brief").json()
+
+    assert brief["proposal"]["title"] == "Library weekend hours"
+    assert brief["summary"]
+
+
+def test_private_preference_aggregation_supports_noise(tmp_path: Path):
+    service = build_service(tmp_path)
+    proposal_id = seed_service(service)
+
+    clean = service.aggregate_preferences(proposal_id)
+    noisy = service.aggregate_preferences(proposal_id, epsilon=0.75, seed=42)
+
+    assert clean["count"] == 3
+    assert noisy["count"] >= 1
+    assert noisy["mean_score"] != clean["mean_score"] or noisy["count"] != clean["count"]