diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bd5590b --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +node_modules/ +.npmrc +.env +.env.* +__tests__/ +coverage/ +.nyc_output/ +dist/ +build/ +.cache/ +*.log +.DS_Store +tmp/ +.tmp/ +__pycache__/ +*.pyc +.venv/ +venv/ +*.egg-info/ +.pytest_cache/ +READY_TO_PUBLISH diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..47c1f3c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,34 @@ +# CivicSwarm Agent Guide + +## Architecture + +- `civicswarm/db.py` defines the SQLite schema and engine setup. +- `civicswarm/analysis.py` contains TF-IDF clustering, extractive summarization, language detection, and lightweight sentiment heuristics. +- `civicswarm/service.py` is the domain layer for proposals, comments, routing, privacy-preserving preference aggregation, the deliberation ledger, dashboard metrics, and civic brief export. +- `civicswarm/api.py` exposes the service through FastAPI. + +## Tech Stack + +- Python 3.11 +- FastAPI for the HTTP surface +- SQLAlchemy Core with SQLite for persistence +- scikit-learn and numpy for topic clustering and routing similarity +- langdetect for best-effort multilingual handling + +## Working Rules + +- Keep resident identity pseudonymous. Use `resident_key` values only; do not add raw identity storage. +- Prefer the smallest correct change. +- Keep logic deterministic where possible so tests remain stable. +- Add or update tests for every behavior change. +- Do not remove or rewrite unrelated files. + +## Testing + +- Install dependencies and run the suite: `bash test.sh` +- `test.sh` must succeed with `pytest` and `python3 -m build`. + +## Packaging + +- Distribution name: `idea198-civicswarm-privacy-preserving` +- README must stay current because it is wired into package metadata. diff --git a/README.md b/README.md index e328e6a..17a4bc2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,69 @@ -# idea198-civicswarm-privacy-preserving +# CivicSwarm -Source logic for Idea #198 \ No newline at end of file +Privacy-preserving neighborhood deliberation router for participatory politics. + +## What It Does + +CivicSwarm stores proposals, comments, resident profiles, preference signals, and deliberation ledger entries in SQLite. It then: + +- clusters comments with TF-IDF + KMeans +- summarizes discussion by detected language +- routes proposal fragments to relevant residents by geography, interests, experience, and language +- aggregates preferences with optional differential-privacy noise +- exports a civic brief with provenance and audit trail + +## Stack + +- Python 3.11 +- FastAPI +- SQLAlchemy Core + SQLite +- scikit-learn +- langdetect + +## Install + +```bash +python3 -m pip install -e . +``` + +For development: + +```bash +python3 -m pip install -e ".[dev]" +``` + +## Run + +```bash +python -m civicswarm +``` + +Environment variables: + +- `CIVICSWARM_DB_PATH` defaults to `civicswarm.sqlite` +- `CIVICSWARM_HOST` defaults to `127.0.0.1` +- `CIVICSWARM_PORT` defaults to `8000` + +## API + +- `POST /proposals` +- `POST /residents` +- `POST /proposals/{proposal_id}/comments` +- `POST /proposals/{proposal_id}/preferences` +- `GET /proposals/{proposal_id}/route` +- `GET /proposals/{proposal_id}/dashboard` +- `GET /proposals/{proposal_id}/brief` +- `GET /proposals/{proposal_id}/ledger` + +## Testing + +```bash +bash test.sh +``` + +That runs `pytest` and `python3 -m build`. + +## Notes + +- Resident identity is kept pseudonymous through `resident_key`. +- The current codebase is backend-first and designed for mobile, SMS, and field-capture integrations. diff --git a/civicswarm/__init__.py b/civicswarm/__init__.py new file mode 100644 index 0000000..9656b3e --- /dev/null +++ b/civicswarm/__init__.py @@ -0,0 +1,6 @@ +"""CivicSwarm package.""" + +from .api import create_app +from .service import CivicSwarmService + +__all__ = ["CivicSwarmService", "create_app"] diff --git a/civicswarm/__main__.py b/civicswarm/__main__.py new file mode 100644 index 0000000..29553de --- /dev/null +++ b/civicswarm/__main__.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import os + +import uvicorn + +from .api import create_app + + +def main() -> None: + db_path = os.environ.get("CIVICSWARM_DB_PATH", "civicswarm.sqlite") + app = create_app(db_path) + uvicorn.run(app, host=os.environ.get("CIVICSWARM_HOST", "127.0.0.1"), port=int(os.environ.get("CIVICSWARM_PORT", "8000"))) + + +if __name__ == "__main__": + main() diff --git a/civicswarm/analysis.py b/civicswarm/analysis.py new file mode 100644 index 0000000..700d305 --- /dev/null +++ b/civicswarm/analysis.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import math +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from typing import Iterable + +import numpy as np +from langdetect import DetectorFactory, detect +from sklearn.cluster import KMeans +from sklearn.feature_extraction.text import TfidfVectorizer + + +DetectorFactory.seed = 0 + +SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+") +WORD_RE = re.compile(r"[A-Za-zÀ-ÿ0-9']+") + +POSITIVE_WORDS = { + "support", + "approve", + "agree", + "welcome", + "benefit", + "safe", + "fair", + "help", + "yes", +} + +NEGATIVE_WORDS = { + "oppose", + "reject", + "harm", + "unsafe", + "unfair", + "no", + "risk", + "delay", + "concern", + "object", +} + + +def detect_language(text: str) -> str: + sample = text.strip() + if len(sample) < 10: + return "und" + try: + return detect(sample) + except Exception: + return "und" + + +def tokenize(text: str) -> list[str]: + return [token.lower() for token in WORD_RE.findall(text)] + + +def split_sentences(text: str) -> list[str]: + parts = [part.strip() for part in SENTENCE_SPLIT.split(text) if part.strip()] + return parts or ([text.strip()] if text.strip() else []) + + +def sentence_centrality_summary(texts: Iterable[str], max_sentences: int = 3) -> list[str]: + sentences: list[str] = [] + for text in texts: + sentences.extend(split_sentences(text)) + + unique_sentences: list[str] = [] + seen = set() + for sentence in sentences: + key = sentence.lower() + if key not in seen: + seen.add(key) + unique_sentences.append(sentence) + + if not unique_sentences: + return [] + if len(unique_sentences) <= max_sentences: + return unique_sentences + + vectorizer = TfidfVectorizer(stop_words="english") + matrix = vectorizer.fit_transform(unique_sentences) + similarity = (matrix * matrix.T).toarray() + scores = similarity.sum(axis=1) + ranked = sorted(range(len(unique_sentences)), key=lambda idx: (-scores[idx], idx))[:max_sentences] + return [unique_sentences[idx] for idx in sorted(ranked)] + + +def summarize_multilingual_comments(comments: list[dict], max_sentences: int = 3) -> dict[str, list[str]]: + by_language: dict[str, list[str]] = defaultdict(list) + for comment in comments: + language = comment.get("language") or detect_language(comment.get("text", "")) + by_language[language].append(comment.get("text", "")) + + return { + language: sentence_centrality_summary(texts, max_sentences=max_sentences) + for language, texts in sorted(by_language.items()) + } + + +def cluster_texts(texts: list[str]) -> list[dict]: + if not texts: + return [] + if len(texts) == 1: + return [{"cluster": 0, "items": [texts[0]], "top_terms": tokenize(texts[0])[:5]}] + + vectorizer = TfidfVectorizer(stop_words="english") + matrix = vectorizer.fit_transform(texts) + n_clusters = max(1, min(int(math.sqrt(len(texts))) or 1, 4)) + if n_clusters == 1: + return [{"cluster": 0, "items": texts, "top_terms": _top_terms(matrix, vectorizer)}] + + model = KMeans(n_clusters=n_clusters, n_init=10, random_state=0) + labels = model.fit_predict(matrix) + clusters: dict[int, list[str]] = defaultdict(list) + for label, text in zip(labels, texts): + clusters[int(label)].append(text) + + return [ + {"cluster": cluster_id, "items": items, "top_terms": _top_terms(matrix[[i for i, label in enumerate(labels) if label == cluster_id]], vectorizer)} + for cluster_id, items in sorted(clusters.items()) + ] + + +def _top_terms(matrix, vectorizer: TfidfVectorizer, limit: int = 5) -> list[str]: + if matrix.shape[0] == 0: + return [] + averaged = np.asarray(matrix.mean(axis=0)).ravel() + terms = np.array(vectorizer.get_feature_names_out()) + top_indices = averaged.argsort()[::-1][:limit] + return [str(terms[index]) for index in top_indices if averaged[index] > 0] + + +def sentiment_score(text: str) -> float: + tokens = tokenize(text) + if not tokens: + return 0.0 + positive = sum(1 for token in tokens if token in POSITIVE_WORDS) + negative = sum(1 for token in tokens if token in NEGATIVE_WORDS) + return (positive - negative) / max(1, len(tokens)) + + +def laplace_noise(scale: float, seed: int | None = None) -> float: + rng = np.random.default_rng(seed) + return float(rng.laplace(0.0, scale)) diff --git a/civicswarm/api.py b/civicswarm/api.py new file mode 100644 index 0000000..560a568 --- /dev/null +++ b/civicswarm/api.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field + +from .service import CivicSwarmService + + +class ProposalCreate(BaseModel): + city: str + title: str + body: str + geography: str = "" + tags: list[str] = Field(default_factory=list) + + +class ResidentCreate(BaseModel): + resident_key: str + geography: str = "" + interests: list[str] = Field(default_factory=list) + lived_experience: list[str] = Field(default_factory=list) + languages: list[str] = Field(default_factory=list) + + +class CommentCreate(BaseModel): + text: str + channel: str = "mobile" + resident_key: str | None = None + geography: str = "" + metadata: dict = Field(default_factory=dict) + + +class PreferenceCreate(BaseModel): + resident_key: str + score: float + channel: str = "mobile" + + +def create_app(db_path: str | Path = ":memory:") -> FastAPI: + service = CivicSwarmService.from_path(db_path) + app = FastAPI(title="CivicSwarm", version="0.1.0") + app.state.service = service + + @app.post("/proposals") + def create_proposal(payload: ProposalCreate): + return service.create_proposal(payload.city, payload.title, payload.body, payload.geography, payload.tags) + + @app.post("/residents") + def create_resident(payload: ResidentCreate): + return service.register_resident(payload.resident_key, payload.geography, payload.interests, payload.lived_experience, payload.languages) + + @app.post("/proposals/{proposal_id}/comments") + def create_comment(proposal_id: int, payload: CommentCreate): + try: + return service.add_comment(proposal_id, payload.text, payload.channel, payload.resident_key, payload.geography, payload.metadata) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + @app.post("/proposals/{proposal_id}/preferences") + def create_preference(proposal_id: int, payload: PreferenceCreate): + return service.submit_preference(proposal_id, payload.resident_key, payload.score, payload.channel) + + @app.get("/proposals/{proposal_id}/route") + def route_proposal(proposal_id: int, top_n: int = 3): + return service.route_proposal(proposal_id, top_n=top_n) + + @app.get("/proposals/{proposal_id}/dashboard") + def dashboard(proposal_id: int): + return service.build_dashboard(proposal_id) + + @app.get("/proposals/{proposal_id}/brief") + def brief(proposal_id: int): + return service.export_brief(proposal_id) + + @app.get("/proposals/{proposal_id}/ledger") + def ledger(proposal_id: int): + return service.list_ledger(proposal_id) + + return app diff --git a/civicswarm/db.py b/civicswarm/db.py new file mode 100644 index 0000000..fce5ba1 --- /dev/null +++ b/civicswarm/db.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from sqlalchemy import ( + JSON, + Column, + DateTime, + Float, + Integer, + MetaData, + String, + Table, + Text, + create_engine, + inspect, +) + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +metadata = MetaData() + +proposals = Table( + "proposals", + metadata, + Column("id", Integer, primary_key=True), + Column("city", String, nullable=False), + Column("title", String, nullable=False), + Column("body", Text, nullable=False), + Column("status", String, nullable=False, default="draft"), + Column("geography", String, nullable=False, default=""), + Column("tags_json", Text, nullable=False, default="[]"), + Column("version", Integer, nullable=False, default=1), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), + Column("updated_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + +proposal_versions = Table( + "proposal_versions", + metadata, + Column("id", Integer, primary_key=True), + Column("proposal_id", Integer, nullable=False), + Column("version", Integer, nullable=False), + Column("rationale", Text, nullable=False, default=""), + Column("objections_json", Text, nullable=False, default="[]"), + Column("resolution_status", String, nullable=False, default="open"), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + +residents = Table( + "residents", + metadata, + Column("id", Integer, primary_key=True), + Column("resident_key", String, nullable=False, unique=True), + Column("geography", String, nullable=False, default=""), + Column("interests_json", Text, nullable=False, default="[]"), + Column("lived_experience_json", Text, nullable=False, default="[]"), + Column("languages_json", Text, nullable=False, default="[]"), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), + Column("updated_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + +comments = Table( + "comments", + metadata, + Column("id", Integer, primary_key=True), + Column("proposal_id", Integer, nullable=False), + Column("resident_key", String, nullable=True), + Column("channel", String, nullable=False), + Column("geography", String, nullable=False, default=""), + Column("language", String, nullable=False, default="und"), + Column("text", Text, nullable=False), + Column("metadata_json", Text, nullable=False, default="{}"), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + +preferences = Table( + "preferences", + metadata, + Column("id", Integer, primary_key=True), + Column("proposal_id", Integer, nullable=False), + Column("resident_key", String, nullable=False), + Column("score", Float, nullable=False), + Column("channel", String, nullable=False), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + +routes = Table( + "routes", + metadata, + Column("id", Integer, primary_key=True), + Column("proposal_id", Integer, nullable=False), + Column("fragment_index", Integer, nullable=False), + Column("resident_key", String, nullable=False), + Column("score", Float, nullable=False), + Column("rationale_json", Text, nullable=False, default="[]"), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + +ledger_entries = Table( + "ledger_entries", + metadata, + Column("id", Integer, primary_key=True), + Column("proposal_id", Integer, nullable=False), + Column("kind", String, nullable=False), + Column("payload_json", Text, nullable=False), + Column("created_at", DateTime(timezone=True), nullable=False, default=utcnow), +) + + +@dataclass(frozen=True) +class Database: + engine: object + + +def create_database(db_url: str): + engine = create_engine(db_url, future=True) + metadata.create_all(engine) + return Database(engine=engine) + + +def sqlite_path_url(path: str | Path) -> str: + p = Path(path) + if str(p) == ":memory:": + return "sqlite+pysqlite:///:memory:" + return f"sqlite+pysqlite:///{p}" diff --git a/civicswarm/service.py b/civicswarm/service.py new file mode 100644 index 0000000..648e851 --- /dev/null +++ b/civicswarm/service.py @@ -0,0 +1,505 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import math +import re +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +import numpy as np +from sqlalchemy import delete, func, insert, select, update + +from .analysis import ( + cluster_texts, + detect_language, + laplace_noise, + sentiment_score, + summarize_multilingual_comments, + tokenize, +) +from .db import ( + comments, + create_database, + ledger_entries, + preferences, + proposal_versions, + proposals, + residents, + routes, + sqlite_path_url, +) + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def _parse_json(value: str | None, default: Any) -> Any: + if not value: + return default + return json.loads(value) + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _normalize_terms(values: Iterable[str]) -> set[str]: + terms: set[str] = set() + for value in values: + terms.update(tokenize(value)) + return terms + + +@dataclass +class CivicSwarmService: + db_url: str + secret: str = "civicswarm" + + def __post_init__(self) -> None: + self.db = create_database(self.db_url) + + @classmethod + def from_path(cls, path: str | Path, secret: str = "civicswarm") -> "CivicSwarmService": + return cls(db_url=sqlite_path_url(path), secret=secret) + + def _token(self, resident_key: str | None) -> str | None: + if resident_key is None: + return None + return hmac.new(self.secret.encode(), resident_key.encode(), hashlib.sha256).hexdigest() + + def create_proposal(self, city: str, title: str, body: str, geography: str = "", tags: list[str] | None = None) -> dict[str, Any]: + tags = tags or [] + with self.db.engine.begin() as conn: + result = conn.execute( + insert(proposals).values( + city=city, + title=title, + body=body, + geography=geography, + tags_json=_json(tags), + status="draft", + version=1, + created_at=_now(), + updated_at=_now(), + ) + ) + proposal_id = result.inserted_primary_key[0] + conn.execute( + insert(ledger_entries).values( + proposal_id=proposal_id, + kind="proposal_created", + payload_json=_json({"title": title, "tags": tags, "geography": geography}), + created_at=_now(), + ) + ) + conn.execute( + insert(proposal_versions).values( + proposal_id=proposal_id, + version=1, + rationale="initial proposal", + objections_json=_json([]), + resolution_status="open", + created_at=_now(), + ) + ) + return self.get_proposal(proposal_id) + + def get_proposal(self, proposal_id: int) -> dict[str, Any]: + with self.db.engine.begin() as conn: + row = conn.execute(select(proposals).where(proposals.c.id == proposal_id)).mappings().first() + if row is None: + raise KeyError(f"proposal {proposal_id} not found") + return self._proposal_dict(row) + + def list_comments(self, proposal_id: int) -> list[dict[str, Any]]: + with self.db.engine.begin() as conn: + rows = conn.execute(select(comments).where(comments.c.proposal_id == proposal_id).order_by(comments.c.id.asc())).mappings().all() + return [self._comment_dict(row) for row in rows] + + def register_resident( + self, + resident_key: str, + geography: str = "", + interests: list[str] | None = None, + lived_experience: list[str] | None = None, + languages: list[str] | None = None, + ) -> dict[str, Any]: + interests = interests or [] + lived_experience = lived_experience or [] + languages = languages or [] + with self.db.engine.begin() as conn: + existing = conn.execute(select(residents).where(residents.c.resident_key == resident_key)).mappings().first() + values = dict( + resident_key=resident_key, + geography=geography, + interests_json=_json(interests), + lived_experience_json=_json(lived_experience), + languages_json=_json(languages), + updated_at=_now(), + ) + if existing is None: + values["created_at"] = _now() + conn.execute(insert(residents).values(**values)) + else: + conn.execute(update(residents).where(residents.c.resident_key == resident_key).values(**values)) + return self.get_resident(resident_key) + + def get_resident(self, resident_key: str) -> dict[str, Any]: + with self.db.engine.begin() as conn: + row = conn.execute(select(residents).where(residents.c.resident_key == resident_key)).mappings().first() + if row is None: + raise KeyError(f"resident {resident_key} not found") + return self._resident_dict(row) + + def add_comment( + self, + proposal_id: int, + text: str, + channel: str = "mobile", + resident_key: str | None = None, + geography: str = "", + metadata: dict[str, Any] | None = None, + ) -> dict[str, Any]: + metadata = metadata or {} + language = detect_language(text) + with self.db.engine.begin() as conn: + result = conn.execute( + insert(comments).values( + proposal_id=proposal_id, + resident_key=self._token(resident_key), + channel=channel, + geography=geography, + language=language, + text=text, + metadata_json=_json(metadata), + created_at=_now(), + ) + ) + comment_id = result.inserted_primary_key[0] + conn.execute( + insert(ledger_entries).values( + proposal_id=proposal_id, + kind="comment_received", + payload_json=_json({"comment_id": comment_id, "channel": channel, "language": language}), + created_at=_now(), + ) + ) + return self.get_comment(comment_id) + + def get_comment(self, comment_id: int) -> dict[str, Any]: + with self.db.engine.begin() as conn: + row = conn.execute(select(comments).where(comments.c.id == comment_id)).mappings().first() + if row is None: + raise KeyError(f"comment {comment_id} not found") + return self._comment_dict(row) + + def submit_preference(self, proposal_id: int, resident_key: str, score: float, channel: str = "mobile") -> dict[str, Any]: + with self.db.engine.begin() as conn: + result = conn.execute( + insert(preferences).values( + proposal_id=proposal_id, + resident_key=self._token(resident_key) or resident_key, + score=float(score), + channel=channel, + created_at=_now(), + ) + ) + preference_id = result.inserted_primary_key[0] + conn.execute( + insert(ledger_entries).values( + proposal_id=proposal_id, + kind="preference_submitted", + payload_json=_json({"preference_id": preference_id, "channel": channel}), + created_at=_now(), + ) + ) + return {"id": preference_id, "proposal_id": proposal_id, "resident_key": resident_key, "score": float(score), "channel": channel} + + def aggregate_preferences(self, proposal_id: int, epsilon: float | None = None, seed: int = 0) -> dict[str, Any]: + with self.db.engine.begin() as conn: + rows = conn.execute(select(preferences.c.score).where(preferences.c.proposal_id == proposal_id)).all() + scores = [float(row[0]) for row in rows] + count = len(scores) + total = float(sum(scores)) + mean = total / count if count else 0.0 + if epsilon and epsilon > 0: + scale = 1.0 / float(epsilon) + total += laplace_noise(scale, seed=seed) + count = max(1, int(round(count + laplace_noise(scale / 2.0, seed=seed + 1)))) + mean = total / count + return {"proposal_id": proposal_id, "count": count, "mean_score": mean, "raw_total": float(sum(scores))} + + def cluster_comments(self, proposal_id: int) -> list[dict[str, Any]]: + comments_data = self.list_comments(proposal_id) + return cluster_texts([comment["text"] for comment in comments_data]) + + def summarize_comments(self, proposal_id: int, max_sentences: int = 3) -> dict[str, list[str]]: + comments_data = self.list_comments(proposal_id) + return summarize_multilingual_comments(comments_data, max_sentences=max_sentences) + + def route_proposal(self, proposal_id: int, top_n: int = 3) -> dict[str, Any]: + proposal = self.get_proposal(proposal_id) + fragments = self._proposal_fragments(proposal["body"]) + with self.db.engine.begin() as conn: + resident_rows = conn.execute(select(residents)).mappings().all() + + routed_fragments = [] + for index, fragment in enumerate(fragments): + ranked = self._rank_residents_for_fragment(proposal, fragment, resident_rows) + top_matches = ranked[:top_n] + routed_fragments.append({"fragment_index": index, "fragment": fragment, "matches": top_matches}) + with self.db.engine.begin() as conn: + conn.execute(delete(routes).where(routes.c.proposal_id == proposal_id, routes.c.fragment_index == index)) + for match in top_matches: + conn.execute( + insert(routes).values( + proposal_id=proposal_id, + fragment_index=index, + resident_key=match["resident_key"], + score=match["score"], + rationale_json=_json(match["reasons"]), + created_at=_now(), + ) + ) + return {"proposal_id": proposal_id, "fragments": routed_fragments} + + def build_dashboard(self, proposal_id: int) -> dict[str, Any]: + proposal = self.get_proposal(proposal_id) + comments_data = self.list_comments(proposal_id) + summaries = self.summarize_comments(proposal_id) + preferences_summary = self.aggregate_preferences(proposal_id) + routes_summary = self.route_proposal(proposal_id) + + consensus_pockets = [] + unresolved_conflicts = [] + outreach_gaps = [] + + for fragment in routes_summary["fragments"]: + fragment_text = fragment["fragment"] + fragment_comments = [comment for comment in comments_data if any(term in comment["text"].lower() for term in tokenize(fragment_text))] + sentiment_values = [sentiment_score(comment["text"]) for comment in fragment_comments] + if sentiment_values and np.mean(sentiment_values) > 0: + consensus_pockets.append({"fragment_index": fragment["fragment_index"], "fragment": fragment_text, "support": float(np.mean(sentiment_values))}) + if sentiment_values and min(sentiment_values) < 0 < max(sentiment_values): + unresolved_conflicts.append({"fragment_index": fragment["fragment_index"], "fragment": fragment_text, "support": float(np.mean(sentiment_values)), "spread": float(max(sentiment_values) - min(sentiment_values))}) + if not fragment["matches"]: + outreach_gaps.append({"fragment_index": fragment["fragment_index"], "fragment": fragment_text}) + + return { + "proposal": proposal, + "summaries": summaries, + "preference_aggregate": preferences_summary, + "consensus_pockets": consensus_pockets, + "unresolved_conflicts": unresolved_conflicts, + "outreach_gaps": outreach_gaps, + } + + def record_ledger_entry(self, proposal_id: int, kind: str, payload: dict[str, Any]) -> dict[str, Any]: + with self.db.engine.begin() as conn: + result = conn.execute( + insert(ledger_entries).values( + proposal_id=proposal_id, + kind=kind, + payload_json=_json(payload), + created_at=_now(), + ) + ) + return {"id": result.inserted_primary_key[0], "proposal_id": proposal_id, "kind": kind, "payload": payload} + + def record_proposal_version(self, proposal_id: int, rationale: str, objections: list[str], resolution_status: str) -> dict[str, Any]: + with self.db.engine.begin() as conn: + current = conn.execute( + select(func.max(proposal_versions.c.version)).where(proposal_versions.c.proposal_id == proposal_id) + ).scalar_one_or_none() + version = int(current or 0) + 1 + result = conn.execute( + insert(proposal_versions).values( + proposal_id=proposal_id, + version=version, + rationale=rationale, + objections_json=_json(objections), + resolution_status=resolution_status, + created_at=_now(), + ) + ) + conn.execute( + update(proposals) + .where(proposals.c.id == proposal_id) + .values(version=version, updated_at=_now(), status=resolution_status) + ) + conn.execute( + insert(ledger_entries).values( + proposal_id=proposal_id, + kind="proposal_version_recorded", + payload_json=_json({"version": version, "resolution_status": resolution_status}), + created_at=_now(), + ) + ) + return {"id": result.inserted_primary_key[0], "proposal_id": proposal_id, "version": version, "resolution_status": resolution_status} + + def list_proposal_versions(self, proposal_id: int) -> list[dict[str, Any]]: + with self.db.engine.begin() as conn: + rows = conn.execute( + select(proposal_versions).where(proposal_versions.c.proposal_id == proposal_id).order_by(proposal_versions.c.version.asc()) + ).mappings().all() + return [ + { + "id": row["id"], + "proposal_id": row["proposal_id"], + "version": row["version"], + "rationale": row["rationale"], + "objections": _parse_json(row["objections_json"], []), + "resolution_status": row["resolution_status"], + "created_at": row["created_at"].isoformat(), + } + for row in rows + ] + + def list_ledger(self, proposal_id: int) -> list[dict[str, Any]]: + with self.db.engine.begin() as conn: + rows = conn.execute(select(ledger_entries).where(ledger_entries.c.proposal_id == proposal_id).order_by(ledger_entries.c.id.asc())).mappings().all() + return [ + { + "id": row["id"], + "proposal_id": row["proposal_id"], + "kind": row["kind"], + "payload": _parse_json(row["payload_json"], {}), + "created_at": row["created_at"].isoformat(), + } + for row in rows + ] + + def export_brief(self, proposal_id: int) -> dict[str, Any]: + proposal = self.get_proposal(proposal_id) + comments_data = self.list_comments(proposal_id) + dashboard = self.build_dashboard(proposal_id) + ledger = self.list_ledger(proposal_id) + clusters = self.cluster_comments(proposal_id) + versions = self.list_proposal_versions(proposal_id) + + provenance = [ + {"comment_id": comment["id"], "language": comment["language"], "channel": comment["channel"]} + for comment in comments_data + ] + return { + "proposal": proposal, + "headline": proposal["title"], + "summary": dashboard["summaries"], + "topic_clusters": clusters, + "preference_aggregate": dashboard["preference_aggregate"], + "consensus_pockets": dashboard["consensus_pockets"], + "unresolved_conflicts": dashboard["unresolved_conflicts"], + "outreach_gaps": dashboard["outreach_gaps"], + "ledger": ledger, + "proposal_versions": versions, + "provenance": provenance, + } + + def _proposal_fragments(self, body: str) -> list[str]: + fragments = [] + for chunk in body.split("\n"): + chunk = chunk.strip(" -\t") + if not chunk: + continue + fragments.extend([part.strip() for part in re.split(r"(?<=[.!?])\s+", chunk) if part.strip()]) + return fragments or ([body.strip()] if body.strip() else []) + + def _rank_residents_for_fragment(self, proposal: dict[str, Any], fragment: str, resident_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + proposal_tags = set(proposal.get("tags", [])) + fragment_terms = _normalize_terms([fragment]) + ranked = [] + for resident in resident_rows: + interests = set(_parse_json(resident["interests_json"], [])) + experience = set(_parse_json(resident["lived_experience_json"], [])) + languages = set(_parse_json(resident["languages_json"], [])) + resident_geo = (resident["geography"] or "").lower() + score = 0.0 + reasons = [] + + if resident_geo and resident_geo in (proposal["geography"] or "").lower(): + score += 0.25 + reasons.append("geography_match") + + overlap = len(_normalize_terms(proposal_tags) & interests) + if overlap: + score += min(0.3, 0.1 * overlap) + reasons.append("interest_overlap") + + experience_overlap = len(fragment_terms & _normalize_terms(experience)) + if experience_overlap: + score += min(0.25, 0.12 * experience_overlap) + reasons.append("lived_experience_match") + + fragment_language = detect_language(fragment) + if fragment_language in languages: + score += 0.1 + reasons.append("language_match") + + profile_text = " ".join([resident["geography"], " ".join(sorted(interests)), " ".join(sorted(experience)), " ".join(sorted(languages))]) + lexical = self._similarity(fragment, profile_text) + score += 0.35 * lexical + if lexical > 0: + reasons.append("lexical_similarity") + + ranked.append( + { + "resident_key": resident["resident_key"], + "score": round(float(score), 4), + "reasons": reasons, + } + ) + + ranked.sort(key=lambda item: (-item["score"], item["resident_key"])) + return [item for item in ranked if item["score"] > 0] + + def _similarity(self, left: str, right: str) -> float: + left_tokens = _normalize_terms([left]) + right_tokens = _normalize_terms([right]) + if not left_tokens or not right_tokens: + return 0.0 + overlap = len(left_tokens & right_tokens) + return overlap / math.sqrt(len(left_tokens) * len(right_tokens)) + + def _proposal_dict(self, row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "city": row["city"], + "title": row["title"], + "body": row["body"], + "status": row["status"], + "geography": row["geography"], + "tags": _parse_json(row["tags_json"], []), + "version": row["version"], + "created_at": row["created_at"].isoformat(), + "updated_at": row["updated_at"].isoformat(), + } + + def _resident_dict(self, row: dict[str, Any]) -> dict[str, Any]: + return { + "resident_key": row["resident_key"], + "geography": row["geography"], + "interests": _parse_json(row["interests_json"], []), + "lived_experience": _parse_json(row["lived_experience_json"], []), + "languages": _parse_json(row["languages_json"], []), + "created_at": row["created_at"].isoformat(), + "updated_at": row["updated_at"].isoformat(), + } + + def _comment_dict(self, row: dict[str, Any]) -> dict[str, Any]: + return { + "id": row["id"], + "proposal_id": row["proposal_id"], + "resident_key": row["resident_key"], + "channel": row["channel"], + "geography": row["geography"], + "language": row["language"], + "text": row["text"], + "metadata": _parse_json(row["metadata_json"], {}), + "created_at": row["created_at"].isoformat(), + } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ab89765 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "idea198-civicswarm-privacy-preserving" +version = "0.1.0" +description = "Privacy-preserving neighborhood deliberation router for participatory politics" +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +authors = [{name = "OpenCode"}] +dependencies = [ + "fastapi>=0.115", + "pydantic>=2.7", + "sqlalchemy>=2.0", + "numpy>=1.26", + "scikit-learn>=1.5", + "langdetect>=1.0.9", + "uvicorn>=0.30", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "build>=1.2", + "httpx>=0.27", +] + +[project.urls] +Homepage = "https://example.invalid/civicswarm" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["civicswarm*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..fbc1778 --- /dev/null +++ b/test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +python3 -m pip install -e ".[dev]" +pytest +python3 -m build diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..163a4f0 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from pathlib import Path + +from civicswarm.api import create_app +from civicswarm.service import CivicSwarmService + + +def build_service(tmp_path: Path) -> CivicSwarmService: + return CivicSwarmService.from_path(tmp_path / "civicswarm.sqlite") + + +def seed_service(service: CivicSwarmService) -> int: + proposal = service.create_proposal( + city="Springfield", + title="School street safety", + body=( + "Add protected bike lanes near schools.\n" + "Install safer crossings for pedestrians.\n" + "Preserve bus access for students and elders." + ), + geography="ward-3", + tags=["transport", "schools", "safety"], + ) + service.register_resident("resident-a", geography="ward-3", interests=["transport", "safety"], lived_experience=["parent"], languages=["en"]) + service.register_resident("resident-b", geography="ward-7", interests=["schools"], lived_experience=["bus rider"], languages=["es", "en"]) + service.register_resident("resident-c", geography="ward-3", interests=["parks"], lived_experience=["elder"], languages=["en"]) + service.add_comment(proposal["id"], "I support safer crossings for children and elders.", resident_key="resident-a", geography="ward-3") + service.add_comment(proposal["id"], "We need bus access and safer sidewalks.", resident_key="resident-b", geography="ward-7") + service.add_comment(proposal["id"], "I worry about delayed bus service.", resident_key="resident-c", geography="ward-3") + service.submit_preference(proposal["id"], "resident-a", 0.9) + service.submit_preference(proposal["id"], "resident-b", 0.4) + service.submit_preference(proposal["id"], "resident-c", 0.1) + return proposal["id"] + + +def test_service_routes_comments_and_exports_brief(tmp_path: Path): + service = build_service(tmp_path) + proposal_id = seed_service(service) + + summary = service.summarize_comments(proposal_id) + assert summary + assert "en" in summary + + routed = service.route_proposal(proposal_id, top_n=2) + assert routed["fragments"] + assert routed["fragments"][0]["matches"] + + dashboard = service.build_dashboard(proposal_id) + assert dashboard["preference_aggregate"]["count"] == 3 + assert dashboard["consensus_pockets"] + + brief = service.export_brief(proposal_id) + assert brief["proposal"]["title"] == "School street safety" + assert brief["provenance"] + assert brief["ledger"] + + +def test_dashboard_surface_exposes_api(tmp_path: Path): + app = create_app(tmp_path / "api.sqlite") + client = __import__("fastapi.testclient", fromlist=["TestClient"]).TestClient(app) + + proposal = client.post( + "/proposals", + json={ + "city": "Springfield", + "title": "Library weekend hours", + "body": "Keep the library open later on weekends for students and families.", + "geography": "ward-1", + "tags": ["libraries", "education"], + }, + ).json() + + client.post("/residents", json={"resident_key": "resident-x", "geography": "ward-1", "interests": ["libraries"], "lived_experience": ["student"], "languages": ["en"]}) + client.post(f"/proposals/{proposal['id']}/comments", json={"text": "I support later hours.", "resident_key": "resident-x", "channel": "sms"}) + brief = client.get(f"/proposals/{proposal['id']}/brief").json() + + assert brief["proposal"]["title"] == "Library weekend hours" + assert brief["summary"] + + +def test_private_preference_aggregation_supports_noise(tmp_path: Path): + service = build_service(tmp_path) + proposal_id = seed_service(service) + + clean = service.aggregate_preferences(proposal_id) + noisy = service.aggregate_preferences(proposal_id, epsilon=0.75, seed=42) + + assert clean["count"] == 3 + assert noisy["count"] >= 1 + assert noisy["mean_score"] != clean["mean_score"] or noisy["count"] != clean["count"]