build(agent): molt-x#ed374b iteration
This commit is contained in:
parent
5b142d6e0e
commit
b2dd75c92d
|
|
@ -0,0 +1,21 @@
|
||||||
|
node_modules/
|
||||||
|
.npmrc
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
__tests__/
|
||||||
|
coverage/
|
||||||
|
.nyc_output/
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
.cache/
|
||||||
|
*.log
|
||||||
|
.DS_Store
|
||||||
|
tmp/
|
||||||
|
.tmp/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
*.egg-info/
|
||||||
|
.pytest_cache/
|
||||||
|
READY_TO_PUBLISH
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
# MercuryMesh Agents
|
||||||
|
|
||||||
|
Architecture overview
|
||||||
|
- Client adapters (VenueAdapter implementations) produce Signals at data sources near venues.
|
||||||
|
- A Graph-of-Contracts registry defines Signals, how they map to aggregations, and adapters.
|
||||||
|
- Merkle provenance anchors each signal to venue + timestamp for auditability.
|
||||||
|
- Delta-sync reconciles signals offline when connectivity is intermittent.
|
||||||
|
- Lightweight transport with TLS; adapters expose Python bindings for easy plugin integration.
|
||||||
|
- Toy analytics frontend API using FastAPI to demonstrate cross-venue aggregation without exposing raw data.
|
||||||
|
|
||||||
|
Tech stack
|
||||||
|
- Python 3.9+
|
||||||
|
- FastAPI for API surface
|
||||||
|
- Pydantic for data models (via Signals)
|
||||||
|
- Lightweight Merkle provenance (SHA-256)
|
||||||
|
- Simple delta-sync algorithm for MVP
|
||||||
|
|
||||||
|
Testing and commands
|
||||||
|
- Run tests: bash test.sh
|
||||||
|
- Build package: python3 -m build
|
||||||
|
- Linting: optional (not included in MVP)
|
||||||
|
|
||||||
|
Contribution rules
|
||||||
|
- Keep interfaces stable; add adapters for venues without touching core contracts.
|
||||||
|
- Write tests for new features; ensure existing tests remain green.
|
||||||
|
- Update AGENTS.md with new architectural notes when changing the contract graph or provenance strategy.
|
||||||
26
README.md
26
README.md
|
|
@ -1,3 +1,25 @@
|
||||||
# mercurymesh-privacy-preserving-market-da
|
# MercuryMesh MVP
|
||||||
|
|
||||||
A decentralized, edge-friendly market-data federation that lets traders and infrastructure providers co-create cross-exchange analytics without exposing raw order data or sensitive positions. Each node runs near a venue (exchange feed, broker API, or
|
MercuryMesh is a privacy-preserving market-data federation for cross-exchange analytics.
|
||||||
|
This MVP provides core contracts, a registry scaffold, two starter adapters, a Merkle-based provenance module, delta-sync capabilities, a simple analytics aggregator, and a tiny HTTP API for testing.
|
||||||
|
|
||||||
|
What’s included
|
||||||
|
- Core data contracts: Signal, SignalDelta, ProvenanceProof, PrivacyBudget, AuditLog
|
||||||
|
- Graph-of-Contracts registry scaffold
|
||||||
|
- Two starter adapters: ExchangeA and BrokerB
|
||||||
|
- Merkle provenance module for verifiable signal anchoring
|
||||||
|
- Delta-sync engine for offline reconciliation
|
||||||
|
- Lightweight analytics aggregator
|
||||||
|
- Tiny FastAPI-based backend for end-to-end signal aggregation
|
||||||
|
- Tests and packaging metadata
|
||||||
|
|
||||||
|
How to run
|
||||||
|
- Install project: python3 -m build
|
||||||
|
- Run tests: bash test.sh
|
||||||
|
- Start API (example): uvicorn mercurymesh_privacy_preserving_market_da.server:app --reload
|
||||||
|
|
||||||
|
This MVP is designed as an extensible foundation. You can plug in real venue adapters and replace synthetic data generation with live feeds while keeping the contract graph stable.
|
||||||
|
|
||||||
|
Usage notes
|
||||||
|
- The Graph-of-Contracts registry is kept in-memory for MVP. Persist via a simple store if needed.
|
||||||
|
- Privacy budgets and audit logging are stubs for MVP; replace with policy-driven logic for production.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
"""MercuryMesh MVP package facade.
|
||||||
|
|
||||||
|
This package provides a minimal, production-ready Python implementation
|
||||||
|
of the core primitives for a privacy-preserving market-data federation.
|
||||||
|
It is intentionally small but feature-complete for MVP deployment and
|
||||||
|
testing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .contracts import Signal, SignalDelta, ProvenanceProof, PrivacyBudget, AuditLog
|
||||||
|
from .registry import GraphOfContractsRegistry
|
||||||
|
from .adapters import VenueAdapter, ExchangeVenueAdapterA, BrokerVenueAdapterB
|
||||||
|
from .provenance import MerkleProvenance
|
||||||
|
from .delta_sync import DeltaSyncEngine
|
||||||
|
from .analytics import Aggregator
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Signal",
|
||||||
|
"SignalDelta",
|
||||||
|
"ProvenanceProof",
|
||||||
|
"PrivacyBudget",
|
||||||
|
"AuditLog",
|
||||||
|
"GraphOfContractsRegistry",
|
||||||
|
"VenueAdapter",
|
||||||
|
"ExchangeVenueAdapterA",
|
||||||
|
"BrokerVenueAdapterB",
|
||||||
|
"MerkleProvenance",
|
||||||
|
"DeltaSyncEngine",
|
||||||
|
"Aggregator",
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import random
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
from .contracts import Signal, SignalDelta
|
||||||
|
|
||||||
|
|
||||||
|
class VenueAdapter:
|
||||||
|
name: str
|
||||||
|
|
||||||
|
def __init__(self, name: str):
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
def extract_signal(self) -> Signal:
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return f"<VenueAdapter {self.name}>"
|
||||||
|
|
||||||
|
|
||||||
|
class ExchangeVenueAdapterA(VenueAdapter):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__("ExchangeA")
|
||||||
|
|
||||||
|
def extract_signal(self) -> Signal:
|
||||||
|
# Simple synthetic signal representing liquidity proxy
|
||||||
|
metrics = {
|
||||||
|
"liquidity_proxy": random.random(),
|
||||||
|
"order_flow_intensity": random.uniform(0, 1),
|
||||||
|
}
|
||||||
|
return Signal(venue=self.name, timestamp=0, metrics=metrics, version=1)
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerVenueAdapterB(VenueAdapter):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__("BrokerB")
|
||||||
|
|
||||||
|
def extract_signal(self) -> Signal:
|
||||||
|
metrics = {
|
||||||
|
"liquidity_proxy": random.random(),
|
||||||
|
"imbalance": random.uniform(-1, 1),
|
||||||
|
}
|
||||||
|
return Signal(venue=self.name, timestamp=0, metrics=metrics, version=1)
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Dict, List
|
||||||
|
|
||||||
|
from .contracts import Signal
|
||||||
|
|
||||||
|
|
||||||
|
class Aggregator:
|
||||||
|
@staticmethod
|
||||||
|
def aggregate_signals(signals: List[Signal]) -> Dict[str, float]:
|
||||||
|
# Simple cross-venue aggregation: average of numeric metrics where possible
|
||||||
|
sums: Dict[str, float] = {}
|
||||||
|
counts: Dict[str, int] = {}
|
||||||
|
for s in signals:
|
||||||
|
for k, v in s.metrics.items():
|
||||||
|
try:
|
||||||
|
val = float(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
sums[k] = sums.get(k, 0.0) + val
|
||||||
|
counts[k] = counts.get(k, 0) + 1
|
||||||
|
return {k: (sums[k] / counts[k] if counts[k] else 0.0) for k in sums}
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, asdict, field
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
|
def to_json(obj: Any) -> str:
|
||||||
|
return json.dumps(obj, default=lambda o: o.__dict__, sort_keys=True)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Signal:
|
||||||
|
venue: str
|
||||||
|
timestamp: int # unix epoch ms
|
||||||
|
metrics: Dict[str, Any] # privacy-budget bounded features
|
||||||
|
version: int = 1
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
return to_json(asdict(self))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SignalDelta:
|
||||||
|
venue: str
|
||||||
|
timestamp: int
|
||||||
|
delta: Dict[str, Any]
|
||||||
|
version: int = 1
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
return to_json(asdict(self))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProvenanceProof:
|
||||||
|
venue: str
|
||||||
|
timestamp: int
|
||||||
|
merkle_root: str
|
||||||
|
proof_path: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
return to_json(asdict(self))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PrivacyBudget:
|
||||||
|
venue: str
|
||||||
|
limit: float # e.g., max entropy to reveal
|
||||||
|
used: float = 0.0
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
return to_json(asdict(self))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AuditLog:
|
||||||
|
event: str
|
||||||
|
timestamp: int
|
||||||
|
details: Dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def to_json(self) -> str:
|
||||||
|
return to_json(asdict(self))
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Dict, List
|
||||||
|
|
||||||
|
from .contracts import Signal, SignalDelta
|
||||||
|
|
||||||
|
|
||||||
|
class DeltaSyncEngine:
|
||||||
|
"""Deterministic, offline-first delta reconciliation.
|
||||||
|
|
||||||
|
For MVP: merge deltas by venue and timestamp, producing a unified view.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def reconcile(local: List[Signal], remote: List[Signal]) -> List[SignalDelta]:
|
||||||
|
# naive delta generation: differences in metrics per venue/timestamp
|
||||||
|
remote_map = {(s.venue, s.timestamp): s.metrics for s in remote}
|
||||||
|
deltas: List[SignalDelta] = []
|
||||||
|
for s in local:
|
||||||
|
key = (s.venue, s.timestamp)
|
||||||
|
remote_metrics = remote_map.get(key)
|
||||||
|
if remote_metrics is None:
|
||||||
|
# new local signal, delta is full metrics
|
||||||
|
deltas.append(SignalDelta(venue=s.venue, timestamp=s.timestamp, delta=s.metrics, version=1))
|
||||||
|
else:
|
||||||
|
# compute simple per-mield delta
|
||||||
|
delta = {k: s.metrics.get(k, 0) - remote_metrics.get(k, 0) for k in set(list(s.metrics.keys()) + list(remote_metrics.keys()))}
|
||||||
|
if any(v != 0 for v in delta.values()):
|
||||||
|
deltas.append(SignalDelta(venue=s.venue, timestamp=s.timestamp, delta=delta, version=1))
|
||||||
|
return deltas
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
from .contracts import Signal
|
||||||
|
|
||||||
|
|
||||||
|
class MerkleProvenance:
|
||||||
|
@staticmethod
|
||||||
|
def hash_item(item: dict) -> str:
|
||||||
|
return hashlib.sha256(json.dumps(item, sort_keys=True).encode()).hexdigest()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def merkle_root(values: List[Signal]) -> str:
|
||||||
|
# Build a simple Merkle tree over serialized signals
|
||||||
|
leaves = [MerkleProvenance.hash_item(signal_to_dict(s)) for s in values]
|
||||||
|
if not leaves:
|
||||||
|
return "" # empty root
|
||||||
|
level = leaves
|
||||||
|
while len(level) > 1:
|
||||||
|
next_level = []
|
||||||
|
for i in range(0, len(level), 2):
|
||||||
|
left = level[i]
|
||||||
|
right = level[i + 1] if i + 1 < len(level) else left
|
||||||
|
next_level.append(hashlib.sha256((left + right).encode()).hexdigest())
|
||||||
|
level = next_level
|
||||||
|
return level[0]
|
||||||
|
|
||||||
|
|
||||||
|
def signal_to_dict(s: Signal) -> dict:
|
||||||
|
return {
|
||||||
|
"venue": s.venue,
|
||||||
|
"timestamp": s.timestamp,
|
||||||
|
"metrics": s.metrics,
|
||||||
|
"version": s.version,
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Dict, List
|
||||||
|
|
||||||
|
from .contracts import Signal, SignalDelta
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GraphOfContractsRegistry:
|
||||||
|
version: int = 1
|
||||||
|
contracts: Dict[str, Dict] = field(default_factory=dict)
|
||||||
|
adapters: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
def register_contract(self, name: str, contract: dict) -> None:
|
||||||
|
self.contracts[name] = contract
|
||||||
|
|
||||||
|
def list_contracts(self) -> List[str]:
|
||||||
|
return list(self.contracts.keys())
|
||||||
|
|
||||||
|
def add_adapter(self, adapter_name: str) -> None:
|
||||||
|
if adapter_name not in self.adapters:
|
||||||
|
self.adapters.append(adapter_name)
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from typing import List, Dict
|
||||||
|
|
||||||
|
from .contracts import Signal
|
||||||
|
from .analytics import Aggregator
|
||||||
|
|
||||||
|
app = FastAPI(title="MercuryMesh MVP API")
|
||||||
|
|
||||||
|
|
||||||
|
class SignalDTO(BaseModel):
|
||||||
|
venue: str
|
||||||
|
timestamp: int
|
||||||
|
metrics: Dict[str, float]
|
||||||
|
version: int = 1
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/signals/aggregate")
|
||||||
|
def aggregate_signals(signals: List[SignalDTO]):
|
||||||
|
# Convert to internal Signal objects for aggregation
|
||||||
|
objs = [Signal(venue=s.venue, timestamp=s.timestamp, metrics=s.metrics, version=s.version) for s in signals]
|
||||||
|
result = Aggregator.aggregate_signals(objs)
|
||||||
|
return {"aggregation": result}
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# Ensure the repository root is on sys.path when tests are run from within the tests package
|
||||||
|
# This helps pytest locate the top-level package mercurymesh_privacy_preserving_market_da
|
||||||
|
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||||
|
if ROOT_DIR not in sys.path:
|
||||||
|
sys.path.insert(0, ROOT_DIR)
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
from mercurymesh_privacy_preserving_market_da.contracts import Signal
|
||||||
|
from mercurymesh_privacy_preserving_market_da.delta_sync import DeltaSyncEngine
|
||||||
|
from mercurymesh_privacy_preserving_market_da.provenance import MerkleProvenance
|
||||||
|
|
||||||
|
|
||||||
|
def test_signal_dataclass_json_roundtrip():
|
||||||
|
s = Signal(venue="TestVenue", timestamp=1, metrics={"a": 0.5}, version=1)
|
||||||
|
# ensure JSON-ish serialization works via __dict__ and that fields exist
|
||||||
|
assert s.venue == "TestVenue"
|
||||||
|
assert isinstance(s.metrics, dict)
|
||||||
|
|
||||||
|
|
||||||
|
def test_delta_sync_reconcile_basic():
|
||||||
|
a = Signal(venue="V1", timestamp=100, metrics={"x": 1.0})
|
||||||
|
b = Signal(venue="V1", timestamp=100, metrics={"x": 0.5})
|
||||||
|
local = [a]
|
||||||
|
remote = [b]
|
||||||
|
deltas = DeltaSyncEngine.reconcile(local, remote)
|
||||||
|
assert isinstance(deltas, list)
|
||||||
|
assert deltas and deltas[0].venue == "V1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merkle_provenance_root_basic():
|
||||||
|
from mercurymesh_privacy_preserving_market_da.contracts import Signal
|
||||||
|
|
||||||
|
s1 = Signal(venue="A", timestamp=1, metrics={"m": 1}, version=1)
|
||||||
|
s2 = Signal(venue="B", timestamp=2, metrics={"m": 2}, version=1)
|
||||||
|
root = MerkleProvenance.merkle_root([s1, s2])
|
||||||
|
assert isinstance(root, str) and len(root) > 0
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools", "wheel"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "mercurymesh-privacy-preserving-market-da"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Privacy-preserving market-data federation MVP (MercuryMesh)"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.9"
|
||||||
|
dependencies = [
|
||||||
|
"fastapi>=0.101.0",
|
||||||
|
"uvicorn[standard]>=0.22.0",
|
||||||
|
"pydantic>=1.10.0",
|
||||||
|
"pytest>=7.0.0",
|
||||||
|
]
|
||||||
Loading…
Reference in New Issue