Monorepo: consolidate 7 repos into one
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 1m5s

Combines shared, blog, market, cart, events, federation, and account
into a single repository. Eliminates submodule sync, sibling model
copying at build time, and per-app CI orchestration.

Changes:
- Remove per-app .git, .gitmodules, .gitea, submodule shared/ dirs
- Remove stale sibling model copies from each app
- Update all 6 Dockerfiles for monorepo build context (root = .)
- Add build directives to docker-compose.yml
- Add single .gitea/workflows/ci.yml with change detection
- Add .dockerignore for monorepo build context
- Create __init__.py for federation and account (cross-app imports)
This commit is contained in:
giles
2026-02-24 19:44:17 +00:00
commit f42042ccb7
895 changed files with 61147 additions and 0 deletions

101
shared/utils/__init__.py Normal file
View File

@@ -0,0 +1,101 @@
import re
from bs4 import BeautifulSoup
import json
import os
from typing import Iterable, Union, List
from quart import request
def soup_of(html: str) -> BeautifulSoup:
return BeautifulSoup(html, "lxml")
def normalize_text(s: str) -> str:
return re.sub(r"\s+", " ", (s or "").strip())
def log(msg: str) -> None:
print(msg, flush=True)
def ensure_dir(path: str) -> None:
os.makedirs(path, exist_ok=True)
def dump_json(path: str, data) -> None:
ensure_dir(os.path.dirname(path))
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def _join_url_parts(parts: List[str]) -> str:
parts = [p for p in parts if p is not None and p != ""]
if not parts:
return ""
# Preserve scheme like "https://"
m = re.match(r"^([a-zA-Z][a-zA-Z0-9+.-]*://)(.*)$", parts[0])
if m:
scheme, first = m.group(1), m.group(2)
else:
scheme, first = "", parts[0]
cleaned = [first.strip("/")]
for seg in parts[1:]:
seg = str(seg)
# If a later segment is already an absolute URL, use it as the base
m2 = re.match(r"^([a-zA-Z][a-zA-Z0-9+.-]*://)(.*)$", seg)
if m2:
scheme, first = m2.group(1), m2.group(2)
cleaned = [first.strip("/")]
elif seg.startswith("?") or seg.startswith("#"):
cleaned[-1] = cleaned[-1] + seg # attach query/fragment
else:
cleaned.append(seg.strip("/"))
url = scheme + "/".join(s for s in cleaned if s != "")
# Preserve trailing slash if caller's last segment had one (and isn't ? or #)
last = str(parts[-1])
if last.endswith("/") and not last.startswith(("?", "#")) and not url.endswith("/"):
url += "/"
return url
def hx_fragment_request() -> bool:
return request.headers.get("HX-Request", "").lower() == "true"
def route_prefix():
return f"{request.scheme}://{request.host}/{request.headers.get('x-forwarded-prefix', '/')}"
def join_url(value: Union[str, Iterable[str]]):
if isinstance(value, str):
parts = [value]
else:
parts = list(value)
return _join_url_parts(parts)
def host_url(value: str='', no_slash=False):
"""
Join g.route with value and ensure the resulting URL has a trailing slash
on the path, but never after query/fragment.
Examples:
http://jjj -> http://jjj/
http://jjj?hello -> http://jjj/?hello
/foo -> /foo/
/foo?x=1#frag -> /foo/?x=1#frag
"""
url = join_url([route_prefix(), value])
# Ensure trailing slash on the PATH (before ? or #)
# Split into: base (no ?/#), optional ?query, optional #fragment
if no_slash:
return url
m = re.match(r'^(?P<base>[^?#]*)(?P<qs>\?[^#]*)?(?P<frag>#.*)?$', url)
if not m:
return url # fallback: return as-is
base = m.group('base') or ""
qs = m.group('qs') or ""
frag = m.group('frag') or ""
if base and not base.endswith('/'):
base += '/'
return f"{base}{qs}{frag}"

236
shared/utils/anchoring.py Normal file
View File

@@ -0,0 +1,236 @@
"""Merkle tree construction and OpenTimestamps anchoring.
Ported from ~/art-dag/activity-pub/anchoring.py.
Builds a SHA256 merkle tree from activity IDs, submits the root to
OpenTimestamps for Bitcoin timestamping, and stores the tree + proof on IPFS.
"""
from __future__ import annotations
import hashlib
import logging
from datetime import datetime, timezone
import httpx
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from shared.models.federation import APActivity, APAnchor
log = logging.getLogger(__name__)
OTS_SERVERS = [
"https://a.pool.opentimestamps.org",
"https://b.pool.opentimestamps.org",
"https://a.pool.eternitywall.com",
]
def _sha256(data: str | bytes) -> str:
"""SHA256 hex digest."""
if isinstance(data, str):
data = data.encode()
return hashlib.sha256(data).hexdigest()
def build_merkle_tree(items: list[str]) -> dict:
"""Build a SHA256 merkle tree from a list of strings (activity IDs).
Returns:
{
"root": hex_str,
"leaves": [hex_str, ...],
"levels": [[hex_str, ...], ...], # bottom-up
}
"""
if not items:
raise ValueError("Cannot build merkle tree from empty list")
# Sort for deterministic ordering
items = sorted(items)
# Leaf hashes
leaves = [_sha256(item) for item in items]
levels = [leaves[:]]
current = leaves[:]
while len(current) > 1:
next_level = []
for i in range(0, len(current), 2):
left = current[i]
right = current[i + 1] if i + 1 < len(current) else left
combined = _sha256(left + right)
next_level.append(combined)
levels.append(next_level)
current = next_level
return {
"root": current[0],
"leaves": leaves,
"levels": levels,
}
def get_merkle_proof(tree: dict, item: str) -> list[dict] | None:
"""Generate a proof-of-membership for an item.
Returns a list of {sibling: hex, position: "left"|"right"} dicts,
or None if the item is not in the tree.
"""
item_hash = _sha256(item)
leaves = tree["leaves"]
try:
idx = leaves.index(item_hash)
except ValueError:
return None
proof = []
for level in tree["levels"][:-1]: # skip root level
if idx % 2 == 0:
sibling_idx = idx + 1
position = "right"
else:
sibling_idx = idx - 1
position = "left"
if sibling_idx < len(level):
proof.append({"sibling": level[sibling_idx], "position": position})
else:
proof.append({"sibling": level[idx], "position": position})
idx = idx // 2
return proof
def verify_merkle_proof(item: str, proof: list[dict], root: str) -> bool:
"""Verify a merkle proof against a root hash."""
current = _sha256(item)
for step in proof:
sibling = step["sibling"]
if step["position"] == "right":
current = _sha256(current + sibling)
else:
current = _sha256(sibling + current)
return current == root
async def submit_to_opentimestamps(merkle_root: str) -> bytes | None:
"""Submit a hash to OpenTimestamps. Returns the (incomplete) OTS proof bytes."""
root_bytes = bytes.fromhex(merkle_root)
for server in OTS_SERVERS:
try:
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
f"{server}/digest",
content=root_bytes,
headers={"Content-Type": "application/x-opentimestamps"},
)
if resp.status_code == 200:
log.info("OTS proof obtained from %s", server)
return resp.content
except Exception:
log.debug("OTS server %s failed", server, exc_info=True)
continue
log.warning("All OTS servers failed for root %s", merkle_root)
return None
async def upgrade_ots_proof(proof_bytes: bytes) -> tuple[bytes, bool]:
"""Try to upgrade an incomplete OTS proof to a Bitcoin-confirmed one.
Returns (proof_bytes, is_confirmed). The proof_bytes may be updated.
"""
# OpenTimestamps upgrade is done via the `ots` CLI or their calendar API.
# For now, return the proof as-is with is_confirmed=False.
# Calendar-based upgrade polling not yet implemented.
return proof_bytes, False
async def create_anchor(
session: AsyncSession,
batch_size: int = 100,
) -> APAnchor | None:
"""Anchor a batch of un-anchored activities.
1. Select activities without an anchor_id
2. Build merkle tree from their activity_ids
3. Store tree on IPFS
4. Submit root to OpenTimestamps
5. Store OTS proof on IPFS
6. Create APAnchor record
7. Link activities to anchor
"""
# Find un-anchored activities
result = await session.execute(
select(APActivity)
.where(
APActivity.anchor_id.is_(None),
APActivity.is_local == True, # noqa: E712
)
.order_by(APActivity.created_at.asc())
.limit(batch_size)
)
activities = result.scalars().all()
if not activities:
log.debug("No un-anchored activities to process")
return None
activity_ids = [a.activity_id for a in activities]
log.info("Anchoring %d activities", len(activity_ids))
# Build merkle tree
tree = build_merkle_tree(activity_ids)
merkle_root = tree["root"]
# Store tree on IPFS
tree_cid = None
ots_proof_cid = None
try:
from shared.utils.ipfs_client import add_json, add_bytes, is_available
if await is_available():
tree_cid = await add_json({
"root": merkle_root,
"leaves": tree["leaves"],
"activity_ids": activity_ids,
"created_at": datetime.now(timezone.utc).isoformat(),
})
log.info("Merkle tree stored on IPFS: %s", tree_cid)
except Exception:
log.exception("IPFS tree storage failed")
# Submit to OpenTimestamps
ots_proof = await submit_to_opentimestamps(merkle_root)
if ots_proof:
try:
from shared.utils.ipfs_client import add_bytes, is_available
if await is_available():
ots_proof_cid = await add_bytes(ots_proof)
log.info("OTS proof stored on IPFS: %s", ots_proof_cid)
except Exception:
log.exception("IPFS OTS proof storage failed")
# Create anchor record
anchor = APAnchor(
merkle_root=merkle_root,
tree_ipfs_cid=tree_cid,
ots_proof_cid=ots_proof_cid,
activity_count=len(activities),
)
session.add(anchor)
await session.flush()
# Link activities to anchor
for a in activities:
a.anchor_id = anchor.id
await session.flush()
log.info(
"Anchor created: root=%s, activities=%d, tree_cid=%s",
merkle_root, len(activities), tree_cid,
)
return anchor

View File

@@ -0,0 +1,54 @@
"""Pure calendar utility functions (no ORM dependencies).
Extracted from events/bp/calendar/services/calendar_view.py so that
blog admin (and any other app) can use them without cross-app imports.
"""
from __future__ import annotations
import calendar as pycalendar
from datetime import datetime, timezone
from quart import request
def parse_int_arg(name: str, default: int | None = None) -> int | None:
"""Parse an integer query parameter from the request."""
val = request.args.get(name, "").strip()
if not val:
return default
try:
return int(val)
except ValueError:
return default
def add_months(year: int, month: int, delta: int) -> tuple[int, int]:
"""Add (or subtract) months to a given year/month, handling year overflow."""
new_month = month + delta
new_year = year + (new_month - 1) // 12
new_month = ((new_month - 1) % 12) + 1
return new_year, new_month
def build_calendar_weeks(year: int, month: int) -> list[list[dict]]:
"""Build a calendar grid for the given year and month.
Returns a list of weeks, where each week is a list of 7 day dicts.
"""
today = datetime.now(timezone.utc).date()
cal = pycalendar.Calendar(firstweekday=0) # 0 = Monday
weeks: list[list[dict]] = []
for week in cal.monthdatescalendar(year, month):
week_days = []
for d in week:
week_days.append(
{
"date": d,
"in_month": (d.month == month),
"is_today": (d == today),
}
)
weeks.append(week_days)
return weeks

View File

@@ -0,0 +1,181 @@
"""RSA key generation and HTTP Signature signing/verification.
Keys are stored in DB (ActorProfile), not the filesystem.
Ported from ~/art-dag/activity-pub/keys.py.
"""
from __future__ import annotations
import base64
import hashlib
import json
from datetime import datetime, timezone
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa, padding
def generate_rsa_keypair() -> tuple[str, str]:
"""Generate an RSA-2048 keypair.
Returns:
(private_pem, public_pem) as UTF-8 strings.
"""
private_key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
)
private_pem = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption(),
).decode()
public_pem = private_key.public_key().public_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PublicFormat.SubjectPublicKeyInfo,
).decode()
return private_pem, public_pem
def sign_request(
private_key_pem: str,
key_id: str,
method: str,
path: str,
host: str,
body: bytes | None = None,
date: str | None = None,
) -> dict[str, str]:
"""Build HTTP Signature headers for an outgoing request.
Returns a dict of headers to merge into the request:
``{"Signature": ..., "Date": ..., "Digest": ..., "Host": ...}``
"""
if date is None:
date = datetime.now(timezone.utc).strftime("%a, %d %b %Y %H:%M:%S GMT")
headers_to_sign = [
f"(request-target): {method.lower()} {path}",
f"host: {host}",
f"date: {date}",
]
out_headers: dict[str, str] = {
"Host": host,
"Date": date,
}
if body is not None:
digest = base64.b64encode(hashlib.sha256(body).digest()).decode()
digest_header = f"SHA-256={digest}"
headers_to_sign.append(f"digest: {digest_header}")
out_headers["Digest"] = digest_header
signed_string = "\n".join(headers_to_sign)
header_names = " ".join(
h.split(":")[0] for h in headers_to_sign
)
private_key = serialization.load_pem_private_key(
private_key_pem.encode(), password=None,
)
signature_bytes = private_key.sign(
signed_string.encode(),
padding.PKCS1v15(),
hashes.SHA256(),
)
signature_b64 = base64.b64encode(signature_bytes).decode()
out_headers["Signature"] = (
f'keyId="{key_id}",'
f'headers="{header_names}",'
f'signature="{signature_b64}",'
f'algorithm="rsa-sha256"'
)
return out_headers
def verify_request_signature(
public_key_pem: str,
signature_header: str,
method: str,
path: str,
headers: dict[str, str],
) -> bool:
"""Verify an incoming HTTP Signature.
Args:
public_key_pem: PEM-encoded public key of the sender.
signature_header: Value of the ``Signature`` header.
method: HTTP method (GET, POST, etc.).
path: Request path (e.g. ``/users/alice/inbox``).
headers: All request headers (case-insensitive keys).
Returns:
True if the signature is valid.
"""
# Parse Signature header
parts: dict[str, str] = {}
for part in signature_header.split(","):
part = part.strip()
eq = part.index("=")
key = part[:eq]
val = part[eq + 1:].strip('"')
parts[key] = val
signed_headers = parts.get("headers", "date").split()
signature_b64 = parts.get("signature", "")
# Reconstruct the signed string
lines: list[str] = []
# Normalize header lookup to lowercase
lc_headers = {k.lower(): v for k, v in headers.items()}
for h in signed_headers:
if h == "(request-target)":
lines.append(f"(request-target): {method.lower()} {path}")
else:
val = lc_headers.get(h, "")
lines.append(f"{h}: {val}")
signed_string = "\n".join(lines)
public_key = serialization.load_pem_public_key(public_key_pem.encode())
try:
public_key.verify(
base64.b64decode(signature_b64),
signed_string.encode(),
padding.PKCS1v15(),
hashes.SHA256(),
)
return True
except Exception:
return False
def create_ld_signature(
private_key_pem: str,
key_id: str,
activity: dict,
) -> dict:
"""Create an RsaSignature2017 Linked Data signature for an activity."""
canonical = json.dumps(activity, sort_keys=True, separators=(",", ":"))
private_key = serialization.load_pem_private_key(
private_key_pem.encode(), password=None,
)
signature_bytes = private_key.sign(
canonical.encode(),
padding.PKCS1v15(),
hashes.SHA256(),
)
signature_b64 = base64.b64encode(signature_bytes).decode()
return {
"type": "RsaSignature2017",
"creator": key_id,
"created": datetime.now(timezone.utc).isoformat(),
"signatureValue": signature_b64,
}

141
shared/utils/ipfs_client.py Normal file
View File

@@ -0,0 +1,141 @@
"""Async IPFS client for content-addressed storage.
All content can be stored on IPFS — blog posts, products, activities, etc.
Ported from ~/art-dag/activity-pub/ipfs_client.py (converted to async httpx).
Config via environment:
IPFS_API — multiaddr or URL (default: /ip4/127.0.0.1/tcp/5001)
IPFS_TIMEOUT — request timeout in seconds (default: 60)
IPFS_GATEWAY_URL — public gateway for CID links (default: https://ipfs.io)
"""
from __future__ import annotations
import json
import logging
import os
import re
import httpx
logger = logging.getLogger(__name__)
class IPFSError(Exception):
"""Raised when an IPFS operation fails."""
# -- Config ------------------------------------------------------------------
IPFS_API = os.getenv("IPFS_API", "/ip4/127.0.0.1/tcp/5001")
IPFS_TIMEOUT = int(os.getenv("IPFS_TIMEOUT", "60"))
IPFS_GATEWAY_URL = os.getenv("IPFS_GATEWAY_URL", "https://ipfs.io")
def _multiaddr_to_url(multiaddr: str) -> str:
"""Convert IPFS multiaddr to HTTP URL."""
dns_match = re.match(r"/dns[46]?/([^/]+)/tcp/(\d+)", multiaddr)
if dns_match:
return f"http://{dns_match.group(1)}:{dns_match.group(2)}"
ip4_match = re.match(r"/ip4/([^/]+)/tcp/(\d+)", multiaddr)
if ip4_match:
return f"http://{ip4_match.group(1)}:{ip4_match.group(2)}"
if multiaddr.startswith("http"):
return multiaddr
return "http://127.0.0.1:5001"
IPFS_BASE_URL = _multiaddr_to_url(IPFS_API)
# -- Async client functions --------------------------------------------------
async def add_bytes(data: bytes, *, pin: bool = True) -> str:
"""Add raw bytes to IPFS.
Returns the CID.
"""
try:
async with httpx.AsyncClient(timeout=IPFS_TIMEOUT) as client:
resp = await client.post(
f"{IPFS_BASE_URL}/api/v0/add",
params={"pin": str(pin).lower()},
files={"file": ("data", data)},
)
resp.raise_for_status()
cid = resp.json()["Hash"]
logger.info("Added to IPFS: %d bytes -> %s", len(data), cid)
return cid
except Exception as e:
logger.error("Failed to add bytes to IPFS: %s", e)
raise IPFSError(f"Failed to add bytes: {e}") from e
async def add_json(data: dict) -> str:
"""Serialize dict to sorted JSON and add to IPFS."""
json_bytes = json.dumps(data, indent=2, sort_keys=True).encode("utf-8")
return await add_bytes(json_bytes, pin=True)
async def get_bytes(cid: str) -> bytes | None:
"""Fetch content from IPFS by CID."""
try:
async with httpx.AsyncClient(timeout=IPFS_TIMEOUT) as client:
resp = await client.post(
f"{IPFS_BASE_URL}/api/v0/cat",
params={"arg": cid},
)
resp.raise_for_status()
logger.info("Retrieved from IPFS: %s (%d bytes)", cid, len(resp.content))
return resp.content
except Exception as e:
logger.error("Failed to get from IPFS: %s", e)
return None
async def pin_cid(cid: str) -> bool:
"""Pin a CID on this node."""
try:
async with httpx.AsyncClient(timeout=IPFS_TIMEOUT) as client:
resp = await client.post(
f"{IPFS_BASE_URL}/api/v0/pin/add",
params={"arg": cid},
)
resp.raise_for_status()
logger.info("Pinned on IPFS: %s", cid)
return True
except Exception as e:
logger.error("Failed to pin on IPFS: %s", e)
return False
async def unpin_cid(cid: str) -> bool:
"""Unpin a CID from this node."""
try:
async with httpx.AsyncClient(timeout=IPFS_TIMEOUT) as client:
resp = await client.post(
f"{IPFS_BASE_URL}/api/v0/pin/rm",
params={"arg": cid},
)
resp.raise_for_status()
logger.info("Unpinned from IPFS: %s", cid)
return True
except Exception as e:
logger.error("Failed to unpin from IPFS: %s", e)
return False
async def is_available() -> bool:
"""Check if IPFS daemon is reachable."""
try:
async with httpx.AsyncClient(timeout=5) as client:
resp = await client.post(f"{IPFS_BASE_URL}/api/v0/id")
return resp.status_code == 200
except Exception:
return False
def gateway_url(cid: str) -> str:
"""Return a public gateway URL for a CID."""
return f"{IPFS_GATEWAY_URL}/ipfs/{cid}"

68
shared/utils/webfinger.py Normal file
View File

@@ -0,0 +1,68 @@
"""WebFinger client for resolving remote AP actor profiles."""
from __future__ import annotations
import logging
import httpx
log = logging.getLogger(__name__)
AP_CONTENT_TYPE = "application/activity+json"
async def resolve_actor(acct: str) -> dict | None:
"""Resolve user@domain to actor JSON via WebFinger + actor fetch.
Args:
acct: Handle in the form ``user@domain`` (no leading ``@``).
Returns:
Actor JSON-LD dict, or None if resolution fails.
"""
acct = acct.lstrip("@")
if "@" not in acct:
return None
_, domain = acct.rsplit("@", 1)
webfinger_url = f"https://{domain}/.well-known/webfinger"
try:
async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
# Step 1: WebFinger lookup
resp = await client.get(
webfinger_url,
params={"resource": f"acct:{acct}"},
headers={"Accept": "application/jrd+json, application/json"},
)
if resp.status_code != 200:
log.debug("WebFinger %s returned %d", webfinger_url, resp.status_code)
return None
data = resp.json()
# Find self link with AP content type
actor_url = None
for link in data.get("links", []):
if link.get("rel") == "self" and link.get("type") in (
AP_CONTENT_TYPE,
"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
):
actor_url = link.get("href")
break
if not actor_url:
log.debug("No AP self link in WebFinger response for %s", acct)
return None
# Step 2: Fetch actor JSON
resp = await client.get(
actor_url,
headers={"Accept": AP_CONTENT_TYPE},
)
if resp.status_code == 200:
return resp.json()
log.debug("Actor fetch %s returned %d", actor_url, resp.status_code)
except Exception:
log.exception("WebFinger resolution failed for %s", acct)
return None