mirror of
https://github.com/vitorpamplona/amethyst.git
synced 2026-10-30 15:48:36 +00:00
Claude
dc43b6ab67
Crawled 743 of 5,846 reachable relays with amy for events signed 2026-10-07..08: 2.36M events, 573 kinds, 386 without a typed quartz class. Adds the crawl/harvest/profile scripts, the kind histogram, the per-kind profiles and researched findings (what each kind is, spec/source, relevance to Amethyst). Raw samples stay local: they hold third-party personal data. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FrQx1ZvQka64aFAnZM1Hjd
52 lines
2.3 KiB
Python
52 lines
2.3 KiB
Python
"""Fold one amy store (events.db) into the compact census DB, then the caller can delete it.
|
|
|
|
census.db keeps:
|
|
ev(id, kind, pubkey, created_at) every distinct event seen (all kinds)
|
|
raw(id, kind, json) full signed JSON, only for kinds quartz has no typed class for
|
|
relay_kind(relay, kind, events) how many events of each kind each relay served
|
|
"""
|
|
import json, sqlite3
|
|
|
|
SCHEMA = """
|
|
create table if not exists ev(id text primary key, kind int, pubkey text, created_at int) without rowid;
|
|
create table if not exists raw(id text primary key, kind int, json text) without rowid;
|
|
create table if not exists relay_kind(relay text, kind int, events int, primary key(relay, kind)) without rowid;
|
|
"""
|
|
|
|
|
|
def open_census(path):
|
|
# check_same_thread=False: crawl.py shares one connection across workers behind a lock.
|
|
c = sqlite3.connect(path, timeout=120, check_same_thread=False)
|
|
c.executescript(SCHEMA)
|
|
c.execute("pragma journal_mode=wal")
|
|
c.execute("pragma synchronous=normal")
|
|
return c
|
|
|
|
|
|
def harvest(census, store_path, relay, since, known):
|
|
src = sqlite3.connect(f"file:{store_path}?mode=ro", uri=True, timeout=60)
|
|
rows = src.execute(
|
|
"select id,pubkey,created_at,kind,tags,content,sig from event_headers where created_at >= ?", (since,)
|
|
).fetchall()
|
|
src.close()
|
|
per_kind = {}
|
|
ev_rows, raw_rows = [], []
|
|
for id_, pk, ca, kind, tags, content, sig in rows:
|
|
per_kind[kind] = per_kind.get(kind, 0) + 1
|
|
ev_rows.append((id_, kind, pk, ca))
|
|
if kind not in known:
|
|
raw_rows.append((id_, kind, json.dumps(
|
|
{"id": id_, "pubkey": pk, "created_at": ca, "kind": kind,
|
|
"tags": json.loads(tags), "content": content, "sig": sig}, ensure_ascii=False)))
|
|
with census:
|
|
census.executemany("insert or ignore into ev values (?,?,?,?)", ev_rows)
|
|
census.executemany("insert or ignore into raw values (?,?,?)", raw_rows)
|
|
census.executemany(
|
|
"insert into relay_kind values (?,?,?) on conflict(relay,kind) do update set events=max(events, excluded.events)",
|
|
[(relay, k, n) for k, n in per_kind.items()])
|
|
return len(rows)
|
|
|
|
|
|
def load_known(tsv):
|
|
return {int(l.split("\t")[0]) for l in open(tsv) if l.strip()}
|