laantungir/amethyst
1
0
Fork 0
mirror of https://github.com/vitorpamplona/amethyst.git synced 2026-07-22 15:52:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(quartz): strip NIP-50 extension tokens before FTS MATCH in the relay store path

Browse Source 
Raw key:value tokens like include:spam reached SQLite FTS MATCH, where
the colon is column-filter syntax — any REQ carrying an extension token
died with CLOSED "no such column: include" instead of matching.

Adds SearchQuery.stripExtensions() plus Filter/List<Filter>
.strippingSearchExtensions() so EventStore users can drop the tokens
before querying, and applies them in LiveEventStore (query, count,
negentropy snapshots). Per NIP-50, unsupported extensions are ignored:
an extensions-only search becomes unconstrained, not match-nothing.
EventSource-backed search relays still receive the raw string since a
real search backend wants the extensions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Tujoyfc2kNLZNVgiLZAR7F
This commit is contained in:
Claude
2026-07-03 18:09:37 +00:00
parent ab7abd9614
commit 7deda28d29
5 changed files with 167 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
quartz/RELAY.md
View File

@@ -257,6 +257,16 @@ q.domain // "example.com"
q.nsfwIncluded // false (NIP-50 default is true when the token is absent)
```
The SQLite store expects `Filter.search` to be plain FTS text: `:` is FTS5
column-filter syntax, so a raw `include:spam` reaching MATCH raises "no such
column: include" instead of matching. Strip the tokens before querying with
`SearchQuery.stripExtensions(raw)` or `filter.strippingSearchExtensions()` —
an extensions-only query collapses to an empty search, which imposes no
constraint (NIP-50: unsupported extensions are ignored, not match-nothing).
The storage-backed server path (`NostrServer` / `LiveEventStore`) already does
this, so relays like geode comply out of the box; `EventSource` backends get
the raw string because a real search backend wants the extensions.
A search/redirector relay is just a custom policy (or, for computed results, a
custom `IEventStore` whose `query` answers the REQ) that reads the parsed query:

20
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt
View File

@@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import com.vitorpamplona.quartz.nip01Core.relay.filters.FilterIndex
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.IdAndTime
import com.vitorpamplona.quartz.nip50Search.strippingSearchExtensions
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.awaitCancellation
import kotlin.concurrent.atomics.AtomicBoolean
@@ -44,6 +45,15 @@ import kotlin.concurrent.atomics.ExperimentalAtomicApi
* match. This avoids the quadratic O(N_subscribers × N_filters_per_sub) per-event walk
* that a SharedFlow-based broadcast would do.
*
* NIP-50 `search` strings are handed to the store with their `key:value`
* extension tokens stripped ([strippingSearchExtensions]): the SQLite FTS
* backend treats `:` as column-filter syntax, so a raw `include:spam`
* would raise "no such column" instead of matching. Per NIP-50, an
* unsupported extension is ignored — an extensions-only search therefore
* becomes unconstrained, not match-nothing. Relays that *do* implement
* extensions serve search through an [EventSource] backend, which
* receives the raw string.
*
* @property store The underlying persistent storage for events.
* @property ingest The group-commit writer pipeline. Accepted events fan out via the
* index; rejected events do not.
@@ -177,7 +187,7 @@ class LiveEventStore(
index.register(filters, sub)
try {
store.query<Event>(filters) { event ->
store.query<Event>(filters.strippingSearchExtensions()) { event ->
seenLocked { seenIds?.add(event.id) }
onEach(event)
}
@@ -198,14 +208,14 @@ class LiveEventStore(
override suspend fun count(
ctx: RequestContext,
filters: List<Filter>,
): Int = store.count(filters)
): Int = store.count(filters.strippingSearchExtensions())
/**
* One-shot snapshot query. Used by NIP-77 negentropy: the server
* needs the full set of event ids matching the filter at the
* moment the NEG-OPEN arrives, not a streamed/live result.
*/
suspend fun snapshotQuery(filter: Filter): List<Event> = store.query(filter)
suspend fun snapshotQuery(filter: Filter): List<Event> = store.query(filter.strippingSearchExtensions())
/**
* Multi-filter snapshot. Unions the per-filter results and
@@ -218,7 +228,7 @@ class LiveEventStore(
val seen = HashSet<String>()
val merged = ArrayList<Event>()
for (f in filters) {
for (e in store.query<Event>(f)) {
for (e in store.query<Event>(f.strippingSearchExtensions())) {
if (seen.add(e.id)) merged += e
}
}
@@ -238,5 +248,5 @@ class LiveEventStore(
override suspend fun snapshotIdsForNegentropy(
filters: List<Filter>,
maxEntries: Int?,
): List<IdAndTime> = store.snapshotIdsForNegentropy(filters, maxEntries)
): List<IdAndTime> = store.snapshotIdsForNegentropy(filters.strippingSearchExtensions(), maxEntries)
}

48
quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt
View File

@@ -20,6 +20,8 @@
*/
package com.vitorpamplona.quartz.nip50Search
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
/**
* Parsed representation of a NIP-50 `search` filter string.
*
@@ -163,9 +165,55 @@ class SearchQuery(
}
private fun isExtensionKey(key: String): Boolean = key.isNotEmpty() && key.all { it in 'a'..'z' }
/**
* Returns [search] with every `key:value` extension token removed,
* leaving only the free-text terms (tokenized as in [parse]).
*
* Backends that hand the search string to an engine with its own
* query syntax — e.g. SQLite FTS, where `:` is column-filter
* syntax and `include:spam` raises "no such column" — must call
* this (or apply the extensions themselves) before querying.
*
* A string with no extension tokens is returned unchanged. An
* extensions-only query collapses to `""`, which event stores
* treat as "no search constraint" — the NIP-50 behaviour for
* relays that don't support an extension is to ignore it, not to
* return nothing.
*/
fun stripExtensions(search: String?): String? {
if (search.isNullOrBlank()) return search
val parsed = parse(search)
return if (parsed.extensions.isEmpty()) search else parsed.terms
}
}
}
/**
* Returns a copy of this filter whose `search` string has all NIP-50
* extension tokens removed (see [SearchQuery.stripExtensions]), or this
* same instance when there is nothing to strip.
*/
fun Filter.strippingSearchExtensions(): Filter {
val stripped = SearchQuery.stripExtensions(search)
return if (stripped == search) this else copy(search = stripped)
}
/**
* Applies [strippingSearchExtensions] to every filter, returning this
* same list when no filter carried extension tokens.
*/
fun List<Filter>.strippingSearchExtensions(): List<Filter> {
var changed = false
val out =
map {
val stripped = it.strippingSearchExtensions()
if (stripped !== it) changed = true
stripped
}
return if (changed) out else this
}
/** NIP-50 `sentiment:` extension values. */
enum class Sentiment(
val code: String,

39
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NostrServerTest.kt
View File

@@ -30,6 +30,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.server.policies.EmptyPolicy
import com.vitorpamplona.quartz.nip01Core.relay.server.policies.IRelayPolicy
import com.vitorpamplona.quartz.nip01Core.store.IEventStore
import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore
import com.vitorpamplona.quartz.utils.EventFactory
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.test.UnconfinedTestDispatcher
import kotlinx.coroutines.test.runTest
@@ -199,6 +200,44 @@ class NostrServerTest {
server.close()
}
@Test
fun reqWithSearchExtensionsIsAnsweredNotErrored() =
runTest {
val dispatcher = UnconfinedTestDispatcher(testScheduler)
val store = EventStore(null)
val server = createServer(dispatcher, store)
// Build through EventFactory so kind 1 materializes as a
// TextNoteEvent — only SearchableEvent instances are FTS-indexed.
store.insert(EventFactory.create(hexId(1), pubkey, 100L, 1, emptyArray(), "bitcoin is money", sig))
store.insert(EventFactory.create(hexId(2), pubkey, 200L, 1, emptyArray(), "coffee time", sig))
val collector = MessageCollector()
val c1 = server.connect(collector.sendCallback)
// The raw string would be FTS5 syntax ("no such column: include")
// if it reached MATCH unstripped — the REQ must answer, not CLOSE.
c1.receive("""["REQ","sub1",{"search":"bitcoin include:spam"}]""")
var events = collector.parsedEventMessages().filterIsInstance<EventMessage>()
assertEquals(1, events.size)
assertEquals("bitcoin is money", events[0].event.content)
assertEquals(1, collector.parsedEventMessages().filterIsInstance<EoseMessage>().size)
assertTrue(collector.rawMessagesContaining("CLOSED").isEmpty())
// Extensions-only search: NIP-50 ignores unsupported extensions,
// so the search constraint drops and the filter is unconstrained.
c1.receive("""["CLOSE","sub1"]""")
collector.messages.clear()
c1.receive("""["REQ","sub2",{"search":"language:en"}]""")
events = collector.parsedEventMessages().filterIsInstance<EventMessage>()
assertEquals(2, events.size)
assertTrue(collector.rawMessagesContaining("CLOSED").isEmpty())
server.close()
}
// -- Live subscription -----------------------------------------------------
@Test

55
quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt
View File

@@ -20,10 +20,12 @@
*/
package com.vitorpamplona.quartz.nip50Search
import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNull
import kotlin.test.assertSame
import kotlin.test.assertTrue
class SearchQueryTest {
@@ -144,4 +146,57 @@ class SearchQueryTest {
val q = SearchQuery.parse("nsfw:false")
assertEquals("nsfw:false", q.toSearchString())
}
@Test
fun stripExtensionsRemovesTokens() {
assertEquals("bitcoin", SearchQuery.stripExtensions("bitcoin include:spam"))
assertEquals("best nostr apps", SearchQuery.stripExtensions("best domain:example.com nostr language:en apps"))
// Unknown extension keys are stripped too — the store can't apply them either.
assertEquals("hello", SearchQuery.stripExtensions("hello foo:bar"))
}
@Test
fun stripExtensionsOnExtensionsOnlyYieldsEmpty() {
// Empty, not null: an empty search imposes no constraint (NIP-50
// says unsupported extensions are ignored, not match-nothing).
assertEquals("", SearchQuery.stripExtensions("include:spam nsfw:false"))
}
@Test
fun stripExtensionsPassesPlainQueriesThrough() {
assertNull(SearchQuery.stripExtensions(null))
assertEquals("", SearchQuery.stripExtensions(""))
assertEquals(" ", SearchQuery.stripExtensions(" "))
assertEquals("best nostr apps", SearchQuery.stripExtensions("best nostr apps"))
// URLs and uppercase-key tokens are terms, not extensions.
assertEquals("check https://example.com out", SearchQuery.stripExtensions("check https://example.com out"))
assertEquals("NASA:cool stuff", SearchQuery.stripExtensions("NASA:cool stuff"))
}
@Test
fun filterStrippingSearchExtensions() {
val filter = Filter(kinds = listOf(1), search = "bitcoin include:spam")
val stripped = filter.strippingSearchExtensions()
assertEquals("bitcoin", stripped.search)
assertEquals(listOf(1), stripped.kinds)
// Nothing to strip -> same instance, no copy.
val plain = Filter(kinds = listOf(1), search = "bitcoin")
assertSame(plain, plain.strippingSearchExtensions())
val noSearch = Filter(kinds = listOf(1))
assertSame(noSearch, noSearch.strippingSearchExtensions())
}
@Test
fun filterListStrippingSearchExtensions() {
val plain = Filter(search = "bitcoin")
val extended = Filter(search = "bitcoin language:en")
val untouched = listOf(plain, Filter(kinds = listOf(1)))
assertSame(untouched, untouched.strippingSearchExtensions())
val mixed = listOf(plain, extended).strippingSearchExtensions()
assertSame(plain, mixed[0])
assertEquals("bitcoin", mixed[1].search)
}
}