feat: migrate parsing logic to Zipline (#1183)

Replaces the Kspoon-based parsing via Retrofit into a lower level
version based on Ksoup directly, implemented in a Zipline module that
can be updated independently of the app itself.

The primary motivation is to be able to update the HTML scraper
independent of the app since the site will likely to continue having
changes in the markup that break my parsers, and cutting emergency
releases is tiresome.

~~This currently doesn't work due to a `stack overflow` error from
QuickJS, but it did work *briefly* before I managed to break it and
never got it working again.~~

Managed to fix the problems through multiple changes

- Improve code size by removing JSON as an intermediary between the
Zipline-defined API and the Android app, instead using a simpler text
based format that's just a straight concatenation. This let us drop
`kotlinx-serialization-json` from the module dependencies.
- Once the code size was under control we started seeing crashes because
we were violating JS threading invariants where JS loading and JS
execution needed to happen on the same thread. This was resolved using
the `DispatcherConfinedLobstersParserService` wrapper that forces all
parser method calls to run through the same Dispatcher as
`ZiplineLoader`.

android/build.gradle.kts

@@ -8,6 +8,9 @@
 
 import dev.msfjarvis.claw.gradle.addTestDependencies
 import dev.zacsweers.metro.gradle.DiagnosticSeverity
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import java.io.File
 
 plugins {
   id("dev.msfjarvis.claw.android-application")
@@ -25,22 +28,107 @@ plugins {
   alias(libs.plugins.kotlin.serialization)
   alias(libs.plugins.dependencyAnalysis)
   alias(libs.plugins.metro)
+  alias(libs.plugins.zipline)
 }
 
+val ziplineDevManifestUrl =
+  providers
+    .gradleProperty("claw.ziplineParserManifestUrl")
+    .orElse("http://10.0.2.2:8080/manifest.zipline.json")
+    .get()
+val ziplineProdManifestUrl = "https://claw.msfjarvis.dev/current/manifest.zipline.json"
+val embeddedZiplineAssetsRoot = layout.buildDirectory.dir("generated/ziplineEmbeddedAssets")
+val embeddedZiplineAssetsRootFile =
+  layout.buildDirectory.get().dir("generated/ziplineEmbeddedAssets").asFile
+val prepareEmbeddedZiplineParser =
+  tasks.register("prepareEmbeddedZiplineParser") {
+    val sourceDir =
+      rootProject.layout.projectDirectory.dir("zipline-parser/build/zipline/Production")
+    val outputRoot = embeddedZiplineAssetsRoot
+    inputs.dir(sourceDir)
+    outputs.dir(outputRoot)
+    dependsOn(":zipline-parser:compileProductionExecutableKotlinJsZipline")
+    description = "Embed compiled Zipline artifacts into the app"
+
+    doLast {
+      val outputDir = outputRoot.get().asFile
+      val ziplineDir = File(outputDir, "zipline")
+      ziplineDir.deleteRecursively()
+      ziplineDir.mkdirs()
+
+      val sourceManifestFile = File(sourceDir.asFile, "manifest.zipline.json")
+      check(sourceManifestFile.exists()) { "Missing Zipline manifest: $sourceManifestFile" }
+
+      @Suppress("UNCHECKED_CAST")
+      val parsed = JsonSlurper().parse(sourceManifestFile) as MutableMap<String, Any?>
+      val modules = parsed["modules"] as? Map<*, *>
+      check(!modules.isNullOrEmpty()) {
+        "Zipline manifest contains no modules: $sourceManifestFile"
+      }
+
+      for ((_, moduleValue) in modules) {
+        val module = moduleValue as Map<*, *>
+        val sourceName = module["url"] as String
+        val targetName = module["sha256"] as String
+        val sourceFile = File(sourceDir.asFile, sourceName)
+        check(sourceFile.exists()) { "Missing Zipline module: $sourceFile" }
+        sourceFile.copyTo(File(ziplineDir, targetName), overwrite = true)
+      }
+
+      val unsigned =
+        ((parsed["unsigned"] as? Map<*, *>)?.toMutableMap() ?: mutableMapOf()).apply {
+          put("freshAtEpochMs", System.currentTimeMillis())
+        }
+      parsed["unsigned"] = unsigned
+
+      val embeddedManifestFile = File(ziplineDir, "zipline-parser.manifest.zipline.json")
+      embeddedManifestFile.writeText(JsonOutput.toJson(parsed))
+    }
+  }
+
 android {
   namespace = "dev.msfjarvis.claw.android"
   defaultConfig.applicationId = "dev.msfjarvis.claw.android"
   defaultConfig.testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
   buildFeatures.compose = true
   experimentalProperties["android.experimental.enableScreenshotTest"] = true
-  buildTypes.create("internal") {
-    matchingFallbacks += "release"
-    signingConfig = signingConfigs["debug"]
-    applicationIdSuffix = ".internal"
-    isDebuggable = true
+
+  sourceSets.getByName("main").assets.directories.add(embeddedZiplineAssetsRootFile.path)
+
+  buildTypes {
+    getByName("debug") {
+      buildConfigField(
+        "String",
+        "ZIPLINE_PARSER_MANIFEST_URL",
+        "\"$ziplineDevManifestUrl\"",
+      )
+      buildConfigField("boolean", "ZIPLINE_PARSER_VERIFY_SIGNATURES", "false")
+    }
+    getByName("release") {
+      buildConfigField(
+        "String",
+        "ZIPLINE_PARSER_MANIFEST_URL",
+        "\"$ziplineProdManifestUrl\"",
+      )
+      buildConfigField("boolean", "ZIPLINE_PARSER_VERIFY_SIGNATURES", "true")
+    }
+    create("internal") {
+      matchingFallbacks += "release"
+      signingConfig = signingConfigs["debug"]
+      applicationIdSuffix = ".internal"
+      isDebuggable = true
+      buildConfigField(
+        "String",
+        "ZIPLINE_PARSER_MANIFEST_URL",
+        "\"$ziplineProdManifestUrl\"",
+      )
+      buildConfigField("boolean", "ZIPLINE_PARSER_VERIFY_SIGNATURES", "true")
+    }
   }
 }
 
+tasks.named("preBuild").configure { dependsOn(prepareEmbeddedZiplineParser) }
+
 aboutLibraries.collect.gitHubApiToken = providers.environmentVariable("GITHUB_TOKEN").orNull
 
 baselineProfile {
@@ -129,6 +217,8 @@ dependencies {
   implementation(libs.sqldelight.runtime)
   implementation(libs.swipe)
   implementation(libs.unfurl)
+  implementation(libs.zipline)
+  implementation(libs.zipline.loader)
   implementation(projects.api)
   implementation(projects.common)
   implementation(projects.core)

android/src/debug/AndroidManifest.xml

@@ -5,7 +5,7 @@
  ~ https://opensource.org/licenses/MIT.
  -->
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
-  <application>
+  <application android:networkSecurityConfig="@xml/network_security_config">
     <activity
       android:name=".MainActivity"
       android:exported="true">

android/src/debug/res/xml/network_security_config.xml

@@ -0,0 +1,12 @@
+<!--
+ ~ Copyright © Harsh Shandilya.
+ ~ Use of this source code is governed by an MIT-style
+ ~ license that can be found in the LICENSE file or at
+ ~ https://opensource.org/licenses/MIT.
+ -->
+<network-security-config xmlns:tools="http://schemas.android.com/tools">
+    <!-- Only allowed in debug builds -->
+    <base-config cleartextTrafficPermitted="true"
+        tools:ignore="InsecureBaseConfiguration">
+    </base-config>
+</network-security-config>

android/src/main/kotlin/dev/msfjarvis/claw/android/glance/HottestPostsWidget.kt

@@ -59,7 +59,7 @@ class HottestPostsWidget : GlanceAppWidget() {
     val posts =
       when (val postsResult = appGraph.lobstersApi.getHottestPosts(1)) {
         is ApiResult.Success -> {
-          val uiPosts = postsResult.value.posts.map(LobstersPost::toUIPost)
+          val uiPosts = postsResult.value.map(LobstersPost::toUIPost)
           // Cache the posts for future use when network is unavailable
           // Using try-catch to prevent widget rendering failures if caching fails
           try {

android/src/main/kotlin/dev/msfjarvis/claw/android/injection/ZiplineParserModule.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.android.injection
+
+import android.content.Context
+import dev.msfjarvis.claw.android.BuildConfig
+import dev.msfjarvis.claw.android.zipline.AndroidZiplineParserClient
+import dev.msfjarvis.claw.api.LobstersParserClient
+import dev.zacsweers.metro.AppScope
+import dev.zacsweers.metro.BindingContainer
+import dev.zacsweers.metro.ContributesTo
+import dev.zacsweers.metro.Named
+import dev.zacsweers.metro.Provides
+import dev.zacsweers.metro.SingleIn
+import okhttp3.OkHttpClient
+
+@BindingContainer
+@ContributesTo(AppScope::class)
+object ZiplineParserModule {
+  @Provides
+  @Named("LobstersParserManifestUrl")
+  fun provideLobstersParserManifestUrl(): String = BuildConfig.ZIPLINE_PARSER_MANIFEST_URL
+
+  @Provides
+  @SingleIn(AppScope::class)
+  fun provideParserClient(
+    context: Context,
+    client: OkHttpClient,
+    @Named("LobstersParserManifestUrl") manifestUrl: String,
+  ): LobstersParserClient =
+    AndroidZiplineParserClient(
+      context = context,
+      manifestUrl = manifestUrl,
+      httpClient = client,
+      verifySignatures = BuildConfig.ZIPLINE_PARSER_VERIFY_SIGNATURES,
+    )
+}

android/src/main/kotlin/dev/msfjarvis/claw/android/ui/decorations/ClawAppBar.kt

@@ -44,7 +44,7 @@ import dev.msfjarvis.claw.common.ui.preview.ThemePreviews
 sealed interface ClawTopBarMode {
   data object Browsing : ClawTopBarMode
 
-  data class Searching(
+  class Searching(
     val query: String,
     val expanded: Boolean = true,
     val requestFocus: Boolean = true,

android/src/main/kotlin/dev/msfjarvis/claw/android/ui/navigation/Destination.kt

@@ -35,7 +35,7 @@ interface NonStackable
 
 @Parcelize
 @Serializable
-data class Reply(
+class Reply(
   val postId: String,
   val commentId: String,
 ) : NavKey, Parcelable

android/src/main/kotlin/dev/msfjarvis/claw/android/viewmodel/ClawViewModel.kt

@@ -27,7 +27,6 @@ import dev.msfjarvis.claw.android.paging.LobstersPagingSource.Companion.PAGE_SIZ
 import dev.msfjarvis.claw.android.paging.LobstersPagingSource.Companion.STARTING_PAGE_INDEX
 import dev.msfjarvis.claw.android.paging.SearchPagingSource
 import dev.msfjarvis.claw.api.LobstersApi
-import dev.msfjarvis.claw.api.toPostsResult
 import dev.msfjarvis.claw.core.coroutines.IODispatcher
 import dev.msfjarvis.claw.core.coroutines.MainDispatcher
 import dev.msfjarvis.claw.model.UIPost
@@ -76,7 +75,7 @@ class ClawViewModel(
               config = PagingConfig(pageSize = PAGE_SIZE),
               initialKey = STARTING_PAGE_INDEX,
               pagingSourceFactory = {
-                pagingSourceFactory.create { page -> api.getHottestPosts(page).toPostsResult() }
+                pagingSourceFactory.create { page -> api.getHottestPosts(page) }
               },
             )
             .flow
@@ -94,7 +93,7 @@ class ClawViewModel(
               config = PagingConfig(pageSize = PAGE_SIZE),
               initialKey = STARTING_PAGE_INDEX,
               pagingSourceFactory = {
-                pagingSourceFactory.create { page -> api.getNewestPosts(page).toPostsResult() }
+                pagingSourceFactory.create { page -> api.getNewestPosts(page) }
               },
             )
             .flow

android/src/main/kotlin/dev/msfjarvis/claw/android/zipline/AndroidZiplineParserClient.kt

@@ -0,0 +1,191 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.android.zipline
+
+import android.content.Context
+import app.cash.zipline.Zipline
+import app.cash.zipline.ZiplineManifest
+import app.cash.zipline.loader.DefaultFreshnessCheckerNotFresh
+import app.cash.zipline.loader.FreshnessChecker
+import app.cash.zipline.loader.LoadResult
+import app.cash.zipline.loader.ManifestVerifier
+import app.cash.zipline.loader.ZiplineLoader
+import dev.msfjarvis.claw.api.LobstersParserClient
+import dev.msfjarvis.claw.parser.LobstersParserService
+import dev.msfjarvis.claw.parser.model.ParserSerializersModule
+import java.io.File
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicReference
+import kotlinx.coroutines.ExecutorCoroutineDispatcher
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.coroutines.withContext
+import okhttp3.OkHttpClient
+import okio.ByteString.Companion.decodeHex
+import okio.FileSystem
+import okio.Path.Companion.toPath
+
+class AndroidZiplineParserClient(
+  private val context: Context,
+  private val manifestUrl: String,
+  private val httpClient: OkHttpClient,
+  private val verifySignatures: Boolean,
+) : LobstersParserClient, AutoCloseable {
+  private val ziplineThread = AtomicReference<Thread>()
+  private val dispatcher: ExecutorCoroutineDispatcher =
+    Executors.newSingleThreadExecutor { runnable ->
+        Thread(
+          null,
+          {
+            ziplineThread.set(Thread.currentThread())
+            runnable.run()
+          },
+          "Claw-ZiplineParser",
+          ZIPLINE_THREAD_STACK_SIZE_BYTES,
+        )
+      }
+      .asCoroutineDispatcher()
+  private val mutex = Mutex()
+  private var loadedZipline: Zipline? = null
+  private var loadedService: LobstersParserService? = null
+
+  override suspend fun service(): LobstersParserService {
+    loadedService?.let {
+      return it
+    }
+    return mutex.withLock { loadedService ?: loadService().also { loadedService = it } }
+  }
+
+  override fun close() {
+    loadedService?.close()
+    loadedService = null
+    loadedZipline?.let { zipline -> onZiplineThread { zipline.close() } }
+    loadedZipline = null
+    dispatcher.close()
+  }
+
+  private suspend fun loadService(): LobstersParserService {
+    extractEmbeddedArtifacts()
+
+    val embeddedDir = File(context.codeCacheDir, "zipline-embedded/zipline")
+    val embeddedManifestFile = File(embeddedDir, "zipline-parser.manifest.zipline.json")
+
+    val manifestVerifier =
+      if (verifySignatures) {
+        ManifestVerifier.Builder()
+          .addEd25519(
+            "key0",
+            "c960a123f76e0312cc05d9e358377b82d1e655d9569e9cff5f4834b1c4f228ce".decodeHex(),
+          )
+          .build()
+      } else {
+        ManifestVerifier.NO_SIGNATURE_CHECKS
+      }
+
+    val loader =
+      ZiplineLoader(
+          dispatcher = dispatcher,
+          manifestVerifier = manifestVerifier,
+          httpClient = httpClient,
+        )
+        .withEmbedded(
+          embeddedFileSystem = FileSystem.SYSTEM,
+          embeddedDir = embeddedDir.absolutePath.toPath(),
+        )
+
+    val freshnessChecker = if (verifySignatures) DefaultFreshnessCheckerNotFresh else AlwaysFresh
+    val effectiveManifestUrl =
+      if (!verifySignatures && embeddedManifestFile.exists()) {
+        embeddedManifestFile.toURI().toString()
+      } else {
+        manifestUrl
+      }
+
+    return withContext(dispatcher) {
+      when (
+        val result =
+          loader.loadOnce(
+            applicationName = "zipline-parser",
+            freshnessChecker = freshnessChecker,
+            manifestUrl = effectiveManifestUrl,
+            serializersModule = ParserSerializersModule,
+          )
+      ) {
+        is LoadResult.Success -> {
+          loadedZipline = result.zipline
+          DispatcherConfinedLobstersParserService(result.zipline.take("LobstersParserService"))
+        }
+        is LoadResult.Failure -> throw result.exception
+      }
+    }
+  }
+
+  internal fun <T> onZiplineThread(block: () -> T): T {
+    return if (Thread.currentThread() === ziplineThread.get()) {
+      block()
+    } else {
+      runBlocking(dispatcher) { block() }
+    }
+  }
+
+  private inner class DispatcherConfinedLobstersParserService(
+    private val delegate: LobstersParserService
+  ) : LobstersParserService {
+    override fun parsePostsPage(html: String) = onZiplineThread { delegate.parsePostsPage(html) }
+
+    override fun parsePostDetails(html: String) = onZiplineThread {
+      delegate.parsePostDetails(html)
+    }
+
+    override fun parseUser(html: String) = onZiplineThread { delegate.parseUser(html) }
+
+    override fun parseTagsPage(html: String) = onZiplineThread { delegate.parseTagsPage(html) }
+
+    override fun parseSearchResults(html: String) = onZiplineThread {
+      delegate.parseSearchResults(html)
+    }
+
+    override fun parseCsrfToken(html: String) = onZiplineThread { delegate.parseCsrfToken(html) }
+
+    override fun parseReplyForm(html: String) = onZiplineThread { delegate.parseReplyForm(html) }
+
+    override fun close() = onZiplineThread { delegate.close() }
+  }
+
+  private fun extractEmbeddedArtifacts() {
+    val assets = context.assets
+    val ziplineAssets = assets.list("zipline").orEmpty()
+    if (ziplineAssets.isEmpty()) return
+
+    val embeddedDir = File(context.codeCacheDir, "zipline-embedded/zipline")
+    embeddedDir.mkdirs()
+
+    for (assetName in ziplineAssets) {
+      assets.open("zipline/$assetName").use { input ->
+        File(embeddedDir, assetName).outputStream().use { output ->
+          input.copyTo(output)
+        }
+      }
+    }
+  }
+
+  private companion object {
+    private const val ZIPLINE_THREAD_STACK_SIZE_BYTES = 8L * 1024L * 1024L
+  }
+
+  /** Always prioritize the embedded files over the remote copy. */
+  private object AlwaysFresh : FreshnessChecker {
+    override fun isFresh(
+      manifest: ZiplineManifest,
+      freshAtEpochMs: Long,
+    ): Boolean {
+      return true
+    }
+  }
+}

api/build.gradle.kts

@@ -11,6 +11,7 @@ plugins {
   alias(libs.plugins.kotlin.serialization)
   alias(libs.plugins.dependencyAnalysis)
   alias(libs.plugins.metro)
+  alias(libs.plugins.zipline)
 }
 
 dependencies {
@@ -19,11 +20,10 @@ dependencies {
   api(libs.okhttp.core)
   api(libs.retrofit)
   api(projects.model)
+  api(projects.ziplineParser)
 
   implementation(libs.eithernet.integration.retrofit)
-  implementation(libs.ksoup)
-  implementation(libs.kspoon)
-  implementation(libs.retrofit.kotlinxSerializationConverter)
+  implementation(libs.zipline.loader)
 
   testImplementation(libs.eithernet.test.fixtures)
   testImplementation(libs.kotlin.reflect)

api/src/main/kotlin/dev/msfjarvis/claw/api/AuthenticatedLobstersApi.kt

@@ -9,6 +9,7 @@ package dev.msfjarvis.claw.api
 import com.slack.eithernet.ApiResult
 import com.slack.eithernet.ApiResult.Failure
 import com.slack.eithernet.ApiResult.Success
+import dev.msfjarvis.claw.model.ReplyForm
 import dev.zacsweers.metro.Inject
 import java.io.IOException
 import okhttp3.MultipartBody

api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersApi.kt

@@ -7,14 +7,12 @@
 package dev.msfjarvis.claw.api
 
 import com.slack.eithernet.ApiResult
-import com.slack.eithernet.ApiResult.Failure
-import com.slack.eithernet.ApiResult.Success
-import dev.burnoo.kspoon.annotation.Selector
+import dev.msfjarvis.claw.model.CSRFToken
 import dev.msfjarvis.claw.model.LobstersPost
 import dev.msfjarvis.claw.model.LobstersPostDetails
+import dev.msfjarvis.claw.model.ReplyForm
 import dev.msfjarvis.claw.model.Tag
 import dev.msfjarvis.claw.model.User
-import kotlinx.serialization.Serializable
 import okhttp3.MultipartBody
 import retrofit2.http.GET
 import retrofit2.http.Header
@@ -23,27 +21,14 @@ import retrofit2.http.POST
 import retrofit2.http.Part
 import retrofit2.http.Path
 
-@Serializable class PostsPage(@Selector("li.story") val posts: List<LobstersPost> = emptyList())
-
-@Serializable class TagsPage(@Selector("ol.category_tags > li") val tags: List<Tag> = emptyList())
-
-fun ApiResult<PostsPage, Unit>.toPostsResult(): ApiResult<List<LobstersPost>, Unit> =
-  when (this) {
-    is Success -> ApiResult.success(value.posts)
-    is Failure.ApiFailure -> ApiResult.apiFailure(error)
-    is Failure.HttpFailure -> ApiResult.httpFailure(code, error)
-    is Failure.NetworkFailure -> ApiResult.networkFailure(error)
-    is Failure.UnknownFailure -> ApiResult.unknownFailure(error)
-  }
-
 /** Simple interface defining an API for lobste.rs */
 interface LobstersApi {
 
   @GET("page/{page}")
-  suspend fun getHottestPosts(@Path("page") page: Int): ApiResult<PostsPage, Unit>
+  suspend fun getHottestPosts(@Path("page") page: Int): ApiResult<List<LobstersPost>, Unit>
 
   @GET("newest/page/{page}")
-  suspend fun getNewestPosts(@Path("page") page: Int): ApiResult<PostsPage, Unit>
+  suspend fun getNewestPosts(@Path("page") page: Int): ApiResult<List<LobstersPost>, Unit>
 
   @GET("s/{postId}")
   suspend fun getPostDetails(@Path("postId") postId: String): ApiResult<LobstersPostDetails, Unit>
@@ -52,7 +37,7 @@ interface LobstersApi {
 
   @GET("/") suspend fun getCSRFToken(): ApiResult<CSRFToken, Unit>
 
-  @GET("tags") suspend fun getTags(): ApiResult<TagsPage, Unit>
+  @GET("tags") suspend fun getTags(): ApiResult<List<Tag>, Unit>
 
   @Multipart
   @POST("comments/{commentId}/upvote")

api/src/main/kotlin/dev/msfjarvis/claw/api/LobstersParserClient.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api
+
+import dev.msfjarvis.claw.parser.LobstersParserService
+
+interface LobstersParserClient {
+  suspend fun service(): LobstersParserService
+}

api/src/main/kotlin/dev/msfjarvis/claw/api/converters/CSRFTokenConverter.kt

@@ -1,37 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.api.converters
-
-import com.fleeksoft.ksoup.Ksoup
-import dev.msfjarvis.claw.api.CSRFToken
-import dev.msfjarvis.claw.api.LobstersApi
-import java.lang.reflect.Type
-import okhttp3.ResponseBody
-import retrofit2.Converter
-import retrofit2.Retrofit
-
-object CSRFTokenConverter : Converter<ResponseBody, CSRFToken> {
-  override fun convert(value: ResponseBody): CSRFToken {
-    val token =
-      Ksoup.parse(value.string(), baseUri = LobstersApi.BASE_URL)
-        .select("meta[name=\"csrf-token\"]")
-        .firstOrNull()
-        ?.attr("content")
-        .orEmpty()
-    return CSRFToken(token)
-  }
-
-  object Factory : Converter.Factory() {
-    override fun responseBodyConverter(
-      type: Type,
-      annotations: Array<out Annotation>,
-      retrofit: Retrofit,
-    ): Converter<ResponseBody, *>? {
-      return if (type == CSRFToken::class.java) CSRFTokenConverter else null
-    }
-  }
-}

api/src/main/kotlin/dev/msfjarvis/claw/api/converters/ParserModelMappings.kt

@@ -0,0 +1,88 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api.converters
+
+import dev.msfjarvis.claw.model.CSRFToken
+import dev.msfjarvis.claw.model.Comment
+import dev.msfjarvis.claw.model.LobstersPost
+import dev.msfjarvis.claw.model.LobstersPostDetails
+import dev.msfjarvis.claw.model.ReplyForm
+import dev.msfjarvis.claw.model.Tag
+import dev.msfjarvis.claw.model.User
+import kotlin.time.Instant
+
+internal fun dev.msfjarvis.claw.parser.model.CSRFToken.toModel(): CSRFToken = CSRFToken(value)
+
+internal fun dev.msfjarvis.claw.parser.model.LobstersPost.toModel(): LobstersPost =
+  LobstersPost(
+    shortId = shortId,
+    createdAt = createdAt,
+    title = title,
+    url = url,
+    description = description,
+    commentCount = commentCount,
+    commentsUrl = commentsUrl,
+    submitter = submitter,
+    userIsAuthor = userIsAuthor,
+    tags = tags,
+  )
+
+internal fun dev.msfjarvis.claw.parser.model.LobstersPostDetails.toModel(): LobstersPostDetails =
+  LobstersPostDetails(
+    shortId = shortId,
+    createdAt = createdAt,
+    title = title,
+    url = url,
+    description = description,
+    commentCount = commentCount,
+    commentsUrl = commentsUrl,
+    submitter = submitter,
+    tags = tags,
+    comments = comments.map { it.toModel() },
+    userIsAuthor = userIsAuthor,
+  )
+
+internal fun dev.msfjarvis.claw.parser.model.Comment.toModel(): Comment =
+  Comment(
+    shortId = shortId,
+    comment = comment,
+    url = url,
+    score = score,
+    timestamp = Instant.fromEpochSeconds(timestamp),
+    edited = edited,
+    parentComment = parentComment,
+    user = user,
+    isUpvoted = isUpvoted,
+  )
+
+internal fun dev.msfjarvis.claw.parser.model.ReplyForm.toModel(): ReplyForm =
+  ReplyForm(
+    authenticityToken = authenticityToken,
+    storyId = storyId,
+    method = method,
+    parentCommentShortId = parentCommentShortId,
+  )
+
+internal fun dev.msfjarvis.claw.parser.model.Tag.toModel(): Tag =
+  Tag(
+    tag = tag,
+    description = description,
+    privileged = privileged,
+    active = active,
+    category = category,
+    isMedia = isMedia,
+    hotnessMod = hotnessMod,
+  )
+
+internal fun dev.msfjarvis.claw.parser.model.User.toModel(): User =
+  User(
+    username = username,
+    about = about,
+    invitedBy = invitedBy,
+    avatarUrl = avatarUrl,
+    createdAt = createdAt,
+  )

api/src/main/kotlin/dev/msfjarvis/claw/api/converters/ReplyFormConverter.kt

@@ -1,40 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.api.converters
-
-import com.fleeksoft.ksoup.Ksoup
-import dev.msfjarvis.claw.api.LobstersApi
-import dev.msfjarvis.claw.api.ReplyForm
-import java.lang.reflect.Type
-import okhttp3.ResponseBody
-import retrofit2.Converter
-import retrofit2.Retrofit
-
-object ReplyFormConverter : Converter<ResponseBody, ReplyForm> {
-  override fun convert(value: ResponseBody): ReplyForm {
-    val document = Ksoup.parse(value.string(), baseUri = LobstersApi.BASE_URL)
-    fun inputValue(name: String): String =
-      document.select("input[name=\"$name\"]").firstOrNull()?.attr("value").orEmpty()
-
-    return ReplyForm(
-      authenticityToken = inputValue("authenticity_token"),
-      storyId = inputValue("story_id"),
-      method = inputValue("_method"),
-      parentCommentShortId = inputValue("parent_comment_short_id"),
-    )
-  }
-
-  object Factory : Converter.Factory() {
-    override fun responseBodyConverter(
-      type: Type,
-      annotations: Array<out Annotation>,
-      retrofit: Retrofit,
-    ): Converter<ResponseBody, *>? {
-      return if (type == ReplyForm::class.java) ReplyFormConverter else null
-    }
-  }
-}

api/src/main/kotlin/dev/msfjarvis/claw/api/converters/SearchConverter.kt

@@ -1,72 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.api.converters
-
-import com.fleeksoft.ksoup.Ksoup
-import com.fleeksoft.ksoup.nodes.Element
-import com.fleeksoft.ksoup.select.Elements
-import dev.msfjarvis.claw.api.LobstersApi
-import dev.msfjarvis.claw.model.LobstersPost
-import java.lang.reflect.Type
-import okhttp3.ResponseBody
-import retrofit2.Converter
-import retrofit2.Retrofit
-
-object SearchConverter : Converter<ResponseBody, List<LobstersPost>> {
-  override fun convert(value: ResponseBody): List<LobstersPost> {
-    return Ksoup.parse(value.string(), baseUri = LobstersApi.BASE_URL)
-      .select("div.story_liner.h-entry")
-      .map(::parsePost)
-  }
-
-  private fun parsePost(elem: Element): LobstersPost {
-    val parent = elem.parent() ?: error("$elem must have a parent")
-    val shortId = parent.attr("data-shortid")
-    val titleElement = elem.select("span.link.h-cite > a")
-    val title = titleElement.text()
-    val url = titleElement.attr("href")
-    val tags = elem.select("span.tags > a").map(Element::text)
-    val (commentCount, commentsUrl) = getCommentsData(elem.select("span.comments_label"))
-    val submitter =
-      elem.select("div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])").text()
-    val userIsAuthor =
-      (elem.select("div.byline > span").first()?.text() ?: "").contains(
-        "authored",
-        ignoreCase = true,
-      )
-    return LobstersPost(
-      shortId = shortId,
-      title = title,
-      url = url,
-      commentCount = commentCount,
-      commentsUrl = commentsUrl,
-      tags = tags,
-      submitter = submitter,
-      // The value of these fields is irrelevant for our use case
-      createdAt = "",
-      description = "",
-      userIsAuthor = userIsAuthor,
-    )
-  }
-
-  private fun getCommentsData(elem: Elements): Pair<Int, String> {
-    val linkElement = elem.select("a")
-    val countString = linkElement.text().trimStart().substringBefore(" ")
-    val commentsUrl = LobstersApi.BASE_URL + linkElement.attr("href")
-    return (countString.toIntOrNull() ?: 0) to commentsUrl
-  }
-
-  object Factory : Converter.Factory() {
-    override fun responseBodyConverter(
-      type: Type,
-      annotations: Array<out Annotation>,
-      retrofit: Retrofit,
-    ): Converter<ResponseBody, List<LobstersPost>> {
-      return SearchConverter
-    }
-  }
-}

api/src/main/kotlin/dev/msfjarvis/claw/api/converters/ZiplineHtmlConverterFactory.kt

@@ -0,0 +1,78 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.api.converters
+
+import dev.msfjarvis.claw.api.LobstersParserClient
+import dev.msfjarvis.claw.model.CSRFToken
+import dev.msfjarvis.claw.model.LobstersPost
+import dev.msfjarvis.claw.model.LobstersPostDetails
+import dev.msfjarvis.claw.model.ReplyForm
+import dev.msfjarvis.claw.model.Tag
+import dev.msfjarvis.claw.model.User
+import java.lang.reflect.ParameterizedType
+import java.lang.reflect.Type
+import java.lang.reflect.WildcardType
+import kotlinx.coroutines.runBlocking
+import okhttp3.ResponseBody
+import retrofit2.Converter
+import retrofit2.Retrofit
+import retrofit2.http.GET
+
+class ZiplineHtmlConverterFactory(private val parserClient: LobstersParserClient) :
+  Converter.Factory() {
+  override fun responseBodyConverter(
+    type: Type,
+    annotations: Array<Annotation>,
+    retrofit: Retrofit,
+  ): Converter<ResponseBody, *>? {
+    val normalizedType = type.unwrapWildcard()
+    val getPath = annotations.filterIsInstance<GET>().singleOrNull()?.value
+    return when {
+      normalizedType.isListOf(LobstersPost::class.java) &&
+        getPath?.startsWith("/search") == true -> {
+        HtmlConverter { html ->
+          parserClient.service().parseSearchResults(html).map { it.toModel() }
+        }
+      }
+      normalizedType.isListOf(LobstersPost::class.java) -> {
+        HtmlConverter { html -> parserClient.service().parsePostsPage(html).map { it.toModel() } }
+      }
+      normalizedType.isListOf(Tag::class.java) -> {
+        HtmlConverter { html -> parserClient.service().parseTagsPage(html).map { it.toModel() } }
+      }
+      normalizedType == CSRFToken::class.java ->
+        HtmlConverter { html -> parserClient.service().parseCsrfToken(html).toModel() }
+      normalizedType == LobstersPostDetails::class.java ->
+        HtmlConverter { html -> parserClient.service().parsePostDetails(html).toModel() }
+      normalizedType == ReplyForm::class.java ->
+        HtmlConverter { html -> parserClient.service().parseReplyForm(html).toModel() }
+      normalizedType == User::class.java ->
+        HtmlConverter { html -> parserClient.service().parseUser(html).toModel() }
+      else -> null
+    }
+  }
+
+  private fun Type.unwrapWildcard(): Type {
+    return if (this is WildcardType) this.upperBounds.singleOrNull() ?: this else this
+  }
+
+  private fun Type.isListOf(raw: Class<*>): Boolean {
+    val normalized = unwrapWildcard()
+    if (normalized !is ParameterizedType) return false
+    val parameter = getParameterUpperBound(0, normalized).unwrapWildcard()
+    return getRawType(normalized) == List::class.java && parameter == raw
+  }
+
+  private class HtmlConverter<T>(private val parse: suspend (String) -> T) :
+    Converter<ResponseBody, T> {
+    override fun convert(value: ResponseBody): T {
+      value.use { body ->
+        return runBlocking { parse(body.string()) }
+      }
+    }
+  }
+}

api/src/main/kotlin/dev/msfjarvis/claw/api/injection/RetrofitModule.kt

@@ -6,17 +6,14 @@
  */
 package dev.msfjarvis.claw.api.injection
 
-import com.fleeksoft.ksoup.Ksoup
 import com.slack.eithernet.integration.retrofit.ApiResultCallAdapterFactory
 import com.slack.eithernet.integration.retrofit.ApiResultConverterFactory
-import dev.burnoo.kspoon.Kspoon
 import dev.msfjarvis.claw.api.AuthenticatedLobstersApi
 import dev.msfjarvis.claw.api.LobstersApi
+import dev.msfjarvis.claw.api.LobstersParserClient
 import dev.msfjarvis.claw.api.LobstersSearchApi
-import dev.msfjarvis.claw.api.converters.CSRFTokenConverter
-import dev.msfjarvis.claw.api.converters.ReplyFormConverter
-import dev.msfjarvis.claw.api.converters.SearchConverter
 import dev.msfjarvis.claw.api.converters.UnitConverter
+import dev.msfjarvis.claw.api.converters.ZiplineHtmlConverterFactory
 import dev.zacsweers.metro.AppScope
 import dev.zacsweers.metro.BindingContainer
 import dev.zacsweers.metro.ContributesTo
@@ -26,17 +23,15 @@ import dev.zacsweers.metro.Named
 import dev.zacsweers.metro.Provides
 import dev.zacsweers.metro.Qualifier
 import dev.zacsweers.metro.SingleIn
-import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
 import retrofit2.CallAdapter
 import retrofit2.Converter
 import retrofit2.Retrofit
-import retrofit2.converter.kotlinx.serialization.asConverterFactory
 import retrofit2.create
 
 /**
- * Ideally the multibindings used here would only use [dagger.multibindings.IntoSet], but its lack
- * of ordering guarantees means that we roll a die on each app launch that [Converter]s and
+ * Ideally the multibindings used here would only use [dev.zacsweers.metro.IntoSet], but its lack of
+ * ordering guarantees means that we roll a die on each app launch that [Converter]s and
  * [CallAdapter]s are in the correct order to be able to deserialize responses. Thus, the module
  * uses [IntoMap] with [IntKey]s to fake the presence of a fixed order by sorting on the key of the
  * injected [Map]s when injecting them into [Retrofit].
@@ -74,7 +69,6 @@ object RetrofitModule {
       .baseUrl(baseUrl)
       .apply { converterFactories.forEach(this::addConverterFactory) }
       .apply { callAdapterFactories.toSortedMap().values.forEach(this::addCallAdapterFactory) }
-      .addConverterFactory(SearchConverter.Factory)
       .build()
   }
 
@@ -99,29 +93,14 @@ object RetrofitModule {
   @Provides
   @IntKey(1)
   @IntoMap
-  fun provideCSRFTokenConverter(): Converter.Factory = CSRFTokenConverter.Factory
+  fun provideZiplineHtmlConverter(parserClient: LobstersParserClient): Converter.Factory =
+    ZiplineHtmlConverterFactory(parserClient)
 
   @Provides
   @IntKey(2)
   @IntoMap
-  fun provideReplyFormConverter(): Converter.Factory = ReplyFormConverter.Factory
-
-  @Provides
-  @IntKey(3)
-  @IntoMap
   fun provideUnitConverter(): Converter.Factory = UnitConverter.Factory
 
-  @Provides
-  @IntKey(4)
-  @IntoMap
-  fun provideKspoonConverter(): Converter.Factory =
-    Kspoon {
-        parse = { html -> Ksoup.parse(html, baseUri = LobstersApi.BASE_URL) }
-        coerceInputValues = true
-      }
-      .toFormat()
-      .asConverterFactory("text/html".toMediaType())
-
   @Provides
   @IntKey(0)
   @IntoMap
@@ -129,8 +108,11 @@ object RetrofitModule {
 
   @Provides
   @SearchApi
-  fun provideConverters(): List<Converter.Factory> =
-    listOf(ApiResultConverterFactory, SearchConverter.Factory)
+  fun provideConverters(parserClient: LobstersParserClient): List<Converter.Factory> =
+    listOf(
+      ApiResultConverterFactory,
+      ZiplineHtmlConverterFactory(parserClient),
+    )
 
   @Provides @Named("LobstersURL") fun provideLobstersUrl(): String = LobstersApi.BASE_URL
 }

api/src/test/kotlin/dev/msfjarvis/claw/api/ApiTest.kt

@@ -6,44 +6,45 @@
  */
 package dev.msfjarvis.claw.api
 
-import com.fleeksoft.ksoup.Ksoup
 import com.google.common.truth.Truth.assertThat
 import com.slack.eithernet.ApiResult.Success
-import com.slack.eithernet.test.newEitherNetController
-import dev.burnoo.kspoon.Kspoon
+import dev.msfjarvis.claw.model.CSRFToken
+import dev.msfjarvis.claw.model.LobstersPost
 import dev.msfjarvis.claw.model.LobstersPostDetails
+import dev.msfjarvis.claw.model.Tag
 import dev.msfjarvis.claw.model.User
+import dev.msfjarvis.claw.parser.LobstersParserServiceImpl
 import dev.msfjarvis.claw.util.TestUtils.assertIs
 import kotlin.time.Instant
 import kotlinx.coroutines.test.runTest
 import org.junit.jupiter.api.Test
 
 class ApiTest {
-  private val wrapper = ApiWrapper(newEitherNetController())
+  private val wrapper = ApiWrapper()
   private val api
     get() = wrapper.api
 
   @Test
   fun `api gets correct number of items`() = runTest {
     val posts = api.getHottestPosts(1)
-    assertIs<Success<PostsPage>>(posts)
-    assertThat(posts.value.posts).hasSize(25)
+    assertIs<Success<List<LobstersPost>>>(posts)
+    assertThat(posts.value).hasSize(25)
   }
 
   @Test
   fun `posts with no urls`() = runTest {
     val posts = api.getHottestPosts(1)
-    assertIs<Success<PostsPage>>(posts)
-    val commentsOnlyPosts = posts.value.posts.asSequence().filter { it.url.isEmpty() }.toSet()
+    assertIs<Success<List<LobstersPost>>>(posts)
+    val commentsOnlyPosts = posts.value.asSequence().filter { it.url.isEmpty() }.toSet()
     assertThat(commentsOnlyPosts).hasSize(0)
   }
 
   @Test
   fun `api parses hottest HTML fixture fields`() = runTest {
     val posts = api.getHottestPosts(1)
-    assertIs<Success<PostsPage>>(posts)
+    assertIs<Success<List<LobstersPost>>>(posts)
 
-    val firstPost = posts.value.posts[0]
+    val firstPost = posts.value[0]
     assertThat(firstPost.shortId).isEqualTo("jp3nva")
     assertThat(firstPost.title).isEqualTo("You probably don't need Yocto, and that's fine")
     assertThat(firstPost.submitter).isEqualTo("rw-rw-rw-")
@@ -55,7 +56,7 @@ class ApiTest {
     assertThat(firstPost.createdAt).isEqualTo("2026-05-29T09:08:12Z")
     Instant.parse(firstPost.createdAt)
 
-    val secondPost = posts.value.posts[1]
+    val secondPost = posts.value[1]
     assertThat(secondPost.shortId).isEqualTo("lc26ar")
     assertThat(secondPost.title).isEqualTo("SQLite Does Not Accept Agentic Code")
     assertThat(secondPost.submitter).isEqualTo("hoistbypetard")
@@ -65,7 +66,7 @@ class ApiTest {
     assertThat(secondPost.tags).containsExactly("vibecoding")
     assertThat(secondPost.userIsAuthor).isFalse()
 
-    val noCommentsPost = posts.value.posts.first { it.shortId == "1fkt8w" }
+    val noCommentsPost = posts.value.first { it.shortId == "1fkt8w" }
     assertThat(noCommentsPost.title).isEqualTo("Patching my guitar amp's firmware")
     assertThat(noCommentsPost.submitter).isEqualTo("mcf")
     assertThat(noCommentsPost.commentCount).isEqualTo(0)
@@ -78,8 +79,8 @@ class ApiTest {
   @Test
   fun `api gets newest posts`() = runTest {
     val posts = api.getNewestPosts(1)
-    assertIs<Success<PostsPage>>(posts)
-    assertThat(posts.value.posts).hasSize(25)
+    assertIs<Success<List<LobstersPost>>>(posts)
+    assertThat(posts.value).hasSize(25)
   }
 
   @Test
@@ -97,13 +98,10 @@ class ApiTest {
 
   @Test
   fun `comments without visible upvoter count have one point from the author`() {
-    val kspoon = Kspoon {
-      parse = { html -> Ksoup.parse(html, baseUri = LobstersApi.BASE_URL) }
-      coerceInputValues = true
-    }
+    val parser = LobstersParserServiceImpl()
 
     val postDetails =
-      kspoon.parse<LobstersPostDetails>(
+      parser.parsePostDetails(
         """
         <ol class="stories">
           <li class="story" data-shortid="story1">
@@ -172,9 +170,9 @@ class ApiTest {
   @Test
   fun `retrieve tags`() = runTest {
     val tags = api.getTags()
-    assertIs<Success<TagsPage>>(tags)
-    assertThat(tags.value.tags).isNotEmpty()
-    val rubyTag = tags.value.tags.first { it.tag == "ruby" }
+    assertIs<Success<List<Tag>>>(tags)
+    assertThat(tags.value).isNotEmpty()
+    val rubyTag = tags.value.first { it.tag == "ruby" }
     assertThat(rubyTag.description).isEqualTo("Ruby programming")
     assertThat(rubyTag.privileged).isFalse()
     assertThat(rubyTag.active).isTrue()
@@ -182,10 +180,10 @@ class ApiTest {
     assertThat(rubyTag.isMedia).isFalse()
     assertThat(rubyTag.hotnessMod).isEqualTo(0.0)
 
-    val newsTag = tags.value.tags.first { it.tag == "news" }
+    val newsTag = tags.value.first { it.tag == "news" }
     assertThat(newsTag.active).isFalse()
 
-    val videoTag = tags.value.tags.first { it.tag == "video" }
+    val videoTag = tags.value.first { it.tag == "video" }
     assertThat(videoTag.isMedia).isTrue()
   }
 }

api/src/test/kotlin/dev/msfjarvis/claw/api/ApiWrapper.kt

@@ -6,79 +6,68 @@
  */
 package dev.msfjarvis.claw.api
 
-import com.fleeksoft.ksoup.Ksoup
-import com.slack.eithernet.ApiResult.Companion.success
-import com.slack.eithernet.test.EitherNetController
-import com.slack.eithernet.test.enqueue
-import dev.burnoo.kspoon.Kspoon
-import dev.msfjarvis.claw.api.converters.CSRFTokenConverter
-import dev.msfjarvis.claw.model.LobstersPostDetails
-import dev.msfjarvis.claw.model.User
+import com.slack.eithernet.integration.retrofit.ApiResultCallAdapterFactory
+import com.slack.eithernet.integration.retrofit.ApiResultConverterFactory
+import dev.msfjarvis.claw.api.converters.UnitConverter
+import dev.msfjarvis.claw.api.converters.ZiplineHtmlConverterFactory
+import dev.msfjarvis.claw.parser.LobstersParserService
+import dev.msfjarvis.claw.parser.LobstersParserServiceImpl
 import dev.msfjarvis.claw.util.TestUtils.getResource
+import okhttp3.Interceptor
 import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.Protocol
+import okhttp3.Response
 import okhttp3.ResponseBody.Companion.toResponseBody
+import retrofit2.Retrofit
+import retrofit2.create
 
-class ApiWrapper(controller: EitherNetController<LobstersApi>) {
-  private val kspoon = Kspoon {
-    parse = { html -> Ksoup.parse(html, baseUri = LobstersApi.BASE_URL) }
-    coerceInputValues = true
-  }
-  private val postsPage: PostsPage = kspoon.parse<PostsPage>(getResource("hottest_page.html"))
-  private val postDetails: LobstersPostDetails =
-    kspoon.parse(getResource("post_details_tdfoqh.html"))
-  val upvotedPostDetails: LobstersPostDetails =
-    kspoon.parse(getResource("post_details_upvoted.html"))
-  private val user: User = kspoon.parse(getResource("msfjarvis.html"))
-  private val tags: TagsPage = kspoon.parse(getResource("tags.html"))
+class ApiWrapper {
+  private val parser = LobstersParserServiceImpl()
+  val upvotedPostDetails = parser.parsePostDetails(getResource("post_details_upvoted.html"))
 
-  val api = controller.api
-  val authenticatedApi = AuthenticatedLobstersApi(api)
+  val api: LobstersApi
+  val authenticatedApi: AuthenticatedLobstersApi
 
   init {
-    controller.enqueue(LobstersApi::getHottestPosts) { success(postsPage) }
-    controller.enqueue(LobstersApi::getHottestPosts) { success(postsPage) }
-    controller.enqueue(LobstersApi::getHottestPosts) { success(postsPage) }
-    controller.enqueue(LobstersApi::getNewestPosts) { success(postsPage) }
-    controller.enqueue(LobstersApi::getPostDetails) { success(postDetails) }
-    controller.enqueue(LobstersApi::getUser) { success(user) }
-    controller.enqueue(LobstersApi::getTags) { success(tags) }
-    controller.enqueue(LobstersApi::getCSRFToken) {
-      success(
-        CSRFTokenConverter.convert(
-          getResource("csrf_page.html").toResponseBody("text/html".toMediaType())
-        )
-      )
+    val parserClient =
+      object : LobstersParserClient {
+        override suspend fun service(): LobstersParserService = LobstersParserServiceImpl()
+      }
+    val retrofit =
+      Retrofit.Builder()
+        .baseUrl("https://lobste.rs/")
+        .client(OkHttpClient.Builder().addInterceptor(FixtureInterceptor()).build())
+        .addConverterFactory(ApiResultConverterFactory)
+        .addConverterFactory(ZiplineHtmlConverterFactory(parserClient))
+        .addConverterFactory(UnitConverter.Factory)
+        .addCallAdapterFactory(ApiResultCallAdapterFactory)
+        .build()
+    api = retrofit.create()
+    authenticatedApi = AuthenticatedLobstersApi(api)
+  }
+
+  private class FixtureInterceptor : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+      val request = chain.request()
+      val body =
+        when (request.url.encodedPath) {
+          "/page/1" -> getResource("hottest_page.html")
+          "/newest/page/1" -> getResource("hottest_page.html")
+          "/s/tdfoqh" -> getResource("post_details_tdfoqh.html")
+          "/~msfjarvis" -> getResource("msfjarvis.html")
+          "/" -> getResource("csrf_page.html")
+          "/tags" -> getResource("tags.html")
+          "/comments/edtrox/reply" -> getResource("reply_form.html")
+          else -> ""
+        }
+      return Response.Builder()
+        .request(request)
+        .protocol(Protocol.HTTP_1_1)
+        .code(200)
+        .message("OK")
+        .body(body.toByteArray().toResponseBody("text/html; charset=utf-8".toMediaType()))
+        .build()
     }
-    controller.enqueue(LobstersApi::getCSRFToken) {
-      success(
-        CSRFTokenConverter.convert(
-          getResource("csrf_page.html").toResponseBody("text/html".toMediaType())
-        )
-      )
-    }
-    controller.enqueue(LobstersApi::getCSRFToken) {
-      success(
-        CSRFTokenConverter.convert(
-          getResource("csrf_page.html").toResponseBody("text/html".toMediaType())
-        )
-      )
-    }
-    controller.enqueue(LobstersApi::getCSRFToken) {
-      success(
-        CSRFTokenConverter.convert(
-          getResource("csrf_page.html").toResponseBody("text/html".toMediaType())
-        )
-      )
-    }
-    controller.enqueue(LobstersApi::upvoteComment) { success(Unit) }
-    controller.enqueue(LobstersApi::unvoteComment) { success(Unit) }
-    controller.enqueue(LobstersApi::getReplyForm) {
-      success(
-        dev.msfjarvis.claw.api.converters.ReplyFormConverter.convert(
-          getResource("reply_form.html").toResponseBody("text/html".toMediaType())
-        )
-      )
-    }
-    controller.enqueue(LobstersApi::postReply) { success(Unit) }
   }
 }

api/src/test/kotlin/dev/msfjarvis/claw/api/AuthenticatedLobstersApiHeaderTest.kt

@@ -8,7 +8,11 @@ package dev.msfjarvis.claw.api
 
 import com.google.common.truth.Truth.assertThat
 import com.slack.eithernet.ApiResult
+import dev.msfjarvis.claw.model.CSRFToken
+import dev.msfjarvis.claw.model.LobstersPost
 import dev.msfjarvis.claw.model.LobstersPostDetails
+import dev.msfjarvis.claw.model.ReplyForm
+import dev.msfjarvis.claw.model.Tag
 import dev.msfjarvis.claw.model.User
 import kotlinx.coroutines.test.runTest
 import okhttp3.MultipartBody
@@ -40,9 +44,11 @@ private class RecordingLobstersApi : LobstersApi {
   var postReplyOrigin: String? = null
   var postReplyAccept: String? = null
 
-  override suspend fun getHottestPosts(page: Int): ApiResult<PostsPage, Unit> = error("unused")
+  override suspend fun getHottestPosts(page: Int): ApiResult<List<LobstersPost>, Unit> =
+    error("unused")
 
-  override suspend fun getNewestPosts(page: Int): ApiResult<PostsPage, Unit> = error("unused")
+  override suspend fun getNewestPosts(page: Int): ApiResult<List<LobstersPost>, Unit> =
+    error("unused")
 
   override suspend fun getPostDetails(postId: String): ApiResult<LobstersPostDetails, Unit> =
     error("unused")
@@ -52,7 +58,7 @@ private class RecordingLobstersApi : LobstersApi {
   override suspend fun getCSRFToken(): ApiResult<CSRFToken, Unit> =
     ApiResult.success(CSRFToken("csrf-token"))
 
-  override suspend fun getTags(): ApiResult<TagsPage, Unit> = error("unused")
+  override suspend fun getTags(): ApiResult<List<Tag>, Unit> = error("unused")
 
   override suspend fun upvoteComment(
     commentId: String,

api/src/test/kotlin/dev/msfjarvis/claw/api/AuthenticatedLobstersApiTest.kt

@@ -7,13 +7,12 @@
 package dev.msfjarvis.claw.api
 
 import com.slack.eithernet.ApiResult.Success
-import com.slack.eithernet.test.newEitherNetController
 import dev.msfjarvis.claw.util.TestUtils.assertIs
 import kotlinx.coroutines.test.runTest
 import org.junit.jupiter.api.Test
 
 class AuthenticatedLobstersApiTest {
-  private val wrapper = ApiWrapper(newEitherNetController())
+  private val wrapper = ApiWrapper()
   private val authenticatedApi = wrapper.authenticatedApi
 
   @Test

api/src/test/kotlin/dev/msfjarvis/claw/api/SearchApiWrapper.kt

@@ -6,24 +6,55 @@
  */
 package dev.msfjarvis.claw.api
 
-import com.slack.eithernet.ApiResult.Companion.success
+import com.slack.eithernet.integration.retrofit.ApiResultCallAdapterFactory
+import com.slack.eithernet.integration.retrofit.ApiResultConverterFactory
 import com.slack.eithernet.test.EitherNetController
-import com.slack.eithernet.test.enqueue
-import dev.msfjarvis.claw.api.converters.SearchConverter
+import dev.msfjarvis.claw.api.converters.ZiplineHtmlConverterFactory
+import dev.msfjarvis.claw.parser.LobstersParserService
+import dev.msfjarvis.claw.parser.LobstersParserServiceImpl
 import dev.msfjarvis.claw.util.TestUtils.getResource
+import okhttp3.Interceptor
 import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.Protocol
+import okhttp3.Response
 import okhttp3.ResponseBody.Companion.toResponseBody
+import retrofit2.Retrofit
+import retrofit2.create
 
 class SearchApiWrapper(controller: EitherNetController<LobstersSearchApi>) {
-  val api = controller.api
+  val api: LobstersSearchApi
 
   init {
-    controller.enqueue(LobstersSearchApi::searchPosts) {
-      success(
-        SearchConverter.convert(
-          getResource("search_chatgpt_page.html").toResponseBody("text/html".toMediaType())
+    val parserClient =
+      object : LobstersParserClient {
+        override suspend fun service(): LobstersParserService = LobstersParserServiceImpl()
+      }
+    val retrofit =
+      Retrofit.Builder()
+        .baseUrl("https://lobste.rs/")
+        .client(OkHttpClient.Builder().addInterceptor(FixtureInterceptor()).build())
+        .addConverterFactory(ApiResultConverterFactory)
+        .addConverterFactory(ZiplineHtmlConverterFactory(parserClient))
+        .addCallAdapterFactory(ApiResultCallAdapterFactory)
+        .build()
+    api = retrofit.create()
+  }
+
+  private class FixtureInterceptor : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+      val request = chain.request()
+      return Response.Builder()
+        .request(request)
+        .protocol(Protocol.HTTP_1_1)
+        .code(200)
+        .message("OK")
+        .body(
+          getResource("search_chatgpt_page.html")
+            .toByteArray()
+            .toResponseBody("text/html; charset=utf-8".toMediaType())
         )
-      )
+        .build()
     }
   }
 }

api/src/test/kotlin/dev/msfjarvis/claw/api/converters/CSRFTokenConverterTest.kt

@@ -1,38 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.api.converters
-
-import com.google.common.truth.Truth.assertThat
-import dev.msfjarvis.claw.util.TestUtils.getResource
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.ResponseBody.Companion.toResponseBody
-import org.junit.jupiter.api.Test
-
-class CSRFTokenConverterTest {
-  @Test
-  fun `converter extracts CSRF token`() {
-    val token =
-      CSRFTokenConverter.convert(
-        getResource("csrf_page.html").toResponseBody("text/html".toMediaType())
-      )
-
-    assertThat(token.value)
-      .isEqualTo(
-        "dvJ8r_CkOImcHQ5ZLUWlJeQVoPEPQ3rK85DNgiZJcehafqwYP8jESW8AhMf0uQGLqqLbsarYiISCghnDaUd6wA"
-      )
-  }
-
-  @Test
-  fun `converter returns empty token when CSRF token is absent`() {
-    val token =
-      CSRFTokenConverter.convert(
-        "<html><head></head><body></body></html>".toResponseBody("text/html".toMediaType())
-      )
-
-    assertThat(token.value).isEmpty()
-  }
-}

api/src/test/kotlin/dev/msfjarvis/claw/api/converters/ReplyFormConverterTest.kt

@@ -1,41 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.api.converters
-
-import com.google.common.truth.Truth.assertThat
-import dev.msfjarvis.claw.util.TestUtils.getResource
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.ResponseBody.Companion.toResponseBody
-import org.junit.jupiter.api.Test
-
-class ReplyFormConverterTest {
-  @Test
-  fun `converter extracts reply form hidden fields`() {
-    val form =
-      ReplyFormConverter.convert(
-        getResource("reply_form.html").toResponseBody("text/html".toMediaType())
-      )
-
-    assertThat(form.authenticityToken)
-      .isEqualTo(
-        "AI0414bnzi152-mE0JTWEtwq5B0ZhALBW1W7rGiG5zR-sFaJjWzdARXFM7w_DbPQqWNjzh9bufWZbXG39v5T6g"
-      )
-    assertThat(form.storyId).isEqualTo("znlkib")
-    assertThat(form.method).isEqualTo("post")
-    assertThat(form.parentCommentShortId).isEqualTo("edtrox")
-  }
-
-  @Test
-  fun `converter returns empty strings when reply form fields are absent`() {
-    val form = ReplyFormConverter.convert("<form></form>".toResponseBody("text/html".toMediaType()))
-
-    assertThat(form.authenticityToken).isEmpty()
-    assertThat(form.storyId).isEmpty()
-    assertThat(form.method).isEmpty()
-    assertThat(form.parentCommentShortId).isEmpty()
-  }
-}

api/src/test/kotlin/dev/msfjarvis/claw/api/converters/SearchConverterTest.kt

@@ -1,54 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.api.converters
-
-import com.google.common.truth.Truth.assertThat
-import dev.msfjarvis.claw.model.LobstersPost
-import dev.msfjarvis.claw.util.TestUtils.getResource
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.ResponseBody.Companion.toResponseBody
-import org.junit.jupiter.api.Test
-
-class SearchConverterTest {
-  @Test
-  fun `converter parses search result HTML`() {
-    val posts =
-      SearchConverter.convert(
-        getResource("search_chatgpt_page.html").toResponseBody("text/html".toMediaType())
-      )
-
-    assertThat(posts).hasSize(20)
-    assertThat(posts)
-      .containsAtLeast(
-        LobstersPost(
-          shortId = "kgem4b",
-          createdAt = "",
-          title = "The social contract of writing",
-          url = "https://jola.dev/posts/the-social-contract-of-writing",
-          description = "",
-          commentCount = 38,
-          commentsUrl = "https://lobste.rs/s/kgem4b/social_contract_writing",
-          submitter = "joladev",
-          userIsAuthor = true,
-          tags = listOf("philosophy", "vibecoding"),
-        ),
-        LobstersPost(
-          shortId = "gydtkf",
-          createdAt = "",
-          title = "AI Resist List",
-          url = "https://airesistlist.org/",
-          description = "",
-          commentCount = 0,
-          commentsUrl = "https://lobste.rs/s/gydtkf/ai_resist_list",
-          submitter = "chobeat",
-          userIsAuthor = false,
-          tags = listOf("ai"),
-        ),
-      )
-      .inOrder()
-  }
-}

build-logic/src/main/kotlin/dev/msfjarvis/claw/gradle/SpotlessPlugin.kt

@@ -43,7 +43,7 @@ class SpotlessPlugin : Plugin<Project> {
         endWithNewline()
         licenseHeaderFile(
           project.file("spotless/license.xml"),
-          "<(adaptive-icon|appwidget-provider|data-extraction-rules|full-backup-content|manifest|vector|resources)",
+          "<(adaptive-icon|appwidget-provider|data-extraction-rules|full-backup-content|manifest|network-security-config|vector|resources)",
         )
       }
     }

common/src/main/kotlin/dev/msfjarvis/claw/common/comments/CommentEntry.kt

@@ -61,8 +61,10 @@ import dev.msfjarvis.claw.common.posts.Submitter
 import dev.msfjarvis.claw.common.posts.TagRow
 import dev.msfjarvis.claw.common.ui.NetworkImage
 import dev.msfjarvis.claw.common.ui.ThemedRichText
+import dev.msfjarvis.claw.model.Comment
 import dev.msfjarvis.claw.model.LinkMetadata
 import dev.msfjarvis.claw.model.UIPost
+import kotlin.time.Clock
 import kotlin.time.Instant
 import kotlinx.collections.immutable.toImmutableList
 
@@ -359,6 +361,25 @@ private fun displayScore(score: Int, initiallyUpvoted: Boolean, isUpvoted: Boole
   }
 }
 
+internal fun previewCommentNode(isUpvoted: Boolean = false) =
+  CommentNode(
+    comment =
+      Comment(
+        shortId = "preview-comment",
+        comment =
+          "<p>This is a preview comment with enough content to evaluate spacing, metadata, and future vote affordances.</p>",
+        score = 42,
+        timestamp = Clock.System.now(),
+        edited = false,
+        parentComment = null,
+        user = "Alice",
+        isUpvoted = isUpvoted,
+      ),
+    isPostAuthor = false,
+    isUnread = true,
+    indentLevel = 0,
+  )
+
 private fun buildCommentAgeString(timestamp: Instant, edited: Boolean): String {
   val now = System.currentTimeMillis()
   val relativeTime =

common/src/main/kotlin/dev/msfjarvis/claw/common/tags/TagFilterViewModel.kt

@@ -54,7 +54,7 @@ class TagFilterViewModel(
         runSuspendCatching<ImmutableList<Tag>> {
             withContext(ioDispatcher) {
               when (val result = api.getTags()) {
-                is Success -> result.value.tags.toImmutableList()
+                is Success -> result.value.toImmutableList()
                 is Failure.NetworkFailure -> throw result.error
                 is Failure.UnknownFailure -> throw result.error
                 is Failure.HttpFailure -> throw result.toError()

common/src/screenshotTest/kotlin/dev/msfjarvis/claw/common/comments/CommentEntryPreviewScreenshotTest.kt

@@ -47,7 +47,7 @@ private fun CommentEntryPreview(
   }
 }
 
-private data class CommentEntryPreviewParameters(
+private class CommentEntryPreviewParameters(
   val isUpvoted: Boolean,
   val hasChildren: Boolean,
   val isExpanded: Boolean,

common/src/test/kotlin/dev/msfjarvis/claw/common/comments/reply/ReplyViewModelTest.kt

@@ -11,12 +11,12 @@ import androidx.compose.ui.text.input.TextFieldValue
 import com.google.common.truth.Truth.assertThat
 import com.slack.eithernet.ApiResult
 import dev.msfjarvis.claw.api.AuthenticatedLobstersApi
-import dev.msfjarvis.claw.api.CSRFToken
 import dev.msfjarvis.claw.api.LobstersApi
-import dev.msfjarvis.claw.api.PostsPage
-import dev.msfjarvis.claw.api.ReplyForm
-import dev.msfjarvis.claw.api.TagsPage
+import dev.msfjarvis.claw.model.CSRFToken
+import dev.msfjarvis.claw.model.LobstersPost
 import dev.msfjarvis.claw.model.LobstersPostDetails
+import dev.msfjarvis.claw.model.ReplyForm
+import dev.msfjarvis.claw.model.Tag
 import dev.msfjarvis.claw.model.User
 import java.io.IOException
 import kotlinx.coroutines.Dispatchers
@@ -109,9 +109,11 @@ private class FakeLobstersApi(
   var postReplyCallCount: Int = 0
     private set
 
-  override suspend fun getHottestPosts(page: Int): ApiResult<PostsPage, Unit> = error("unused")
+  override suspend fun getHottestPosts(page: Int): ApiResult<List<LobstersPost>, Unit> =
+    error("unused")
 
-  override suspend fun getNewestPosts(page: Int): ApiResult<PostsPage, Unit> = error("unused")
+  override suspend fun getNewestPosts(page: Int): ApiResult<List<LobstersPost>, Unit> =
+    error("unused")
 
   override suspend fun getPostDetails(postId: String): ApiResult<LobstersPostDetails, Unit> =
     error("unused")
@@ -121,7 +123,7 @@ private class FakeLobstersApi(
   override suspend fun getCSRFToken(): ApiResult<CSRFToken, Unit> =
     ApiResult.success(CSRFToken("csrf"))
 
-  override suspend fun getTags(): ApiResult<TagsPage, Unit> = error("unused")
+  override suspend fun getTags(): ApiResult<List<Tag>, Unit> = error("unused")
 
   override suspend fun upvoteComment(
     commentId: String,

docs/ZIPLINE_UPDATE.md

@@ -0,0 +1,9 @@
+# Releasing an update for the Zipline module
+
+1. Set `ZIPLINE_SIGNING_KEY` in environment
+2. Build the production JS bundle: `gradle :zipline-parser:jsTest :zipline-parser:compileProductionExecutableKotlinJsZipline`
+3. Zip up the files at `zipline-parser/build/zipline/Production/` and upload them to wailord.
+   1. `zip -r -j zipline-release.zip zipline-parser/build/zipline/Production/`
+4. On wailord unzip the folder then run the deployment script from the Claw repo
+   1. `unzip -d release zipline-release.zip`
+   2. `scripts/deploy-zipline-parser.sh release ./release /var/lib/claw-deploy/`

gradle/libs.versions.toml

@@ -18,8 +18,7 @@ glance = "1.2.0-rc01"
 haze = "2.0.0-alpha02"
 screenshot = "0.0.1-alpha15"
 junit = "6.1.0"
-konvert = "4.5.0"
-kotlin = "2.3.21"
+kotlin = "2.3.20"
 kotlinResult = "2.3.1"
 lifecycle = "2.11.0-rc01"
 metro = "1.1.1"
@@ -35,6 +34,7 @@ sqldelight = "2.3.2"
 sqlite = "2.7.0-alpha06"
 tracing-perfetto = "1.0.1"
 workmanager = "2.11.2"
+zipline = "1.27.0"
 
 [libraries]
 aboutLibraries-compose-core = { module = "com.mikepenz:aboutlibraries-compose-core", version.ref = "aboutLibraries" }
@@ -113,7 +113,6 @@ haze = { module = "dev.chrisbanes.haze:haze", version.ref = "haze" }
 haze-blur = { module = "dev.chrisbanes.haze:haze-blur", version.ref = "haze" }
 htmlconverter = "be.digitalia.compose.htmlconverter:htmlconverter:1.1.1"
 ksoup = "com.fleeksoft.ksoup:ksoup:0.2.6"
-kspoon = "dev.burnoo.kspoon:kspoon:0.2.4"
 # Referenced in build-logic
 #noinspection UnusedVersionCatalogEntry
 junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" }
@@ -125,8 +124,6 @@ junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine" }
 junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher" }
 #noinspection UnusedVersionCatalogEntry
 junit-legacy = "junit:junit:4.13.2"
-konvert-annotations = { module = "io.mcarle:konvert-annotations", version.ref = "konvert" }
-konvert-processor = { module = "io.mcarle:konvert", version.ref = "konvert" }
 kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin" }
 kotlinResult = { module = "com.michael-bull.kotlin-result:kotlin-result", version.ref = "kotlinResult" }
 kotlinResult-coroutines = { module = "com.michael-bull.kotlin-result:kotlin-result-coroutines", version.ref = "kotlinResult" }
@@ -167,6 +164,8 @@ swipe = "me.saket.swipe:swipe:1.3.0"
 #noinspection UnusedVersionCatalogEntry
 truth = "com.google.truth:truth:1.4.5"
 unfurl = "me.saket.unfurl:unfurl:2.3.0"
+zipline = { module = "app.cash.zipline:zipline", version.ref = "zipline" }
+zipline-loader = { module = "app.cash.zipline:zipline-loader", version.ref = "zipline" }
 kotlin-parcelize-runtime = { module = "org.jetbrains.kotlin:kotlin-parcelize-runtime", version.ref = "kotlin" }
 kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" }
 
@@ -178,10 +177,10 @@ dependencyAnalysis = "com.autonomousapps.dependency-analysis:3.14.1"
 baselineprofile = { id = "androidx.baselineprofile", version.ref = "benchmark" }
 kotlin-composeCompiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" }
 kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" }
-ksp = "com.google.devtools.ksp:2.3.9"
 licensee = "app.cash.licensee:1.14.1"
 metro = { id = "dev.zacsweers.metro", version.ref = "metro" }
 screenshot = { id = "com.android.compose.screenshot", version.ref = "screenshot" }
 modulegraphassert = "com.jraska.module.graph.assertion:2.9.1"
 poko = "dev.drewhamilton.poko:0.22.1"
 sqldelight = { id = "app.cash.sqldelight", version.ref = "sqldelight" }
+zipline = { id = "app.cash.zipline", version.ref = "zipline" }

model/build.gradle.kts

@@ -4,23 +4,41 @@
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT.
  */
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask
+
 plugins {
-  id("dev.msfjarvis.claw.kotlin-jvm")
+  kotlin("multiplatform")
   alias(libs.plugins.kotlin.serialization)
-  alias(libs.plugins.poko)
-  alias(libs.plugins.ksp)
   alias(libs.plugins.dependencyAnalysis)
 }
 
-dependencies {
-  api(libs.kotlinx.datetime)
-  api(libs.kotlinx.serialization.core)
-  api(libs.kspoon)
-  api(projects.database.core)
+kotlin {
+  jvm()
+  js {
+    browser()
+  }
 
-  implementation(libs.ksoup)
-
-  compileOnly(libs.konvert.annotations)
-
-  ksp(libs.konvert.processor)
+  sourceSets {
+    commonMain {
+      dependencies {
+        api(libs.kotlinx.datetime)
+        api(libs.kotlinx.serialization.core)
+      }
+    }
+    jvmMain {
+      dependencies {
+        api(projects.database.core)
+      }
+    }
+    jvmTest {
+      dependencies {
+        implementation(kotlin("test"))
+      }
+    }
+  }
+}
+
+@Suppress("UnstableApiUsage")
+tasks.withType(KotlinCompilationTask::class.java).configureEach {
+  compilerOptions.freeCompilerArgs.add("-Xskip-prerelease-check")
 }

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/CSRFToken.kt

@@ -4,10 +4,9 @@
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT.
  */
-@file:Suppress("LongParameterList")
-
 package dev.msfjarvis.claw.model
 
-import dev.drewhamilton.poko.Poko
+import kotlin.jvm.JvmInline
+import kotlinx.serialization.Serializable
 
-@Poko class LinkMetadata(val url: String, val faviconUrl: String?)
+@Serializable @JvmInline value class CSRFToken(val value: String)

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/Comment.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import kotlin.time.Instant
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+class Comment(
+  val shortId: String,
+  val comment: String,
+  val url: String = "",
+  val score: Int = 1,
+  val timestamp: Instant,
+  val edited: Boolean = false,
+  val parentComment: String? = null,
+  @SerialName("commenting_user") val user: String = "",
+  val isUpvoted: Boolean = false,
+)

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/LinkMetadata.kt

@@ -0,0 +1,9 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+class LinkMetadata(val url: String, val faviconUrl: String?)

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/LobstersPost.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+class LobstersPost(
+  val shortId: String,
+  val createdAt: String = "",
+  val title: String,
+  val url: String = "",
+  val description: String = "",
+  val commentCount: Int = 0,
+  val commentsUrl: String = "",
+  @SerialName("submitter_user") val submitter: String,
+  @SerialName("user_is_author") val userIsAuthor: Boolean = false,
+  val tags: List<String> = emptyList(),
+) {
+  override fun equals(other: Any?): Boolean {
+    if (this === other) return true
+    if (other == null || this::class != other::class) return false
+
+    other as LobstersPost
+
+    if (commentCount != other.commentCount) return false
+    if (userIsAuthor != other.userIsAuthor) return false
+    if (shortId != other.shortId) return false
+    if (createdAt != other.createdAt) return false
+    if (title != other.title) return false
+    if (url != other.url) return false
+    if (description != other.description) return false
+    if (commentsUrl != other.commentsUrl) return false
+    if (submitter != other.submitter) return false
+    if (tags != other.tags) return false
+
+    return true
+  }
+
+  override fun hashCode(): Int {
+    var result = commentCount
+    result = 31 * result + userIsAuthor.hashCode()
+    result = 31 * result + shortId.hashCode()
+    result = 31 * result + createdAt.hashCode()
+    result = 31 * result + title.hashCode()
+    result = 31 * result + url.hashCode()
+    result = 31 * result + description.hashCode()
+    result = 31 * result + commentsUrl.hashCode()
+    result = 31 * result + submitter.hashCode()
+    result = 31 * result + tags.hashCode()
+    return result
+  }
+}

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/LobstersPostDetails.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+class LobstersPostDetails(
+  val shortId: String,
+  val createdAt: String = "",
+  val title: String,
+  val url: String = "",
+  val description: String = "",
+  val commentCount: Int = 0,
+  val commentsUrl: String = "",
+  @SerialName("submitter_user") val submitter: String,
+  val tags: List<String> = emptyList(),
+  val comments: List<Comment> = emptyList(),
+  @SerialName("user_is_author") val userIsAuthor: Boolean = false,
+)

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/ReplyForm.kt

@@ -4,9 +4,12 @@
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT.
  */
-package dev.msfjarvis.claw.api
+package dev.msfjarvis.claw.model
 
-data class ReplyForm(
+import kotlinx.serialization.Serializable
+
+@Serializable
+class ReplyForm(
   val authenticityToken: String,
   val storyId: String,
   val method: String,

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/Tag.kt

@@ -0,0 +1,21 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+class Tag(
+  val tag: String,
+  val description: String,
+  val privileged: Boolean = false,
+  val active: Boolean = true,
+  val category: String = "",
+  @SerialName("is_media") val isMedia: Boolean = false,
+  @SerialName("hotness_mod") val hotnessMod: Double = 0.0,
+)

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/TagBlock.kt

@@ -6,20 +6,12 @@
  */
 package dev.msfjarvis.claw.model
 
-import dev.drewhamilton.poko.Poko
+import kotlin.time.Clock
 
-/**
- * Represents a blocked tag with an optional expiration date.
- *
- * @property tag The tag name to block
- * @property expirationMillis The expiration timestamp in milliseconds since epoch, or null for
- *   permanent blocks
- */
-@Poko
 class TagBlock(val tag: String, val expirationMillis: Long?) {
   val isPermanent: Boolean
     get() = expirationMillis == null
 
   val isExpired: Boolean
-    get() = expirationMillis?.let { it < System.currentTimeMillis() } ?: false
+    get() = expirationMillis?.let { it < Clock.System.now().toEpochMilliseconds() } ?: false
 }

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/UIPost.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import kotlinx.serialization.SerialName
+
+@Suppress("LongParameterList")
+data class UIPost(
+  val shortId: String,
+  val createdAt: String,
+  val title: String,
+  val url: String,
+  val description: String,
+  val commentCount: Int,
+  val commentsUrl: String,
+  @SerialName("submitter_user") val submitter: String,
+  val tags: List<String>,
+  val comments: List<Comment> = emptyList(),
+  @SerialName("user_is_author") val userIsAuthor: Boolean = false,
+) {
+  companion object
+}

model/src/commonMain/kotlin/dev/msfjarvis/claw/model/User.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+class User(
+  val username: String,
+  val about: String = "",
+  @SerialName("invited_by_user") val invitedBy: String? = null,
+  val avatarUrl: String = "",
+  val createdAt: String = "",
+)

model/src/jvmMain/kotlin/dev/msfjarvis/claw/model/DatabasePostMappers.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import dev.msfjarvis.claw.database.local.CachedRemotePost
+import dev.msfjarvis.claw.database.local.SavedPost
+
+fun UIPost.Companion.fromSavedPost(post: SavedPost): UIPost =
+  UIPost(
+    shortId = post.shortId,
+    createdAt = post.createdAt,
+    title = post.title,
+    url = post.url,
+    description = post.description,
+    commentCount = post.commentCount ?: 0,
+    commentsUrl = post.commentsUrl,
+    submitter = post.submitterName,
+    tags = post.tags,
+    userIsAuthor = post.userIsAuthor,
+  )
+
+fun UIPost.Companion.fromCachedRemotePost(post: CachedRemotePost): UIPost =
+  UIPost(
+    shortId = post.shortId,
+    createdAt = post.createdAt,
+    title = post.title,
+    url = post.url,
+    description = post.description,
+    commentCount = post.commentCount ?: 0,
+    commentsUrl = post.commentsUrl,
+    submitter = post.submitterName,
+    tags = post.tags,
+    userIsAuthor = post.userIsAuthor,
+  )

model/src/jvmMain/kotlin/dev/msfjarvis/claw/model/ModelMappers.kt

@@ -0,0 +1,66 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.model
+
+import dev.msfjarvis.claw.database.local.SavedPost
+
+fun LobstersPost.toUIPost(): UIPost =
+  UIPost(
+    shortId = shortId,
+    createdAt = createdAt,
+    title = title,
+    url = url,
+    description = description,
+    commentCount = commentCount,
+    commentsUrl = commentsUrl,
+    submitter = submitter,
+    tags = tags,
+    userIsAuthor = userIsAuthor,
+  )
+
+fun LobstersPostDetails.toUIPost(): UIPost =
+  UIPost(
+    shortId = shortId,
+    createdAt = createdAt,
+    title = title,
+    url = url,
+    description = description,
+    commentCount = commentCount,
+    commentsUrl = commentsUrl,
+    submitter = submitter,
+    tags = tags,
+    comments = comments,
+    userIsAuthor = userIsAuthor,
+  )
+
+fun LobstersPostDetails.toSavedPost(): SavedPost =
+  SavedPost(
+    shortId = shortId,
+    title = title,
+    url = url,
+    createdAt = createdAt,
+    commentCount = commentCount,
+    commentsUrl = commentsUrl,
+    submitterName = submitter,
+    tags = tags,
+    description = description,
+    userIsAuthor = userIsAuthor,
+  )
+
+fun UIPost.toSavedPost(): SavedPost =
+  SavedPost(
+    shortId = shortId,
+    title = title,
+    url = url,
+    createdAt = createdAt,
+    commentCount = commentCount,
+    commentsUrl = commentsUrl,
+    submitterName = submitter,
+    tags = tags,
+    description = description,
+    userIsAuthor = userIsAuthor,
+  )

model/src/main/kotlin/dev/msfjarvis/claw/model/Comment.kt

@@ -1,87 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-@file:Suppress("LongParameterList")
-
-package dev.msfjarvis.claw.model
-
-import dev.burnoo.kspoon.SelectorHtmlTextMode
-import dev.burnoo.kspoon.annotation.Selector
-import dev.drewhamilton.poko.Poko
-import kotlin.time.Instant
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-@Serializable
-@Poko
-@Selector("div.comment")
-class Comment(
-  @Selector(":root", attr = "data-shortid") val shortId: String,
-  @Selector("div.comment_text", textMode = SelectorHtmlTextMode.InnerHtml) val comment: String,
-  @Selector("div.byline a[href^=/c/]", attr = "abs:href", defValue = "") val url: String = "",
-  @Serializable(with = CommentScoreSerializer::class)
-  @Selector("div.voters a.upvoter", defValue = "1")
-  val score: Int = 1,
-  @Serializable(with = CommentInstantSerializer::class)
-  @Selector("div.byline a[href^=/c/] time", attr = "data-at-unix", defValue = "")
-  val timestamp: Instant,
-  @Serializable(with = CommentEditedSerializer::class)
-  @Selector("div.byline span", defValue = "")
-  val edited: Boolean = false,
-  @Serializable(with = EmptyStringAsNullSerializer::class)
-  @Selector(":root", attr = "data-parent-shortid", defValue = "")
-  val parentComment: String? = null,
-  @Selector("div.byline a[href^=/~/]", defValue = "")
-  @SerialName("commenting_user")
-  val user: String = "",
-  @Serializable(with = CommentUpvotedSerializer::class)
-  @Selector(":root", attr = "class", defValue = "")
-  val isUpvoted: Boolean = false,
-)
-
-internal object CommentInstantSerializer : KSerializer<Instant> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("CommentInstant", PrimitiveKind.STRING)
-
-  override fun deserialize(decoder: Decoder): Instant = decoder.decodeString().parseInstantOrEpoch()
-
-  override fun serialize(encoder: Encoder, value: Instant) = encoder.encodeString(value.toString())
-}
-
-internal object CommentScoreSerializer : KSerializer<Int> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("CommentScore", PrimitiveKind.INT)
-
-  override fun deserialize(decoder: Decoder): Int =
-    decoder.decodeString().trim().takeUnless { it == "~" }?.toIntOrNull() ?: 1
-
-  override fun serialize(encoder: Encoder, value: Int) = encoder.encodeInt(value)
-}
-
-internal object EmptyStringAsNullSerializer : KSerializer<String?> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("EmptyStringAsNull", PrimitiveKind.STRING)
-
-  override fun deserialize(decoder: Decoder): String? = decoder.decodeString().ifBlank { null }
-
-  override fun serialize(encoder: Encoder, value: String?) = encoder.encodeString(value.orEmpty())
-}
-
-internal object CommentUpvotedSerializer : KSerializer<Boolean> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("CommentUpvoted", PrimitiveKind.BOOLEAN)
-
-  override fun deserialize(decoder: Decoder): Boolean =
-    decoder.decodeString().split(Regex("\\s+")).contains("upvoted")
-
-  override fun serialize(encoder: Encoder, value: Boolean) = encoder.encodeBoolean(value)
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/CommentEditedSerializer.kt

@@ -1,23 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.model
-
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-internal object CommentEditedSerializer : KSerializer<Boolean> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("CommentEdited", PrimitiveKind.BOOLEAN)
-
-  override fun deserialize(decoder: Decoder): Boolean = decoder.decodeString().contains("edited")
-
-  override fun serialize(encoder: Encoder, value: Boolean) = encoder.encodeBoolean(value)
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/CommentsSerializer.kt

@@ -1,84 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.model
-
-import com.fleeksoft.ksoup.nodes.Element
-import dev.burnoo.kspoon.decoder.KspoonDecoder
-import kotlinx.serialization.InternalSerializationApi
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.descriptors.StructureKind
-import kotlinx.serialization.descriptors.buildSerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-@OptIn(InternalSerializationApi::class)
-internal object CommentsSerializer : KSerializer<List<Comment>> {
-  override val descriptor: SerialDescriptor = buildSerialDescriptor("Comments", StructureKind.LIST)
-
-  override fun deserialize(decoder: Decoder): List<Comment> {
-    val elements = (decoder as KspoonDecoder).decodeElements()
-    val seen = mutableSetOf<String>()
-    return buildList {
-      elements.forEach { subtree -> addSubtree(subtree, parentComment = null, seen) }
-    }
-  }
-
-  private fun MutableList<Comment>.addSubtree(
-    subtree: Element,
-    parentComment: String?,
-    seen: MutableSet<String>,
-  ) {
-    val commentElement =
-      if (subtree.`is`("div.comment")) subtree
-      else subtree.children().firstOrNull { it.`is`("div.comment") } ?: return
-    val shortId = commentElement.attr("data-shortid")
-    if (!seen.add(shortId)) return
-    val comment = commentElement.toComment(parentComment)
-    add(comment)
-    val childContainer = if (subtree.`is`("div.comment")) subtree.parent() ?: subtree else subtree
-    childContainer
-      .children()
-      .filter { it.`is`("ol.comments") }
-      .flatMap { comments -> comments.children().filter { it.`is`("li.comments_subtree") } }
-      .forEach { child -> addSubtree(child, parentComment = comment.shortId, seen) }
-  }
-
-  private fun Element.toComment(parentComment: String?): Comment {
-    val byline = selectFirst("div.byline")
-    val timestamp = byline?.selectFirst("a[href^=/c/] time")?.attr("data-at-unix").orEmpty()
-    val parsedTimestamp = timestamp.parseInstantOrEpoch()
-    val isEdited = byline?.text()?.contains("edited") == true
-    return Comment(
-      shortId = attr("data-shortid"),
-      comment = selectFirst("div.comment_text")?.html().orEmpty(),
-      url = selectFirst("div.byline a[href^=/c/]")?.absUrl("href").orEmpty(),
-      score =
-        children()
-          .firstOrNull { it.hasClass("voters") }
-          ?.children()
-          ?.firstOrNull { it.hasClass("upvoter") }
-          ?.text()
-          ?.trim()
-          ?.takeUnless { it == "~" }
-          ?.toIntOrNull() ?: 1,
-      timestamp = parsedTimestamp,
-      edited = isEdited,
-      parentComment = parentComment,
-      user =
-        getElementsByClass("byline")
-          .flatMap { it.getElementsByTag("a") }
-          .firstOrNull { it.attr("href").contains("/~") && it.text().isNotBlank() }
-          ?.text()
-          .orEmpty(),
-      isUpvoted = classNames().contains("upvoted"),
-    )
-  }
-
-  override fun serialize(encoder: Encoder, value: List<Comment>) =
-    throw UnsupportedOperationException()
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/DateTimeUtils.kt

@@ -1,44 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.model
-
-import kotlin.time.Instant
-import kotlinx.datetime.LocalDateTime
-import kotlinx.datetime.TimeZone
-import kotlinx.datetime.format.FormatStringsInDatetimeFormats
-import kotlinx.datetime.format.byUnicodePattern
-import kotlinx.datetime.toInstant
-
-internal fun String.parseInstantOrEpoch(): Instant =
-  parseInstantOrNull() ?: Instant.fromEpochSeconds(0)
-
-internal fun String.parseInstantOrNull(): Instant? {
-  return when {
-    isBlank() -> Instant.fromEpochSeconds(0)
-    all(Char::isDigit) -> toLongOrNull()?.let(Instant::fromEpochSeconds)
-    else -> parseIsoInstant() ?: parseLegacyLobstersDateTime()?.toInstant(TimeZone.UTC)
-  }
-}
-
-private fun String.parseIsoInstant(): Instant? =
-  try {
-    Instant.parse(this)
-  } catch (_: IllegalArgumentException) {
-    null
-  }
-
-private fun String.parseLegacyLobstersDateTime(): LocalDateTime? =
-  try {
-    LEGACY_LOBSTERS_DATE_TIME_FORMAT.parse(this)
-  } catch (_: IllegalArgumentException) {
-    null
-  }
-
-@OptIn(FormatStringsInDatetimeFormats::class)
-private val LEGACY_LOBSTERS_DATE_TIME_FORMAT = LocalDateTime.Format {
-  byUnicodePattern("yyyy-MM-dd HH:mm:ss")
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/LobstersPost.kt

@@ -1,94 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-@file:Suppress("LongParameterList")
-
-package dev.msfjarvis.claw.model
-
-import dev.burnoo.kspoon.annotation.Selector
-import dev.drewhamilton.poko.Poko
-import io.mcarle.konvert.api.KonvertTo
-import io.mcarle.konvert.api.Mapping
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-@Serializable
-@Poko
-@KonvertTo(
-  value = UIPost::class,
-  mappings = [Mapping(target = "submitter", expression = "it.submitter")],
-)
-class LobstersPost(
-  @Selector(":root", attr = "data-shortid") val shortId: String,
-  @Serializable(with = CreatedAtSerializer::class)
-  @Selector("> div.story_liner div.byline > time", attr = "data-at-unix", defValue = "")
-  val createdAt: String = "",
-  @Selector("> div.story_liner span.link.h-cite > a") val title: String,
-  @Selector("> div.story_liner span.link.h-cite > a", attr = "abs:href", defValue = "")
-  val url: String = "",
-  @Selector("> div.story_liner a.description_present", attr = "title", defValue = "")
-  val description: String = "",
-  @Serializable(with = CommentCountSerializer::class)
-  @Selector(
-    "> div.story_liner span.comments_label a",
-    regex = "(\\d+ comments?|no comments)",
-    defValue = "0",
-  )
-  val commentCount: Int = 0,
-  @Selector("> div.story_liner span.comments_label a", attr = "abs:href", defValue = "")
-  val commentsUrl: String = "",
-  @Selector("> div.story_liner div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])")
-  @SerialName("submitter_user")
-  val submitter: String,
-  @Serializable(with = UserIsAuthorSerializer::class)
-  @Selector(
-    "> div.story_liner div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])",
-    attr = "class",
-    defValue = "",
-  )
-  @SerialName("user_is_author")
-  val userIsAuthor: Boolean = false,
-  @Selector("> div.story_liner span.tags > a") val tags: List<String> = emptyList(),
-)
-
-internal object CreatedAtSerializer : KSerializer<String> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("CreatedAt", PrimitiveKind.STRING)
-
-  override fun deserialize(decoder: Decoder): String {
-    val value = decoder.decodeString()
-    return if (value.isBlank()) "" else value.parseInstantOrNull()?.toString().orEmpty()
-  }
-
-  override fun serialize(encoder: Encoder, value: String) = encoder.encodeString(value)
-}
-
-internal object CommentCountSerializer : KSerializer<Int> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("CommentCount", PrimitiveKind.INT)
-
-  override fun deserialize(decoder: Decoder): Int {
-    return "\\d+".toRegex().find(decoder.decodeString())?.value?.toInt() ?: 0
-  }
-
-  override fun serialize(encoder: Encoder, value: Int) = encoder.encodeInt(value)
-}
-
-internal object UserIsAuthorSerializer : KSerializer<Boolean> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("UserIsAuthor", PrimitiveKind.BOOLEAN)
-
-  override fun deserialize(decoder: Decoder): Boolean =
-    decoder.decodeString().split(' ').contains("user_is_author")
-
-  override fun serialize(encoder: Encoder, value: Boolean) = encoder.encodeBoolean(value)
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/LobstersPostDetails.kt

@@ -1,67 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-@file:Suppress("LongParameterList")
-
-package dev.msfjarvis.claw.model
-
-import dev.burnoo.kspoon.SelectorHtmlTextMode
-import dev.burnoo.kspoon.annotation.Selector
-import dev.drewhamilton.poko.Poko
-import dev.msfjarvis.claw.database.local.SavedPost
-import io.mcarle.konvert.api.KonvertTo
-import io.mcarle.konvert.api.Mapping
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-
-@Serializable
-@Poko
-@KonvertTo(value = UIPost::class)
-@KonvertTo(
-  value = SavedPost::class,
-  mappings = [Mapping(source = "submitter", target = "submitterName")],
-)
-class LobstersPostDetails(
-  @Selector("ol.stories > li.story", attr = "data-shortid") val shortId: String,
-  @Serializable(with = CreatedAtSerializer::class)
-  @Selector("ol.stories > li.story div.byline > time", attr = "data-at-unix", defValue = "")
-  val createdAt: String = "",
-  @Selector("ol.stories > li.story span.link.h-cite > a") val title: String,
-  @Selector("ol.stories > li.story span.link.h-cite > a", attr = "abs:href", defValue = "")
-  val url: String = "",
-  @Selector(
-    "div.story_content div.story_text",
-    textMode = SelectorHtmlTextMode.InnerHtml,
-    defValue = "",
-  )
-  val description: String = "",
-  @Serializable(with = CommentCountSerializer::class)
-  @Selector(
-    "ol.stories > li.story span.comments_label a",
-    regex = "(\\d+ comments?|\\d+|no comments)",
-    defValue = "0",
-  )
-  val commentCount: Int = 0,
-  @Selector("ol.stories > li.story span.comments_label a", attr = "abs:href", defValue = "")
-  val commentsUrl: String = "",
-  @Selector(
-    "ol.stories > li.story div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])"
-  )
-  @SerialName("submitter_user")
-  val submitter: String,
-  @Selector("ol.stories > li.story span.tags > a") val tags: List<String> = emptyList(),
-  @Serializable(with = CommentsSerializer::class)
-  @Selector("ol.comments > li.comments_subtree")
-  val comments: List<Comment> = emptyList(),
-  @Serializable(with = UserIsAuthorSerializer::class)
-  @Selector(
-    "ol.stories > li.story div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])",
-    attr = "class",
-    defValue = "",
-  )
-  @SerialName("user_is_author")
-  val userIsAuthor: Boolean = false,
-)

model/src/main/kotlin/dev/msfjarvis/claw/model/Tag.kt

@@ -1,68 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.model
-
-import dev.burnoo.kspoon.annotation.Selector
-import dev.drewhamilton.poko.Poko
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-@Serializable
-@Poko
-class Tag(
-  @Selector("> a.tag") val tag: String,
-  @Selector("> span:not(.byline)") val description: String,
-  @Selector(":root", attr = "data-privileged", defValue = "false") val privileged: Boolean = false,
-  @Serializable(with = ActiveTagSerializer::class)
-  @Selector("> span:not(.byline)", attr = "class", defValue = "")
-  val active: Boolean = true,
-  @Selector(":root", attr = "data-category", defValue = "") val category: String = "",
-  @Serializable(with = MediaTagSerializer::class)
-  @Selector("> a.tag", attr = "class", defValue = "")
-  @SerialName("is_media")
-  val isMedia: Boolean = false,
-  @Serializable(with = HotnessModSerializer::class)
-  @Selector(":root", attr = "data-hotness-mod", defValue = "0.0")
-  @SerialName("hotness_mod")
-  val hotnessMod: Double = 0.0,
-)
-
-internal object ActiveTagSerializer : KSerializer<Boolean> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("ActiveTag", PrimitiveKind.BOOLEAN)
-
-  override fun deserialize(decoder: Decoder): Boolean =
-    !decoder.decodeString().split(Regex("\\s+")).contains("inactive_tag")
-
-  override fun serialize(encoder: Encoder, value: Boolean) = encoder.encodeBoolean(value)
-}
-
-internal object MediaTagSerializer : KSerializer<Boolean> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("MediaTag", PrimitiveKind.BOOLEAN)
-
-  override fun deserialize(decoder: Decoder): Boolean =
-    decoder.decodeString().split(Regex("\\s+")).contains("tag_is_media")
-
-  override fun serialize(encoder: Encoder, value: Boolean) = encoder.encodeBoolean(value)
-}
-
-internal object HotnessModSerializer : KSerializer<Double> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("HotnessMod", PrimitiveKind.DOUBLE)
-
-  override fun deserialize(decoder: Decoder): Double =
-    decoder.decodeString().toDoubleOrNull() ?: 0.0
-
-  override fun serialize(encoder: Encoder, value: Double) = encoder.encodeDouble(value)
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/UIPost.kt

@@ -1,58 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.model
-
-import dev.msfjarvis.claw.database.local.CachedRemotePost
-import dev.msfjarvis.claw.database.local.SavedPost
-import io.mcarle.konvert.api.KonvertFrom
-import io.mcarle.konvert.api.KonvertTo
-import io.mcarle.konvert.api.Mapping
-import kotlinx.serialization.SerialName
-
-@KonvertTo(
-  value = SavedPost::class,
-  mappings = [Mapping(source = "submitter", target = "submitterName")],
-)
-@KonvertTo(
-  value = CachedRemotePost::class,
-  mappings =
-    [
-      Mapping(source = "submitter", target = "submitterName"),
-      Mapping(target = "insertionOrder", constant = "0"),
-    ],
-)
-data class UIPost(
-  val shortId: String,
-  val createdAt: String,
-  val title: String,
-  val url: String,
-  val description: String,
-  val commentCount: Int,
-  val commentsUrl: String,
-  @SerialName("submitter_user") val submitter: String,
-  val tags: List<String>,
-  val comments: List<Comment> = emptyList(),
-  @SerialName("user_is_author") val userIsAuthor: Boolean = false,
-) {
-  @KonvertFrom(
-    value = SavedPost::class,
-    mappings =
-      [
-        Mapping(source = "submitterName", target = "submitter"),
-        Mapping(target = "commentCount", expression = "it.commentCount ?: 0"),
-      ],
-  )
-  @KonvertFrom(
-    value = CachedRemotePost::class,
-    mappings =
-      [
-        Mapping(source = "submitterName", target = "submitter"),
-        Mapping(target = "commentCount", expression = "it.commentCount ?: 0"),
-      ],
-  )
-  companion object
-}

model/src/main/kotlin/dev/msfjarvis/claw/model/User.kt

@@ -1,38 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-@file:Suppress("LongParameterList")
-
-package dev.msfjarvis.claw.model
-
-import dev.burnoo.kspoon.SelectorHtmlTextMode
-import dev.burnoo.kspoon.annotation.Selector
-import dev.drewhamilton.poko.Poko
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-
-@Serializable
-@Poko
-class User(
-  @Selector("#inside > h1") val username: String,
-  @Selector(
-    "section.profile .shorten_first_p",
-    textMode = SelectorHtmlTextMode.InnerHtml,
-    defValue = "",
-  )
-  val about: String = "",
-  @Serializable(with = EmptyStringAsNullSerializer::class)
-  @Selector(
-    "section.profile .labelled_grid label:contains(Joined) + span a[href^=/~/]",
-    defValue = "",
-  )
-  @SerialName("invited_by_user")
-  val invitedBy: String? = null,
-  @Selector("section.profile #gravatar img.avatar", attr = "abs:src", defValue = "")
-  val avatarUrl: String = "",
-  @Selector("section.profile .labelled_grid label:contains(Joined) + span time", defValue = "")
-  val createdAt: String = "",
-)

model/src/main/kotlin/dev/msfjarvis/claw/serialization/JavaInstantSerializer.kt

@@ -1,24 +0,0 @@
-/*
- * Copyright © Harsh Shandilya.
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- */
-package dev.msfjarvis.claw.serialization
-
-import kotlin.time.Instant
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-class JavaInstantSerializer : KSerializer<Instant> {
-  override val descriptor: SerialDescriptor =
-    PrimitiveSerialDescriptor("Instant", PrimitiveKind.STRING)
-
-  override fun deserialize(decoder: Decoder): Instant = Instant.parse(decoder.decodeString())
-
-  override fun serialize(encoder: Encoder, value: Instant) = encoder.encodeString(value.toString())
-}

scripts/deploy-zipline-parser.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+manifest_name="manifest.zipline.json"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  deploy-zipline-parser.sh release <source-dir> <target-dir> [timestamp]
+  deploy-zipline-parser.sh rollback <target-dir> <timestamp>
+  deploy-zipline-parser.sh list <target-dir>
+  deploy-zipline-parser.sh status <target-dir>
+  deploy-zipline-parser.sh prune <target-dir>
+
+Commands:
+  release   Create a timestamped release in <target-dir>/releases/<timestamp>
+            and atomically repoint <target-dir>/current to it.
+  rollback  Repoint <target-dir>/current to an existing release timestamp.
+  list      List available release timestamps.
+  status    Print the current active release.
+  prune     Delete all but the 5 most recent releases. Never deletes the active release.
+EOF
+}
+
+err() {
+  echo "error: $*" >&2
+  exit 1
+}
+
+timestamp_now() {
+  date -u +%Y%m%dT%H%M%SZ
+}
+
+require_dir() {
+  local dir="$1"
+  [[ -d "${dir}" ]] || err "directory does not exist: ${dir}"
+}
+
+release_command() {
+  local src_dir="$1"
+  local target_dir="$2"
+  local timestamp="${3:-$(timestamp_now)}"
+  local manifest_path="${src_dir}/${manifest_name}"
+  local releases_dir="${target_dir}/releases"
+  local release_dir="${releases_dir}/${timestamp}"
+  local tmp_link
+
+  require_dir "${src_dir}"
+  [[ -f "${manifest_path}" ]] || err "missing manifest: ${manifest_path}"
+
+  shopt -s nullglob
+  local module_files=("${src_dir}"/*.zipline)
+  shopt -u nullglob
+  [[ "${#module_files[@]}" -gt 0 ]] || err "no .zipline module files found in ${src_dir}"
+  [[ ! -e "${release_dir}" ]] || err "release already exists: ${release_dir}"
+
+  mkdir -p "${releases_dir}"
+  mkdir -p "${release_dir}"
+
+  echo "Creating release ${timestamp} in ${release_dir}"
+  for module_path in "${module_files[@]}"; do
+    local module_name
+    module_name="$(basename "${module_path}")"
+    echo "Copying module: ${module_name}"
+    cp "${module_path}" "${release_dir}/${module_name}"
+  done
+
+  echo "Copying manifest last: ${manifest_name}"
+  cp "${manifest_path}" "${release_dir}/${manifest_name}"
+
+  tmp_link="${target_dir}/.current.${timestamp}.tmp"
+  ln -sfn "${release_dir}" "${tmp_link}"
+  mv -Tf "${tmp_link}" "${target_dir}/current"
+
+  echo "Activated release ${timestamp}"
+}
+
+rollback_command() {
+  local target_dir="$1"
+  local timestamp="$2"
+  local release_dir="${target_dir}/releases/${timestamp}"
+  local tmp_link="${target_dir}/.current.${timestamp}.tmp"
+
+  require_dir "${target_dir}"
+  [[ -d "${release_dir}" ]] || err "release does not exist: ${release_dir}"
+  [[ -f "${release_dir}/${manifest_name}" ]] || err "release is missing manifest: ${release_dir}/${manifest_name}"
+
+  ln -sfn "${release_dir}" "${tmp_link}"
+  mv -Tf "${tmp_link}" "${target_dir}/current"
+
+  echo "Rolled back current release to ${timestamp}"
+}
+
+list_command() {
+  local target_dir="$1"
+  local releases_dir="${target_dir}/releases"
+
+  require_dir "${target_dir}"
+  mkdir -p "${releases_dir}"
+  find "${releases_dir}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort
+}
+
+status_command() {
+  local target_dir="$1"
+  local current_link="${target_dir}/current"
+
+  require_dir "${target_dir}"
+  [[ -L "${current_link}" ]] || err "current release symlink is missing: ${current_link}"
+
+  local current_target
+  current_target="$(readlink "${current_link}")"
+  local current_timestamp
+  current_timestamp="$(basename "${current_target}")"
+
+  echo "current=${current_timestamp}"
+  echo "path=${current_target}"
+}
+
+prune_command() {
+  local target_dir="$1"
+  local releases_dir="${target_dir}/releases"
+  local current_link="${target_dir}/current"
+
+  require_dir "${target_dir}"
+  mkdir -p "${releases_dir}"
+
+  local current_timestamp=""
+  if [[ -L "${current_link}" ]]; then
+    current_timestamp="$(basename "$(readlink "${current_link}")")"
+  fi
+
+  mapfile -t releases < <(find "${releases_dir}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sort)
+
+  if [[ "${#releases[@]}" -le 5 ]]; then
+    echo "Nothing to prune"
+    return 0
+  fi
+
+  local delete_count=$((${#releases[@]} - 5))
+  for ((i=0; i<delete_count; i++)); do
+    local timestamp="${releases[$i]}"
+    if [[ -n "${current_timestamp}" && "${timestamp}" == "${current_timestamp}" ]]; then
+      echo "Skipping active release: ${timestamp}"
+      continue
+    fi
+    echo "Pruning release: ${timestamp}"
+    rm -rf "${releases_dir}/${timestamp}"
+  done
+}
+
+main() {
+  local command="${1:-}"
+  case "${command}" in
+    release)
+      [[ "$#" -eq 3 || "$#" -eq 4 ]] || { usage >&2; exit 1; }
+      release_command "$2" "$3" "${4:-}"
+      ;;
+    rollback)
+      [[ "$#" -eq 3 ]] || { usage >&2; exit 1; }
+      rollback_command "$2" "$3"
+      ;;
+    list)
+      [[ "$#" -eq 2 ]] || { usage >&2; exit 1; }
+      list_command "$2"
+      ;;
+    status)
+      [[ "$#" -eq 2 ]] || { usage >&2; exit 1; }
+      status_command "$2"
+      ;;
+    prune)
+      [[ "$#" -eq 2 ]] || { usage >&2; exit 1; }
+      prune_command "$2"
+      ;;
+    -h|--help)
+      usage
+      ;;
+    *)
+      usage >&2
+      exit 1
+      ;;
+  esac
+}
+
+main "$@"

scripts/release-zipline-parser.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly archive_name="zipline-release.zip"
+readonly production_dir="zipline-parser/build/zipline/Production"
+readonly remote_host="msfjarvis@wailord"
+readonly remote_deploy_script_rel="git-repos/compose-lobsters/scripts/deploy-zipline-parser.sh"
+readonly remote_target_dir="/var/lib/claw-deploy/"
+readonly gradle_tasks=(
+  :zipline-parser:jsTest
+  :zipline-parser:compileProductionExecutableKotlinJsZipline
+)
+
+usage() {
+  cat <<'EOF'
+Usage:
+  scripts/release-zipline-parser.sh
+
+Build the production Zipline parser bundle, archive it, upload it to wailord,
+and activate it using the deploy script on the server.
+
+Required environment:
+  ZIPLINE_SIGNING_KEY   Signing key used during the production Zipline build.
+EOF
+}
+
+err() {
+  echo "error: $*" >&2
+  exit 1
+}
+
+require_command() {
+  local command="$1"
+  command -v "${command}" >/dev/null 2>&1 || err "missing required command: ${command}"
+}
+
+main() {
+  [[ "${1:-}" =~ ^(-h|--help)$ ]] && { usage; exit 0; }
+  [[ "$#" -eq 0 ]] || { usage >&2; exit 1; }
+
+  [[ -n "${ZIPLINE_SIGNING_KEY:-}" ]] || err "ZIPLINE_SIGNING_KEY must be set"
+
+  require_command build-brief
+  require_command zip
+  require_command rsync
+  require_command ssh
+  require_command unzip
+
+  echo "Building production Zipline bundle"
+  build-brief ./gradlew "${gradle_tasks[@]}"
+
+  [[ -d "${production_dir}" ]] || err "missing production output directory: ${production_dir}"
+
+  local archive_path
+  archive_path="$(mktemp -t zipline-release.XXXXXX).zip"
+  trap "rm -f -- ${archive_path@Q}" EXIT
+
+  echo "Creating release archive: ${archive_path}"
+  (
+    cd "${production_dir}"
+    zip -r -j "${archive_path}" .
+  )
+
+  local remote_archive_name
+  remote_archive_name="$(basename "${archive_path}")"
+  echo "Uploading archive to ${remote_host}:~/${remote_archive_name}"
+  rsync "${archive_path}" "${remote_host}:~/${remote_archive_name}"
+
+  echo "Deploying archive on ${remote_host}"
+  ssh "${remote_host}" "ARCHIVE_NAME=${remote_archive_name@Q} DEPLOY_SCRIPT_REL=${remote_deploy_script_rel@Q} TARGET_DIR=${remote_target_dir@Q} bash -s" <<'EOF'
+set -euo pipefail
+
+archive_path="${HOME}/${ARCHIVE_NAME}"
+deploy_script="${HOME}/${DEPLOY_SCRIPT_REL}"
+release_dir="$(mktemp -d)"
+trap 'rm -rf "${release_dir}" "${archive_path}"' EXIT
+
+unzip -d "${release_dir}" "${archive_path}"
+"${deploy_script}" release "${release_dir}" "${TARGET_DIR}"
+EOF
+
+  echo "Zipline parser release deployed successfully"
+}
+
+main "$@"

settings.gradle.kts

@@ -70,7 +70,7 @@ develocity {
 }
 
 dependencyResolutionManagement {
-  repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
+  repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)
   repositories {
     google {
       mavenContent { releasesOnly() }
@@ -99,6 +99,52 @@ dependencyResolutionManagement {
       name = "Sonatype Snapshots"
       mavenContent { snapshotsOnly() }
     }
+    exclusiveContent {
+      forRepository {
+        ivy("https://download.jetbrains.com/kotlin/native/builds") {
+          name = "Kotlin Native"
+          patternLayout {
+            listOf(
+                "macos-x86_64",
+                "macos-aarch64",
+                "osx-x86_64",
+                "osx-aarch64",
+                "linux-x86_64",
+                "windows-x86_64",
+              )
+              .forEach { os ->
+                listOf("dev", "releases").forEach { stage ->
+                  artifact("$stage/[revision]/$os/[artifact]-[revision].[ext]")
+                }
+              }
+          }
+          metadataSources { artifact() }
+        }
+      }
+      filter { includeModuleByRegex(".*", ".*kotlin-native-prebuilt.*") }
+    }
+    exclusiveContent {
+      forRepository {
+        ivy("https://nodejs.org/dist/") {
+          name = "Node Distributions at $url"
+          patternLayout { artifact("v[revision]/[artifact](-v[revision]-[classifier]).[ext]") }
+          metadataSources { artifact() }
+          content { includeModule("org.nodejs", "node") }
+        }
+      }
+      filter { includeGroup("org.nodejs") }
+    }
+    exclusiveContent {
+      forRepository {
+        ivy("https://github.com/yarnpkg/yarn/releases/download") {
+          name = "Yarn Distributions at $url"
+          patternLayout { artifact("v[revision]/[artifact](-v[revision]).[ext]") }
+          metadataSources { artifact() }
+          content { includeModule("com.yarnpkg", "yarn") }
+        }
+      }
+      filter { includeGroup("com.yarnpkg") }
+    }
     mavenCentral { mavenContent { releasesOnly() } }
   }
 }
@@ -107,4 +153,15 @@ rootProject.name = "Claw"
 
 enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
 
-include("android", "api", "benchmark", "common", "core", "database:core", "database:impl", "model")
+include(
+  "android",
+  "api",
+  "benchmark",
+  "common",
+  "core",
+  "database:core",
+  "database:impl",
+  "model",
+  "zipline-parser",
+  "zipline-parser-api",
+)

zipline-parser-api/build.gradle.kts

@@ -0,0 +1,29 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+plugins {
+  kotlin("multiplatform")
+}
+
+kotlin {
+  jvm()
+  js {
+    nodejs()
+  }
+
+  sourceSets {
+    commonMain {
+      dependencies {
+        api(libs.zipline)
+      }
+    }
+    jvmTest {
+      dependencies {
+        implementation(kotlin("test"))
+      }
+    }
+  }
+}

zipline-parser-api/src/commonMain/kotlin/dev/msfjarvis/claw/parser/model/ParserModels.kt

@@ -0,0 +1,75 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.model
+
+import kotlin.jvm.JvmInline
+
+class LobstersPost(
+  val shortId: String,
+  val createdAt: String = "",
+  val title: String,
+  val url: String = "",
+  val description: String = "",
+  val commentCount: Int = 0,
+  val commentsUrl: String = "",
+  val submitter: String,
+  val userIsAuthor: Boolean = false,
+  val tags: List<String> = emptyList(),
+)
+
+class LobstersPostDetails(
+  val shortId: String,
+  val createdAt: String = "",
+  val title: String,
+  val url: String = "",
+  val description: String = "",
+  val commentCount: Int = 0,
+  val commentsUrl: String = "",
+  val submitter: String,
+  val tags: List<String> = emptyList(),
+  val comments: List<Comment> = emptyList(),
+  val userIsAuthor: Boolean = false,
+)
+
+class Comment(
+  val shortId: String,
+  val comment: String,
+  val url: String = "",
+  val score: Int = 1,
+  val timestamp: Long,
+  val edited: Boolean = false,
+  val parentComment: String? = null,
+  val user: String = "",
+  val isUpvoted: Boolean = false,
+)
+
+class User(
+  val username: String,
+  val about: String = "",
+  val invitedBy: String? = null,
+  val avatarUrl: String = "",
+  val createdAt: String = "",
+)
+
+class Tag(
+  val tag: String,
+  val description: String,
+  val privileged: Boolean = false,
+  val active: Boolean = true,
+  val category: String = "",
+  val isMedia: Boolean = false,
+  val hotnessMod: Double = 0.0,
+)
+
+@JvmInline value class CSRFToken(val value: String)
+
+class ReplyForm(
+  val authenticityToken: String,
+  val storyId: String,
+  val method: String,
+  val parentCommentShortId: String,
+)

zipline-parser-api/src/commonMain/kotlin/dev/msfjarvis/claw/parser/model/ParserSerializers.kt

@@ -0,0 +1,331 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.model
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.descriptors.PrimitiveKind
+import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.modules.SerializersModule
+
+val ParserSerializersModule: SerializersModule = SerializersModule {
+  contextual(LobstersPost::class, LobstersPostSerializer)
+  contextual(LobstersPostDetails::class, LobstersPostDetailsSerializer)
+  contextual(Comment::class, CommentSerializer)
+  contextual(User::class, UserSerializer)
+  contextual(Tag::class, TagSerializer)
+  contextual(CSRFToken::class, CSRFTokenSerializer)
+  contextual(ReplyForm::class, ReplyFormSerializer)
+}
+
+internal object LobstersPostSerializer : KSerializer<LobstersPost> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor("dev.msfjarvis.claw.parser.model.LobstersPost", PrimitiveKind.STRING)
+
+  override fun serialize(encoder: Encoder, value: LobstersPost) {
+    encoder.encodeString(
+      PacketWriter()
+        .string(value.shortId)
+        .string(value.createdAt)
+        .string(value.title)
+        .string(value.url)
+        .string(value.description)
+        .int(value.commentCount)
+        .string(value.commentsUrl)
+        .string(value.submitter)
+        .boolean(value.userIsAuthor)
+        .stringList(value.tags)
+        .build()
+    )
+  }
+
+  override fun deserialize(decoder: Decoder): LobstersPost {
+    val reader = PacketReader(decoder.decodeString())
+    return LobstersPost(
+      shortId = reader.string(),
+      createdAt = reader.string(),
+      title = reader.string(),
+      url = reader.string(),
+      description = reader.string(),
+      commentCount = reader.int(),
+      commentsUrl = reader.string(),
+      submitter = reader.string(),
+      userIsAuthor = reader.boolean(),
+      tags = reader.stringList(),
+    )
+  }
+}
+
+internal object LobstersPostDetailsSerializer : KSerializer<LobstersPostDetails> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor(
+      "dev.msfjarvis.claw.parser.model.LobstersPostDetails",
+      PrimitiveKind.STRING,
+    )
+
+  override fun serialize(encoder: Encoder, value: LobstersPostDetails) {
+    val writer =
+      PacketWriter()
+        .string(value.shortId)
+        .string(value.createdAt)
+        .string(value.title)
+        .string(value.url)
+        .string(value.description)
+        .int(value.commentCount)
+        .string(value.commentsUrl)
+        .string(value.submitter)
+        .stringList(value.tags)
+        .int(value.comments.size)
+    value.comments.forEach { writer.string(CommentSerializer.toPayload(it)) }
+    encoder.encodeString(writer.boolean(value.userIsAuthor).build())
+  }
+
+  override fun deserialize(decoder: Decoder): LobstersPostDetails {
+    val reader = PacketReader(decoder.decodeString())
+    val shortId = reader.string()
+    val createdAt = reader.string()
+    val title = reader.string()
+    val url = reader.string()
+    val description = reader.string()
+    val commentCount = reader.int()
+    val commentsUrl = reader.string()
+    val submitter = reader.string()
+    val tags = reader.stringList()
+    val comments =
+      List(reader.int()) {
+        val reader = PacketReader(reader.string())
+        Comment(
+          shortId = reader.string(),
+          comment = reader.string(),
+          url = reader.string(),
+          score = reader.int(),
+          timestamp = reader.long(),
+          edited = reader.boolean(),
+          parentComment = reader.nullableString(),
+          user = reader.string(),
+          isUpvoted = reader.boolean(),
+        )
+      }
+    return LobstersPostDetails(
+      shortId = shortId,
+      createdAt = createdAt,
+      title = title,
+      url = url,
+      description = description,
+      commentCount = commentCount,
+      commentsUrl = commentsUrl,
+      submitter = submitter,
+      tags = tags,
+      comments = comments,
+      userIsAuthor = reader.boolean(),
+    )
+  }
+}
+
+internal object CommentSerializer : KSerializer<Comment> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor("dev.msfjarvis.claw.parser.model.Comment", PrimitiveKind.STRING)
+
+  override fun serialize(encoder: Encoder, value: Comment) {
+    encoder.encodeString(toPayload(value))
+  }
+
+  override fun deserialize(decoder: Decoder): Comment {
+    val reader = PacketReader(decoder.decodeString())
+    return Comment(
+      shortId = reader.string(),
+      comment = reader.string(),
+      url = reader.string(),
+      score = reader.int(),
+      timestamp = reader.long(),
+      edited = reader.boolean(),
+      parentComment = reader.nullableString(),
+      user = reader.string(),
+      isUpvoted = reader.boolean(),
+    )
+  }
+
+  fun toPayload(value: Comment): String =
+    PacketWriter()
+      .string(value.shortId)
+      .string(value.comment)
+      .string(value.url)
+      .int(value.score)
+      .long(value.timestamp)
+      .boolean(value.edited)
+      .nullableString(value.parentComment)
+      .string(value.user)
+      .boolean(value.isUpvoted)
+      .build()
+}
+
+internal object UserSerializer : KSerializer<User> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor("dev.msfjarvis.claw.parser.model.User", PrimitiveKind.STRING)
+
+  override fun serialize(encoder: Encoder, value: User) {
+    encoder.encodeString(
+      PacketWriter()
+        .string(value.username)
+        .string(value.about)
+        .nullableString(value.invitedBy)
+        .string(value.avatarUrl)
+        .string(value.createdAt)
+        .build()
+    )
+  }
+
+  override fun deserialize(decoder: Decoder): User {
+    val reader = PacketReader(decoder.decodeString())
+    return User(
+      username = reader.string(),
+      about = reader.string(),
+      invitedBy = reader.nullableString(),
+      avatarUrl = reader.string(),
+      createdAt = reader.string(),
+    )
+  }
+}
+
+internal object TagSerializer : KSerializer<Tag> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor("dev.msfjarvis.claw.parser.model.Tag", PrimitiveKind.STRING)
+
+  override fun serialize(encoder: Encoder, value: Tag) {
+    encoder.encodeString(
+      PacketWriter()
+        .string(value.tag)
+        .string(value.description)
+        .boolean(value.privileged)
+        .boolean(value.active)
+        .string(value.category)
+        .boolean(value.isMedia)
+        .double(value.hotnessMod)
+        .build()
+    )
+  }
+
+  override fun deserialize(decoder: Decoder): Tag {
+    val reader = PacketReader(decoder.decodeString())
+    return Tag(
+      tag = reader.string(),
+      description = reader.string(),
+      privileged = reader.boolean(),
+      active = reader.boolean(),
+      category = reader.string(),
+      isMedia = reader.boolean(),
+      hotnessMod = reader.double(),
+    )
+  }
+}
+
+internal object CSRFTokenSerializer : KSerializer<CSRFToken> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor("dev.msfjarvis.claw.parser.model.CSRFToken", PrimitiveKind.STRING)
+
+  override fun serialize(encoder: Encoder, value: CSRFToken) {
+    encoder.encodeString(value.value)
+  }
+
+  override fun deserialize(decoder: Decoder): CSRFToken = CSRFToken(decoder.decodeString())
+}
+
+internal object ReplyFormSerializer : KSerializer<ReplyForm> {
+  override val descriptor: SerialDescriptor =
+    PrimitiveSerialDescriptor("dev.msfjarvis.claw.parser.model.ReplyForm", PrimitiveKind.STRING)
+
+  override fun serialize(encoder: Encoder, value: ReplyForm) {
+    encoder.encodeString(
+      PacketWriter()
+        .string(value.authenticityToken)
+        .string(value.storyId)
+        .string(value.method)
+        .string(value.parentCommentShortId)
+        .build()
+    )
+  }
+
+  override fun deserialize(decoder: Decoder): ReplyForm {
+    val reader = PacketReader(decoder.decodeString())
+    return ReplyForm(
+      authenticityToken = reader.string(),
+      storyId = reader.string(),
+      method = reader.string(),
+      parentCommentShortId = reader.string(),
+    )
+  }
+}
+
+private class PacketWriter {
+  private val parts = StringBuilder()
+
+  fun string(value: String): PacketWriter {
+    parts.append(value.length).append(':').append(value)
+    return this
+  }
+
+  fun nullableString(value: String?): PacketWriter {
+    parts.append(value?.length ?: -1).append(':')
+    if (value != null) parts.append(value)
+    return this
+  }
+
+  fun int(value: Int): PacketWriter = string(value.toString())
+
+  fun long(value: Long): PacketWriter = string(value.toString())
+
+  fun double(value: Double): PacketWriter = string(value.toString())
+
+  fun boolean(value: Boolean): PacketWriter = string(if (value) "1" else "0")
+
+  fun stringList(values: List<String>): PacketWriter {
+    int(values.size)
+    values.forEach(::string)
+    return this
+  }
+
+  fun build(): String = parts.toString()
+}
+
+private class PacketReader(private val payload: String) {
+  private var index = 0
+
+  fun string(): String {
+    val separatorIndex = payload.indexOf(':', index)
+    if (separatorIndex == -1) throw SerializationException("Malformed payload")
+    val length = payload.substring(index, separatorIndex).toInt()
+    index = separatorIndex + 1
+    if (length < 0) throw SerializationException("Expected non-null string")
+    val endIndex = index + length
+    if (endIndex > payload.length) throw SerializationException("Malformed payload")
+    return payload.substring(index, endIndex).also { index = endIndex }
+  }
+
+  fun nullableString(): String? {
+    val separatorIndex = payload.indexOf(':', index)
+    if (separatorIndex == -1) throw SerializationException("Malformed payload")
+    val length = payload.substring(index, separatorIndex).toInt()
+    index = separatorIndex + 1
+    if (length < 0) return null
+    val endIndex = index + length
+    if (endIndex > payload.length) throw SerializationException("Malformed payload")
+    return payload.substring(index, endIndex).also { index = endIndex }
+  }
+
+  fun int(): Int = string().toInt()
+
+  fun long(): Long = string().toLong()
+
+  fun double(): Double = string().toDouble()
+
+  fun boolean(): Boolean = string() == "1"
+
+  fun stringList(): List<String> = List(int()) { string() }
+}

zipline-parser/api/zipline-api.toml

@@ -0,0 +1,27 @@
+[dev.msfjarvis.claw.parser.LobstersParserService]
+
+functions = [
+  # fun close(): kotlin.Unit
+  "moYx+T3e",
+
+  # fun parseCsrfToken(kotlin.String): dev.msfjarvis.claw.parser.model.CSRFToken
+  "XO8wcclo",
+
+  # fun parsePostDetails(kotlin.String): dev.msfjarvis.claw.parser.model.LobstersPostDetails
+  "K6Im4ZQe",
+
+  # fun parsePostsPage(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.parser.model.LobstersPost>
+  "qs/WBwxE",
+
+  # fun parseReplyForm(kotlin.String): dev.msfjarvis.claw.parser.model.ReplyForm
+  "Gv0fENP3",
+
+  # fun parseSearchResults(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.parser.model.LobstersPost>
+  "MYlj2kop",
+
+  # fun parseTagsPage(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.parser.model.Tag>
+  "fR8RFEx1",
+
+  # fun parseUser(kotlin.String): dev.msfjarvis.claw.parser.model.User
+  "K2X252uP",
+]

zipline-parser/build.gradle.kts

@@ -0,0 +1,53 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+import app.cash.zipline.loader.SignatureAlgorithmId
+
+plugins {
+  kotlin("multiplatform")
+  alias(libs.plugins.zipline)
+}
+
+kotlin {
+  jvm()
+  js {
+    nodejs()
+    binaries.executable()
+  }
+
+  sourceSets {
+    commonMain {
+      dependencies {
+        api(projects.ziplineParserApi)
+        implementation(libs.ksoup)
+      }
+    }
+    jvmTest {
+      dependencies {
+        implementation(kotlin("test"))
+      }
+      resources.srcDir("../api/src/test/resources")
+    }
+    jsTest {
+      dependencies {
+        implementation(kotlin("test"))
+      }
+    }
+  }
+}
+
+zipline {
+  mainFunction.set("dev.msfjarvis.claw.parser.launchZipline")
+  val signingKey = providers.environmentVariable("ZIPLINE_SIGNING_KEY").orNull
+  if (signingKey != null) {
+    signingKeys {
+      create("key0") {
+        algorithmId = SignatureAlgorithmId.Ed25519
+        privateKeyHex = signingKey
+      }
+    }
+  }
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/LobstersParserService.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser
+
+import app.cash.zipline.ZiplineService
+import dev.msfjarvis.claw.parser.model.CSRFToken
+import dev.msfjarvis.claw.parser.model.LobstersPost
+import dev.msfjarvis.claw.parser.model.LobstersPostDetails
+import dev.msfjarvis.claw.parser.model.ReplyForm
+import dev.msfjarvis.claw.parser.model.Tag
+import dev.msfjarvis.claw.parser.model.User
+
+interface LobstersParserService : ZiplineService {
+  fun parsePostsPage(html: String): List<LobstersPost>
+
+  fun parsePostDetails(html: String): LobstersPostDetails
+
+  fun parseUser(html: String): User
+
+  fun parseTagsPage(html: String): List<Tag>
+
+  fun parseSearchResults(html: String): List<LobstersPost>
+
+  fun parseCsrfToken(html: String): CSRFToken
+
+  fun parseReplyForm(html: String): ReplyForm
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/LobstersParserServiceImpl.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser
+
+import dev.msfjarvis.claw.parser.internal.parseCsrfToken as parseCsrfTokenHtml
+import dev.msfjarvis.claw.parser.internal.parsePostDetails as parsePostDetailsHtml
+import dev.msfjarvis.claw.parser.internal.parsePostsPage as parsePostsPageHtml
+import dev.msfjarvis.claw.parser.internal.parseReplyForm as parseReplyFormHtml
+import dev.msfjarvis.claw.parser.internal.parseSearchResults as parseSearchResultsHtml
+import dev.msfjarvis.claw.parser.internal.parseTagsPage as parseTagsPageHtml
+import dev.msfjarvis.claw.parser.internal.parseUser as parseUserHtml
+import dev.msfjarvis.claw.parser.model.CSRFToken
+import dev.msfjarvis.claw.parser.model.LobstersPost
+import dev.msfjarvis.claw.parser.model.LobstersPostDetails
+import dev.msfjarvis.claw.parser.model.ReplyForm
+import dev.msfjarvis.claw.parser.model.Tag
+import dev.msfjarvis.claw.parser.model.User
+
+class LobstersParserServiceImpl : LobstersParserService {
+  override fun parsePostsPage(html: String): List<LobstersPost> = parsePostsPageHtml(html)
+
+  override fun parsePostDetails(html: String): LobstersPostDetails = parsePostDetailsHtml(html)
+
+  override fun parseUser(html: String): User = parseUserHtml(html)
+
+  override fun parseTagsPage(html: String): List<Tag> = parseTagsPageHtml(html)
+
+  override fun parseSearchResults(html: String): List<LobstersPost> = parseSearchResultsHtml(html)
+
+  override fun parseCsrfToken(html: String): CSRFToken = parseCsrfTokenHtml(html)
+
+  override fun parseReplyForm(html: String): ReplyForm = parseReplyFormHtml(html)
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/AuthParsers.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.Ksoup
+import dev.msfjarvis.claw.parser.model.CSRFToken
+import dev.msfjarvis.claw.parser.model.ReplyForm
+
+internal fun parseCsrfToken(html: String): CSRFToken {
+  val token =
+    Ksoup.parse(html, baseUri = BASE_URL)
+      .select("meta[name=csrf-token]")
+      .firstOrNull()
+      ?.attr("content")
+      .orEmpty()
+  return CSRFToken(token)
+}
+
+internal fun parseReplyForm(html: String): ReplyForm {
+  val document = Ksoup.parse(html, baseUri = BASE_URL)
+  fun inputValue(name: String): String =
+    document.select("input[name=$name]").firstOrNull()?.attr("value").orEmpty()
+
+  return ReplyForm(
+    authenticityToken = inputValue("authenticity_token"),
+    storyId = inputValue("story_id"),
+    method = inputValue("_method"),
+    parentCommentShortId = inputValue("parent_comment_short_id"),
+  )
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/CommentParsers.kt

@@ -0,0 +1,76 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.nodes.Element
+import dev.msfjarvis.claw.parser.model.Comment
+
+internal fun parseComments(root: Element): List<Comment> {
+  val seen = mutableSetOf<String>()
+  val workStack = ArrayDeque<Pair<Element, String?>>()
+  root.select("ol.comments > li.comments_subtree").asReversed().forEach { subtree ->
+    workStack.addLast(subtree to null)
+  }
+  return buildList {
+    while (workStack.isNotEmpty()) {
+      val (subtree, parentComment) = workStack.removeLast()
+      val commentElement =
+        if (subtree.`is`("div.comment")) subtree
+        else subtree.children().firstOrNull { it.`is`("div.comment") } ?: continue
+      val shortId = commentElement.attr("data-shortid")
+      if (!seen.add(shortId)) continue
+      val comment = commentElement.toComment(parentComment)
+      add(comment)
+      val childContainer = if (subtree.`is`("div.comment")) subtree.parent() ?: subtree else subtree
+      childContainer.children().asReversed().forEach { childList ->
+        if (!childList.`is`("ol.comments")) return@forEach
+        childList.children().asReversed().forEach { child ->
+          if (!child.`is`("li.comments_subtree")) return@forEach
+          workStack.addLast(child to comment.shortId)
+        }
+      }
+    }
+  }
+}
+
+private fun Element.toComment(parentComment: String?): Comment {
+  val byline = selectFirst("div.byline")
+  val timestamp = byline?.selectFirst("a[href^=/c/] time")?.attr("data-at-unix").orEmpty()
+  val isEdited = byline?.text()?.contains("edited") == true
+  return Comment(
+    shortId = attr("data-shortid"),
+    comment = selectFirst("div.comment_text")?.html().orEmpty(),
+    url = selectFirst("div.byline a[href^=/c/]")?.absUrl("href").orEmpty(),
+    score =
+      children()
+        .firstOrNull { it.hasClass("voters") }
+        ?.children()
+        ?.firstOrNull { it.hasClass("upvoter") }
+        ?.text()
+        ?.trim()
+        ?.takeUnless { it == "~" }
+        ?.toIntOrNull() ?: 1,
+    timestamp = timestamp.toEpochSeconds(),
+    edited = isEdited,
+    parentComment = parentComment,
+    user =
+      getElementsByClass("byline")
+        .flatMap { it.getElementsByTag("a") }
+        .firstOrNull { it.attr("href").contains("/~") && it.text().isNotBlank() }
+        ?.text()
+        .orEmpty(),
+    isUpvoted = classNames().contains("upvoted"),
+  )
+}
+
+private fun String.toEpochSeconds(): Long {
+  return when {
+    isBlank() -> 0L
+    all(Char::isDigit) -> toLong()
+    else -> 0L
+  }
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/ParserConstants.kt

@@ -4,6 +4,6 @@
  * license that can be found in the LICENSE file or at
  * https://opensource.org/licenses/MIT.
  */
-package dev.msfjarvis.claw.api
+package dev.msfjarvis.claw.parser.internal
 
-@JvmInline value class CSRFToken(val value: String)
+internal const val BASE_URL = "https://lobste.rs"

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/PostDetailsParsers.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.Ksoup
+import dev.msfjarvis.claw.parser.model.LobstersPostDetails
+
+private val commentCountRegex by lazy(LazyThreadSafetyMode.NONE) { "\\d+".toRegex() }
+private const val STORY_SELECTOR = "ol.stories > li.story"
+private const val SUBMITTER_SELECTOR =
+  "ol.stories > li.story div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])"
+
+internal fun parsePostDetails(html: String): LobstersPostDetails {
+  val document = Ksoup.parse(html, baseUri = BASE_URL)
+  val storyElement = document.select(STORY_SELECTOR)
+  val timestampElement = document.select("$STORY_SELECTOR div.byline > time")
+  val titleElement = document.select("$STORY_SELECTOR span.link.h-cite > a")
+  val commentsElement = document.select("$STORY_SELECTOR span.comments_label a")
+  val submitterElement = document.select(SUBMITTER_SELECTOR)
+  val tags = document.select("$STORY_SELECTOR span.tags > a").map { it.text() }
+  return LobstersPostDetails(
+    shortId = storyElement.attr("data-shortid"),
+    createdAt = normalizeCreatedAt(timestampElement.attr("data-at-unix")),
+    title = titleElement.text(),
+    url = titleElement.attr("abs:href"),
+    description = document.select("div.story_content div.story_text").html(),
+    commentCount = commentCountRegex.find(commentsElement.text())?.value?.toInt() ?: 0,
+    commentsUrl = commentsElement.attr("abs:href"),
+    submitter = submitterElement.text(),
+    tags = tags,
+    comments = parseComments(document),
+    userIsAuthor = submitterElement.attr("class").split(' ').contains("user_is_author"),
+  )
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/PostParsers.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.nodes.Element
+import dev.msfjarvis.claw.parser.model.LobstersPost
+
+private val commentCountRegex by lazy(LazyThreadSafetyMode.NONE) { "\\d+".toRegex() }
+
+internal fun parsePostsPage(html: String): List<LobstersPost> {
+  return Ksoup.parse(html, baseUri = BASE_URL).select("li.story").map(::parsePost)
+}
+
+private fun parsePost(element: Element): LobstersPost {
+  val titleElement = element.select("> div.story_liner span.link.h-cite > a")
+  val commentElement = element.select("> div.story_liner span.comments_label a")
+  val submitterElement =
+    element.select(
+      "> div.story_liner div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])"
+    )
+
+  val timestampElement = element.select("> div.story_liner div.byline > time")
+  val descriptionElement = element.select("> div.story_liner a.description_present")
+  val tags = element.select("> div.story_liner span.tags > a").map(Element::text)
+
+  return LobstersPost(
+    shortId = element.attr("data-shortid"),
+    createdAt = normalizeCreatedAt(timestampElement.attr("data-at-unix")),
+    title = titleElement.text(),
+    url = titleElement.attr("abs:href"),
+    description = descriptionElement.attr("title"),
+    commentCount = commentCountRegex.find(commentElement.text())?.value?.toInt() ?: 0,
+    commentsUrl = commentElement.attr("abs:href"),
+    submitter = submitterElement.text(),
+    userIsAuthor = submitterElement.attr("class").split(' ').contains("user_is_author"),
+    tags = tags,
+  )
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/SearchParsers.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.nodes.Element
+import dev.msfjarvis.claw.parser.model.LobstersPost
+
+internal fun parseSearchResults(html: String): List<LobstersPost> {
+  return Ksoup.parse(html, baseUri = BASE_URL)
+    .select("div.story_liner.h-entry")
+    .map(::parseSearchPost)
+}
+
+private fun parseSearchPost(elem: Element): LobstersPost {
+  val parent = elem.parent() ?: error("$elem must have a parent")
+  val titleElement = elem.select("span.link.h-cite > a")
+  val linkElement = elem.select("span.comments_label a")
+  val commentCount = linkElement.text().trimStart().substringBefore(" ").toIntOrNull() ?: 0
+  return LobstersPost(
+    shortId = parent.attr("data-shortid"),
+    title = titleElement.text(),
+    url = titleElement.attr("abs:href"),
+    commentCount = commentCount,
+    commentsUrl = BASE_URL + linkElement.attr("href"),
+    tags = elem.select("span.tags > a").map(Element::text),
+    submitter =
+      elem.select("div.byline > a[href^=/~]:not([tabindex]):not([aria-hidden=true])").text(),
+    createdAt = "",
+    description = "",
+    userIsAuthor =
+      (elem.select("div.byline > span").firstOrNull()?.text() ?: "").contains(
+        "authored",
+        ignoreCase = true,
+      ),
+  )
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/TagParsers.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.Ksoup
+import com.fleeksoft.ksoup.nodes.Element
+import dev.msfjarvis.claw.parser.model.Tag
+
+internal fun parseTagsPage(html: String): List<Tag> {
+  return Ksoup.parse(html, baseUri = BASE_URL).select("ol.category_tags > li").map(::parseTag)
+}
+
+private fun parseTag(element: Element): Tag {
+  val tagElement = element.select("> a.tag")
+  val descriptionElement = element.select("> span:not(.byline)")
+  return Tag(
+    tag = tagElement.text(),
+    description = descriptionElement.text(),
+    privileged = element.attr("data-privileged").toBoolean(),
+    active = !descriptionElement.hasClass("inactive_tag"),
+    category = element.attr("data-category"),
+    isMedia = tagElement.hasClass("tag_is_media"),
+    hotnessMod = element.attr("data-hotness-mod").toDoubleOrNull() ?: 0.0,
+  )
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/TimeParsers.kt

@@ -0,0 +1,18 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import kotlin.time.Instant
+
+internal fun normalizeCreatedAt(value: String): String {
+  return when {
+    value.isBlank() -> ""
+    value.all(Char::isDigit) -> Instant.fromEpochSeconds(value.toLong()).toString()
+    runCatching { Instant.parse(value) }.isSuccess -> value
+    else -> value
+  }
+}

zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/UserParsers.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser.internal
+
+import com.fleeksoft.ksoup.Ksoup
+import dev.msfjarvis.claw.parser.model.User
+
+internal fun parseUser(html: String): User {
+  val document = Ksoup.parse(html, baseUri = BASE_URL)
+  return User(
+    username = document.select("#inside > h1").text(),
+    about = document.select("section.profile .shorten_first_p").html(),
+    invitedBy =
+      document
+        .select("section.profile .labelled_grid label:contains(Joined) + span a[href^=/~/]")
+        .text()
+        .ifBlank { null },
+    avatarUrl = document.select("section.profile #gravatar img.avatar").attr("abs:src"),
+    createdAt =
+      document.select("section.profile .labelled_grid label:contains(Joined) + span time").text(),
+  )
+}

zipline-parser/src/jsMain/kotlin/dev/msfjarvis/claw/parser/ParserEntryPoint.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser
+
+import app.cash.zipline.Zipline
+import dev.msfjarvis.claw.parser.model.ParserSerializersModule
+import kotlin.js.ExperimentalJsExport
+import kotlin.js.JsExport
+
+@OptIn(ExperimentalJsExport::class)
+@JsExport
+fun launchZipline() {
+  Zipline.get(ParserSerializersModule)
+    .bind<LobstersParserService>(
+      name = "LobstersParserService",
+      instance = LobstersParserServiceImpl(),
+    )
+}

zipline-parser/src/jvmTest/kotlin/dev/msfjarvis/claw/parser/RealLobstersHtmlParserJvmTest.kt

@@ -0,0 +1,131 @@
+/*
+ * Copyright © Harsh Shandilya.
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+package dev.msfjarvis.claw.parser
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+
+class RealLobstersHtmlParserJvmTest {
+  @Test
+  fun parsesPostsPage() {
+    val html = checkNotNull(javaClass.classLoader.getResource("hottest_page.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    val posts = service.parsePostsPage(html)
+
+    assertTrue(posts.isNotEmpty())
+  }
+
+  @Test
+  fun parsesSearchResults() {
+    val html =
+      checkNotNull(javaClass.classLoader.getResource("search_chatgpt_page.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    assertTrue(service.parseSearchResults(html).isNotEmpty())
+  }
+
+  @Test
+  fun searchResultsUseAbsoluteStoryUrls() {
+    val html =
+      """
+      <div class="story_liner h-entry">
+        <span class="link h-cite"><a href="/s/abcd12/local_story">Local story</a></span>
+        <span class="comments_label"><a href="/s/abcd12/local_story">1 comment</a></span>
+        <span class="tags"><a>meta</a></span>
+        <div class="byline"><a href="/~someone">someone</a></div>
+      </div>
+      """
+        .trimIndent()
+    val service = LobstersParserServiceImpl()
+
+    val result = service.parseSearchResults(html).single()
+
+    assertEquals("https://lobste.rs/s/abcd12/local_story", result.url)
+  }
+
+  @Test
+  fun parsesUserPage() {
+    val html = checkNotNull(javaClass.classLoader.getResource("msfjarvis.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    assertTrue(service.parseUser(html).username.isNotBlank())
+  }
+
+  @Test
+  fun parsesTagsPage() {
+    val html = checkNotNull(javaClass.classLoader.getResource("tags.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    assertTrue(service.parseTagsPage(html).isNotEmpty())
+  }
+
+  @Test
+  fun parsesCsrfToken() {
+    val html = checkNotNull(javaClass.classLoader.getResource("csrf_page.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    assertTrue(service.parseCsrfToken(html).value.isNotBlank())
+  }
+
+  @Test
+  fun parsesReplyForm() {
+    val html = checkNotNull(javaClass.classLoader.getResource("reply_form.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    assertTrue(service.parseReplyForm(html).authenticityToken.isNotBlank())
+  }
+
+  @Test
+  fun parsesPostDetails() {
+    val html =
+      checkNotNull(javaClass.classLoader.getResource("post_details_tdfoqh.html")).readText()
+    val service = LobstersParserServiceImpl()
+
+    val details = service.parsePostDetails(html)
+
+    assertTrue(details.title.isNotBlank())
+    assertTrue(details.comments.isNotEmpty())
+  }
+
+  @Test
+  fun commentsWithoutVisibleUpvoterCountHaveOnePointFromTheAuthor() {
+    val html =
+      """
+      <ol class="stories">
+        <li class="story" data-shortid="story1">
+          <span class="link h-cite"><a href="/s/story1/test">Test story</a></span>
+          <div class="byline">
+            <a class="u-author" href="/~/submitter">submitter</a>
+            <time data-at-unix="1710000000"></time>
+          </div>
+        </li>
+      </ol>
+      <ol class="comments">
+        <li class="comments_subtree">
+          <div class="comment" data-shortid="abc123">
+            <div class="voters"></div>
+            <div class="details">
+              <div class="byline">
+                <a href="/~/author">author</a>
+                <a href="/c/abc123"><time data-at-unix="1710000000"></time></a>
+              </div>
+              <div class="comment_text"><p>Hello</p></div>
+            </div>
+          </div>
+        </li>
+      </ol>
+      """
+        .trimIndent()
+    val service = LobstersParserServiceImpl()
+
+    val details = service.parsePostDetails(html)
+
+    kotlin.test.assertEquals(1, details.comments.single().score)
+  }
+}