zapstore-mirror/dev.msfjarvis.claw.android
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: extract zipline parser api module

Browse Source
This commit is contained in:
Harsh Shandilya
2026-06-08 17:12:56 +05:30
parent 7b80e1726a
commit b2adb28275
16 changed files with 232 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/main/kotlin/dev/msfjarvis/claw/api/converters/ParserModelMappings.kt
+87
View File
@@ -0,0 +1,87 @@
/*
* Copyright © Harsh Shandilya.
* Use of this source code is governed by an MIT-style
* license that can be found in the LICENSE file or at
* https://opensource.org/licenses/MIT.
*/
package dev.msfjarvis.claw.api.converters
import dev.msfjarvis.claw.model.CSRFToken
import dev.msfjarvis.claw.model.Comment
import dev.msfjarvis.claw.model.LobstersPost
import dev.msfjarvis.claw.model.LobstersPostDetails
import dev.msfjarvis.claw.model.ReplyForm
import dev.msfjarvis.claw.model.Tag
import dev.msfjarvis.claw.model.User
internal fun dev.msfjarvis.claw.parser.model.CSRFToken.toModel(): CSRFToken = CSRFToken(value)
internal fun dev.msfjarvis.claw.parser.model.LobstersPost.toModel(): LobstersPost =
LobstersPost(
shortId = shortId,
createdAt = createdAt,
title = title,
url = url,
description = description,
commentCount = commentCount,
commentsUrl = commentsUrl,
submitter = submitter,
userIsAuthor = userIsAuthor,
tags = tags,
)
internal fun dev.msfjarvis.claw.parser.model.LobstersPostDetails.toModel(): LobstersPostDetails =
LobstersPostDetails(
shortId = shortId,
createdAt = createdAt,
title = title,
url = url,
description = description,
commentCount = commentCount,
commentsUrl = commentsUrl,
submitter = submitter,
tags = tags,
comments = comments.map { it.toModel() },
userIsAuthor = userIsAuthor,
)
internal fun dev.msfjarvis.claw.parser.model.Comment.toModel(): Comment =
Comment(
shortId = shortId,
comment = comment,
url = url,
score = score,
timestamp = timestamp,
edited = edited,
parentComment = parentComment,
user = user,
isUpvoted = isUpvoted,
)
internal fun dev.msfjarvis.claw.parser.model.ReplyForm.toModel(): ReplyForm =
ReplyForm(
authenticityToken = authenticityToken,
storyId = storyId,
method = method,
parentCommentShortId = parentCommentShortId,
)
internal fun dev.msfjarvis.claw.parser.model.Tag.toModel(): Tag =
Tag(
tag = tag,
description = description,
privileged = privileged,
active = active,
category = category,
isMedia = isMedia,
hotnessMod = hotnessMod,
)
internal fun dev.msfjarvis.claw.parser.model.User.toModel(): User =
User(
username = username,
about = about,
invitedBy = invitedBy,
avatarUrl = avatarUrl,
createdAt = createdAt,
)
api/src/main/kotlin/dev/msfjarvis/claw/api/converters/ZiplineHtmlConverterFactory.kt
+9 -7
View File
@@ -34,22 +34,24 @@ class ZiplineHtmlConverterFactory(private val parserClient: LobstersParserClient
return when {
normalizedType.isListOf(LobstersPost::class.java) &&
getPath?.startsWith("/search") == true -> {
HtmlConverter { html -> parserClient.service().parseSearchResults(html) }
HtmlConverter { html ->
parserClient.service().parseSearchResults(html).map { it.toModel() }
}
}
normalizedType.isListOf(LobstersPost::class.java) -> {
HtmlConverter { html -> parserClient.service().parsePostsPage(html) }
HtmlConverter { html -> parserClient.service().parsePostsPage(html).map { it.toModel() } }
}
normalizedType.isListOf(Tag::class.java) -> {
HtmlConverter { html -> parserClient.service().parseTagsPage(html) }
HtmlConverter { html -> parserClient.service().parseTagsPage(html).map { it.toModel() } }
}
normalizedType == CSRFToken::class.java ->
HtmlConverter { html -> parserClient.service().parseCsrfToken(html) }
HtmlConverter { html -> parserClient.service().parseCsrfToken(html).toModel() }
normalizedType == LobstersPostDetails::class.java ->
HtmlConverter { html -> parserClient.service().parsePostDetails(html) }
HtmlConverter { html -> parserClient.service().parsePostDetails(html).toModel() }
normalizedType == ReplyForm::class.java ->
HtmlConverter { html -> parserClient.service().parseReplyForm(html) }
HtmlConverter { html -> parserClient.service().parseReplyForm(html).toModel() }
normalizedType == User::class.java ->
HtmlConverter { html -> parserClient.service().parseUser(html) }
HtmlConverter { html -> parserClient.service().parseUser(html).toModel() }
else -> null
}
}
settings.gradle.kts
+1
View File
@@ -163,4 +163,5 @@ include(
"database:impl",
"model",
"zipline-parser",
"zipline-parser-api",
)
zipline-parser-api/build.gradle.kts
+24
View File
@@ -0,0 +1,24 @@
/*
* Copyright © Harsh Shandilya.
* Use of this source code is governed by an MIT-style
* license that can be found in the LICENSE file or at
* https://opensource.org/licenses/MIT.
*/
plugins {
kotlin("multiplatform")
}
kotlin {
jvm()
js {
browser()
}
sourceSets {
commonMain {
dependencies {
api(libs.zipline)
}
}
}
}
zipline-parser-api/src/commonMain/kotlin/dev/msfjarvis/claw/parser/model/ParserModels.kt
+76
View File
@@ -0,0 +1,76 @@
/*
* Copyright © Harsh Shandilya.
* Use of this source code is governed by an MIT-style
* license that can be found in the LICENSE file or at
* https://opensource.org/licenses/MIT.
*/
package dev.msfjarvis.claw.parser.model
import kotlin.jvm.JvmInline
import kotlin.time.Instant
data class LobstersPost(
val shortId: String,
val createdAt: String = "",
val title: String,
val url: String = "",
val description: String = "",
val commentCount: Int = 0,
val commentsUrl: String = "",
val submitter: String,
val userIsAuthor: Boolean = false,
val tags: List<String> = emptyList(),
)
data class LobstersPostDetails(
val shortId: String,
val createdAt: String = "",
val title: String,
val url: String = "",
val description: String = "",
val commentCount: Int = 0,
val commentsUrl: String = "",
val submitter: String,
val tags: List<String> = emptyList(),
val comments: List<Comment> = emptyList(),
val userIsAuthor: Boolean = false,
)
data class Comment(
val shortId: String,
val comment: String,
val url: String = "",
val score: Int = 1,
val timestamp: Instant,
val edited: Boolean = false,
val parentComment: String? = null,
val user: String = "",
val isUpvoted: Boolean = false,
)
data class User(
val username: String,
val about: String = "",
val invitedBy: String? = null,
val avatarUrl: String = "",
val createdAt: String = "",
)
data class Tag(
val tag: String,
val description: String,
val privileged: Boolean = false,
val active: Boolean = true,
val category: String = "",
val isMedia: Boolean = false,
val hotnessMod: Double = 0.0,
)
@JvmInline value class CSRFToken(val value: String)
data class ReplyForm(
val authenticityToken: String,
val storyId: String,
val method: String,
val parentCommentShortId: String,
)
zipline-parser/api/zipline-api.toml
+14 -14
View File
@@ -4,24 +4,24 @@ functions = [
# fun close(): kotlin.Unit
"moYx+T3e",
# fun parseCsrfToken(kotlin.String): dev.msfjarvis.claw.model.CSRFToken
"gkMJS2GI",
# fun parseCsrfToken(kotlin.String): dev.msfjarvis.claw.parser.model.CSRFToken
"XO8wcclo",
# fun parsePostDetails(kotlin.String): dev.msfjarvis.claw.model.LobstersPostDetails
"i9+0xMCY",
# fun parsePostDetails(kotlin.String): dev.msfjarvis.claw.parser.model.LobstersPostDetails
"K6Im4ZQe",
# fun parsePostsPage(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.model.LobstersPost>
"zBKz0A5i",
# fun parsePostsPage(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.parser.model.LobstersPost>
"qs/WBwxE",
# fun parseReplyForm(kotlin.String): dev.msfjarvis.claw.model.ReplyForm
"thcq4/sB",
# fun parseReplyForm(kotlin.String): dev.msfjarvis.claw.parser.model.ReplyForm
"Gv0fENP3",
# fun parseSearchResults(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.model.LobstersPost>
"72nr7mKP",
# fun parseSearchResults(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.parser.model.LobstersPost>
"MYlj2kop",
# fun parseTagsPage(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.model.Tag>
"YlLfGEit",
# fun parseTagsPage(kotlin.String): kotlin.collections.List<dev.msfjarvis.claw.parser.model.Tag>
"fR8RFEx1",
# fun parseUser(kotlin.String): dev.msfjarvis.claw.model.User
"6gFIDp9n",
# fun parseUser(kotlin.String): dev.msfjarvis.claw.parser.model.User
"K2X252uP",
]
zipline-parser/build.gradle.kts
+1 -2
View File
@@ -22,9 +22,8 @@ kotlin {
sourceSets {
commonMain {
dependencies {
api(projects.model)
api(projects.ziplineParserApi)
api(libs.kotlinx.serialization.core)
api(libs.zipline)
implementation(libs.ksoup)
}
}
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/LobstersParserService.kt
+6 -6
View File
@@ -7,12 +7,12 @@
package dev.msfjarvis.claw.parser
import app.cash.zipline.ZiplineService
import dev.msfjarvis.claw.model.CSRFToken
import dev.msfjarvis.claw.model.LobstersPost
import dev.msfjarvis.claw.model.LobstersPostDetails
import dev.msfjarvis.claw.model.ReplyForm
import dev.msfjarvis.claw.model.Tag
import dev.msfjarvis.claw.model.User
import dev.msfjarvis.claw.parser.model.CSRFToken
import dev.msfjarvis.claw.parser.model.LobstersPost
import dev.msfjarvis.claw.parser.model.LobstersPostDetails
import dev.msfjarvis.claw.parser.model.ReplyForm
import dev.msfjarvis.claw.parser.model.Tag
import dev.msfjarvis.claw.parser.model.User
interface LobstersParserService : ZiplineService {
fun parsePostsPage(html: String): List<LobstersPost>
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/LobstersParserServiceImpl.kt
+6 -6
View File
@@ -6,12 +6,6 @@
*/
package dev.msfjarvis.claw.parser
import dev.msfjarvis.claw.model.CSRFToken
import dev.msfjarvis.claw.model.LobstersPost
import dev.msfjarvis.claw.model.LobstersPostDetails
import dev.msfjarvis.claw.model.ReplyForm
import dev.msfjarvis.claw.model.Tag
import dev.msfjarvis.claw.model.User
import dev.msfjarvis.claw.parser.internal.parseCsrfToken as parseCsrfTokenHtml
import dev.msfjarvis.claw.parser.internal.parsePostDetails as parsePostDetailsHtml
import dev.msfjarvis.claw.parser.internal.parsePostsPage as parsePostsPageHtml
@@ -19,6 +13,12 @@ import dev.msfjarvis.claw.parser.internal.parseReplyForm as parseReplyFormHtml
import dev.msfjarvis.claw.parser.internal.parseSearchResults as parseSearchResultsHtml
import dev.msfjarvis.claw.parser.internal.parseTagsPage as parseTagsPageHtml
import dev.msfjarvis.claw.parser.internal.parseUser as parseUserHtml
import dev.msfjarvis.claw.parser.model.CSRFToken
import dev.msfjarvis.claw.parser.model.LobstersPost
import dev.msfjarvis.claw.parser.model.LobstersPostDetails
import dev.msfjarvis.claw.parser.model.ReplyForm
import dev.msfjarvis.claw.parser.model.Tag
import dev.msfjarvis.claw.parser.model.User
class LobstersParserServiceImpl : LobstersParserService {
override fun parsePostsPage(html: String): List<LobstersPost> = parsePostsPageHtml(html)
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/AuthParsers.kt
+2 -2
View File
@@ -7,8 +7,8 @@
package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.Ksoup
import dev.msfjarvis.claw.model.CSRFToken
import dev.msfjarvis.claw.model.ReplyForm
import dev.msfjarvis.claw.parser.model.CSRFToken
import dev.msfjarvis.claw.parser.model.ReplyForm
private const val BASE_URL = "https://lobste.rs"
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/CommentParsers.kt
+1 -1
View File
@@ -7,7 +7,7 @@
package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.nodes.Element
import dev.msfjarvis.claw.model.Comment
import dev.msfjarvis.claw.parser.model.Comment
import kotlin.time.Instant
internal fun parseComments(root: Element): List<Comment> {
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/PostDetailsParsers.kt
+1 -1
View File
@@ -7,7 +7,7 @@
package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.Ksoup
import dev.msfjarvis.claw.model.LobstersPostDetails
import dev.msfjarvis.claw.parser.model.LobstersPostDetails
private const val BASE_URL = "https://lobste.rs"
private val commentCountRegex = "\\d+".toRegex()
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/PostParsers.kt
+1 -1
View File
@@ -8,7 +8,7 @@ package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.Ksoup
import com.fleeksoft.ksoup.nodes.Element
import dev.msfjarvis.claw.model.LobstersPost
import dev.msfjarvis.claw.parser.model.LobstersPost
private const val BASE_URL = "https://lobste.rs"
private val commentCountRegex = "\\d+".toRegex()
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/SearchParsers.kt
+1 -1
View File
@@ -8,7 +8,7 @@ package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.Ksoup
import com.fleeksoft.ksoup.nodes.Element
import dev.msfjarvis.claw.model.LobstersPost
import dev.msfjarvis.claw.parser.model.LobstersPost
private const val BASE_URL = "https://lobste.rs"
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/TagParsers.kt
+1 -1
View File
@@ -8,7 +8,7 @@ package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.Ksoup
import com.fleeksoft.ksoup.nodes.Element
import dev.msfjarvis.claw.model.Tag
import dev.msfjarvis.claw.parser.model.Tag
private const val BASE_URL = "https://lobste.rs"
zipline-parser/src/commonMain/kotlin/dev/msfjarvis/claw/parser/internal/UserParsers.kt
+1 -1
View File
@@ -7,7 +7,7 @@
package dev.msfjarvis.claw.parser.internal
import com.fleeksoft.ksoup.Ksoup
import dev.msfjarvis.claw.model.User
import dev.msfjarvis.claw.parser.model.User
private const val BASE_URL = "https://lobste.rs"