feat: optimize persisted post state

- Read posts are an append only list, so instead of re-fetching it from the database
  every time it is now cached on repository init and then subsequent `markRead` calls
  just manually update the set and avoid the unnecessary trip through SQLite.
- Since ReadPosts will grow forever in the current set up, I've optimised the table for
  on disk space savings by creating it without ROWID. In a local test with 100K rows the
  difference between no ROWID and the default was about 1.27 MB versus 3.08 MB. I am not
  at 100K yet AFAICT, but this is still just a nice thing to have under my belt.
- Splits out more focused queries for SavedPosts table since re-fetching every column
  to extract a field or every row to do a count is pretty wasteful.
- There is also now an index for SavedPosts with createdAt and shortId fields since they
  are fetched the most often.
- A lot of queries were unnecessarily going through Flow-based APIs when they were used
  for oneshot operations, those are now direct calls.
- Foreign key relations can cause `INSERT OR REPLACE` to accidentally delete a different
  table's row which I don't use yet, but I found the `ON CONFLICT DO UPDATE SET` that
  deals with it so I've adopted it pre-emptively so future me can be confused and dig up
  this commit message.

android/src/main/kotlin/dev/msfjarvis/claw/android/glance/SavedPostsWidget.kt

@@ -41,14 +41,13 @@ import dev.msfjarvis.claw.model.UIPost
 import dev.msfjarvis.claw.model.fromSavedPost
 import kotlinx.collections.immutable.ImmutableList
 import kotlinx.collections.immutable.toImmutableList
-import kotlinx.coroutines.flow.first
 
 class SavedPostsWidget : GlanceAppWidget() {
 
   override suspend fun provideGlance(context: Context, id: GlanceId) {
     val appGraph = (context.applicationContext as ClawApplication).appGraph
     val posts = appGraph.savedPostsRepository.getRecentPosts(50)
-    val postWindow = posts.first().map(UIPost::fromSavedPost).toImmutableList()
+    val postWindow = posts.map { UIPost.fromSavedPost(it) }.toImmutableList()
     provideContent { LobstersGlanceTheme { Content(postWindow) } }
   }
 

android/src/main/kotlin/dev/msfjarvis/claw/android/viewmodel/ClawViewModel.kt

@@ -110,7 +110,7 @@ class ClawViewModel(
       )
       .flow
   val savedPosts = savedPostsRepository.savedPosts.map { it.map(UIPost.Companion::fromSavedPost) }
-  val savedPostsCount = savedPostsRepository.savedPosts.map { it.size.toLong() }
+  val savedPostsCount = savedPostsRepository.savedPostsCount
   val savedPostsByMonth
     get() =
       savedPostsRepository.savedPostsSortedByDate.map { posts ->
@@ -130,10 +130,11 @@ class ClawViewModel(
 
   init {
     viewModelScope.launch {
-      savedPosts.collectLatest { _savedPosts = it.map(UIPost::shortId).toSet() }
+      savedPostsRepository.savedPostIds.collectLatest { _savedPosts = it.toSet() }
     }
     viewModelScope.launch {
-      readPostsRepository.readPosts.collectLatest { _readPosts = it.toSet() }
+      readPostsRepository.initialize()
+      readPostsRepository.readPosts.collectLatest { _readPosts = it }
     }
   }
 

android/src/main/kotlin/dev/msfjarvis/claw/android/viewmodel/DataTransferRepository.kt

@@ -14,7 +14,6 @@ import java.io.InputStream
 import java.io.OutputStream
 import kotlin.time.Instant
 import kotlinx.coroutines.CoroutineDispatcher
-import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.withContext
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.builtins.ListSerializer
@@ -43,7 +42,7 @@ class DataTransferRepository(
   }
 
   suspend fun exportPostsAsJson(output: OutputStream) {
-    val posts = savedPostsRepository.savedPosts.first()
+    val posts = savedPostsRepository.getSavedPosts()
     withContext(ioDispatcher) { json.encodeToStream(serializer, posts, output) }
   }
 
@@ -51,7 +50,7 @@ class DataTransferRepository(
     fun computeTimestamp(post: SavedPost): Long =
       Instant.parse(post.createdAt).toEpochMilliseconds()
 
-    val posts = savedPostsRepository.savedPosts.first()
+    val posts = savedPostsRepository.getSavedPosts()
     val header =
       """
       <!DOCTYPE NETSCAPE-Bookmark-file-1>

android/src/main/kotlin/dev/msfjarvis/claw/android/viewmodel/ReadPostsRepository.kt

@@ -6,24 +6,52 @@
  */
 package dev.msfjarvis.claw.android.viewmodel
 
-import app.cash.sqldelight.coroutines.asFlow
-import app.cash.sqldelight.coroutines.mapToList
 import dev.msfjarvis.claw.core.coroutines.DatabaseReadDispatcher
 import dev.msfjarvis.claw.core.coroutines.DatabaseWriteDispatcher
 import dev.msfjarvis.claw.database.local.ReadPostsQueries
+import dev.zacsweers.metro.AppScope
 import dev.zacsweers.metro.Inject
+import dev.zacsweers.metro.SingleIn
 import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.flow.update
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
 
 @Inject
+@SingleIn(AppScope::class)
 class ReadPostsRepository(
   private val readPostsQueries: ReadPostsQueries,
   @param:DatabaseReadDispatcher private val readDispatcher: CoroutineDispatcher,
   @param:DatabaseWriteDispatcher private val writeDispatcher: CoroutineDispatcher,
 ) {
-  val readPosts = readPostsQueries.selectAllPosts().asFlow().mapToList(readDispatcher)
+  private val initializationMutex = Mutex()
+  private val _readPosts = MutableStateFlow(emptySet<String>())
+  val readPosts: StateFlow<Set<String>> = _readPosts.asStateFlow()
+  private var initialized = false
+
+  suspend fun initialize() {
+    initializationMutex.withLock { initializeLocked() }
+  }
 
   suspend fun markRead(postId: String) {
-    withContext(writeDispatcher) { readPostsQueries.markRead(postId) }
+    initializationMutex.withLock {
+      initializeLocked()
+      withContext(writeDispatcher) {
+        readPostsQueries.markRead(postId).executeAsOneOrNull()?.let { markedId ->
+          _readPosts.update { it + markedId }
+        }
+      }
+    }
+  }
+
+  private suspend fun initializeLocked() {
+    if (initialized) return
+    _readPosts.value =
+      withContext(readDispatcher) { readPostsQueries.selectAllPosts().executeAsList().toSet() }
+    initialized = true
   }
 }

android/src/main/kotlin/dev/msfjarvis/claw/android/viewmodel/SavedPostsRepository.kt

@@ -9,6 +9,7 @@ package dev.msfjarvis.claw.android.viewmodel
 import android.util.Log
 import app.cash.sqldelight.coroutines.asFlow
 import app.cash.sqldelight.coroutines.mapToList
+import app.cash.sqldelight.coroutines.mapToOne
 import dev.msfjarvis.claw.android.BuildConfig
 import dev.msfjarvis.claw.core.coroutines.DatabaseReadDispatcher
 import dev.msfjarvis.claw.core.coroutines.DatabaseWriteDispatcher
@@ -29,26 +30,31 @@ class SavedPostsRepository(
   val savedPosts = savedPostQueries.selectAllPosts().asFlow().mapToList(readDispatcher)
   val savedPostsSortedByDate =
     savedPostQueries.selectAllPostsSortedByDate().asFlow().mapToList(readDispatcher)
+  val savedPostsCount = savedPostQueries.selectCount().asFlow().mapToOne(readDispatcher)
+  val savedPostIds = savedPostQueries.selectPostIds().asFlow().mapToList(readDispatcher)
 
-  fun getPostsFromLastNDays(days: Long) =
-    savedPostQueries.selectPostsFromLastNDays(days.toString()).asFlow().mapToList(readDispatcher)
+  suspend fun getSavedPosts() =
+    withContext(readDispatcher) { savedPostQueries.selectAllPosts().executeAsList() }
 
-  fun getRecentPosts(limit: Long) =
-    savedPostQueries.selectRecentPosts(limit).asFlow().mapToList(readDispatcher)
+  suspend fun getPostIdsFromLastNDays(days: Long) =
+    withContext(readDispatcher) {
+      savedPostQueries.selectPostIdsFromLastNDays(days.toString()).executeAsList()
+    }
+
+  suspend fun getRecentPosts(limit: Long) =
+    withContext(readDispatcher) { savedPostQueries.selectRecentPosts(limit).executeAsList() }
 
   suspend fun toggleSave(post: UIPost) {
-    val exists =
-      withContext(readDispatcher) { savedPostQueries.postExists(post.shortId).executeAsOne() }
-    if (exists) {
-      if (BuildConfig.DEBUG) {
-        Log.d(TAG, "Removing post: ${post.shortId}")
+    withContext(writeDispatcher) {
+      savedPostQueries.transaction {
+        val removed = savedPostQueries.deletePost(post.shortId).executeAsOneOrNull() != null
+        if (removed) {
+          if (BuildConfig.DEBUG) Log.d(TAG, "Removing post: ${post.shortId}")
+        } else {
+          if (BuildConfig.DEBUG) Log.d(TAG, "Saving post: ${post.shortId}")
+          savedPostQueries.insertOrReplacePost(post.toSavedPost())
+        }
       }
-      withContext(writeDispatcher) { savedPostQueries.deletePost(post.shortId) }
-    } else {
-      if (BuildConfig.DEBUG) {
-        Log.d(TAG, "Saving post: ${post.shortId}")
-      }
-      withContext(writeDispatcher) { savedPostQueries.insertOrReplacePost(post.toSavedPost()) }
     }
   }
 

android/src/main/kotlin/dev/msfjarvis/claw/android/work/SavedPostUpdaterWorker.kt

@@ -26,7 +26,6 @@ import dev.zacsweers.metro.ContributesIntoMap
 import dev.zacsweers.metro.binding
 import kotlin.random.Random
 import kotlinx.coroutines.delay
-import kotlinx.coroutines.flow.first
 
 /**
  * WorkManager-backed [CoroutineWorker] that gets all the posts from [SavedPostsRepository] that
@@ -43,7 +42,7 @@ class SavedPostUpdaterWorker(
   private val lobstersApi: LobstersApi,
 ) : CoroutineWorker(context, params) {
   override suspend fun doWork(): Result {
-    val postsToUpdate = savedPostsRepository.getPostsFromLastNDays(DAYS_TO_UPDATE).first()
+    val postsToUpdate = savedPostsRepository.getPostIdsFromLastNDays(DAYS_TO_UPDATE)
 
     if (postsToUpdate.isEmpty()) {
       return Result.success()
@@ -52,7 +51,7 @@ class SavedPostUpdaterWorker(
     val updatedPosts = mutableListOf<SavedPost>()
 
     for ((index, post) in postsToUpdate.withIndex()) {
-      when (val result = lobstersApi.getPostDetails(post.shortId)) {
+      when (val result = lobstersApi.getPostDetails(post)) {
         is Success -> {
           updatedPosts.add(result.value.toSavedPost())
         }

database/core/src/main/sqldelight/dev/msfjarvis/claw/database/local/ReadPosts.sq

@@ -1,15 +1,16 @@
 CREATE TABLE IF NOT EXISTS ReadPosts(
   id TEXT NOT NULL PRIMARY KEY
-);
+) WITHOUT ROWID;
 
 selectAllPosts:
-SELECT *
+SELECT id
 FROM ReadPosts;
 
 markRead:
-INSERT OR IGNORE
-INTO ReadPosts(id)
-VALUES (?);
+INSERT INTO ReadPosts(id)
+VALUES (:id)
+ON CONFLICT(id) DO NOTHING
+RETURNING id;
 
 markUnread:
 DELETE FROM ReadPosts

database/core/src/main/sqldelight/dev/msfjarvis/claw/database/local/SavedPost.sq

@@ -16,19 +16,49 @@ CREATE TABLE IF NOT EXISTS SavedPost(
   userIsAuthor INTEGER AS Boolean NOT NULL DEFAULT 0
 );
 
+CREATE INDEX IF NOT EXISTS SavedPost_createdAt_shortId
+ON SavedPost(datetime(createdAt) DESC, shortId);
+
 insertOrReplacePost:
-INSERT OR REPLACE
-INTO SavedPost
-VALUES ?;
+INSERT INTO SavedPost
+VALUES ?
+ON CONFLICT(shortId) DO UPDATE SET
+  title = excluded.title,
+  url = excluded.url,
+  createdAt = excluded.createdAt,
+  commentCount = excluded.commentCount,
+  commentsUrl = excluded.commentsUrl,
+  submitterName = excluded.submitterName,
+  tags = excluded.tags,
+  description = excluded.description,
+  userIsAuthor = excluded.userIsAuthor
+WHERE title IS NOT excluded.title
+  OR url IS NOT excluded.url
+  OR createdAt IS NOT excluded.createdAt
+  OR commentCount IS NOT excluded.commentCount
+  OR commentsUrl IS NOT excluded.commentsUrl
+  OR submitterName IS NOT excluded.submitterName
+  OR tags IS NOT excluded.tags
+  OR description IS NOT excluded.description
+  OR userIsAuthor IS NOT excluded.userIsAuthor;
 
 selectAllPosts:
-SELECT *
+SELECT shortId, title, url, createdAt, commentCount, commentsUrl, submitterName, tags, description, userIsAuthor
 FROM SavedPost;
 
-selectPostsFromLastNDays:
-SELECT *
+selectPostIds:
+SELECT shortId
+FROM SavedPost;
+
+selectPostIdsFromLastNDays:
+SELECT shortId
 FROM SavedPost
-WHERE datetime(createdAt) >= datetime('now', '-' || ? || ' days');
+WHERE datetime(createdAt) >= datetime('now', '-' || :days || ' days');
+
+selectPostsInIdRange:
+SELECT shortId, title, url, createdAt, commentCount, commentsUrl, submitterName, tags, description, userIsAuthor
+FROM SavedPost
+WHERE shortId >= :lowerId AND shortId <= :upperId;
 
 selectCount:
 SELECT COUNT(*)
@@ -41,22 +71,16 @@ FROM SavedPost;
 deletePost:
 DELETE
 FROM SavedPost
-WHERE shortId = ?;
-
-postExists:
-SELECT EXISTS(
-  SELECT 1
-  FROM SavedPost
-  WHERE shortId = ?
-);
+WHERE shortId = :shortId
+RETURNING shortId;
 
 selectRecentPosts:
-SELECT *
+SELECT shortId, title, url, createdAt, commentCount, commentsUrl, submitterName, tags, description, userIsAuthor
 FROM SavedPost
 ORDER BY datetime(createdAt) DESC
 LIMIT :limit;
 
 selectAllPostsSortedByDate:
-SELECT *
+SELECT shortId, title, url, createdAt, commentCount, commentsUrl, submitterName, tags, description, userIsAuthor
 FROM SavedPost
 ORDER BY datetime(createdAt) DESC;

database/core/src/main/sqldelight/migrations/12.sqm

@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS SavedPost_createdAt_shortId
+ON SavedPost(datetime(createdAt) DESC, shortId);

database/core/src/main/sqldelight/migrations/13.sqm

@@ -0,0 +1,10 @@
+ALTER TABLE ReadPosts RENAME TO ReadPosts_Old;
+
+CREATE TABLE ReadPosts(
+  id TEXT NOT NULL PRIMARY KEY
+) WITHOUT ROWID;
+
+INSERT INTO ReadPosts(id)
+SELECT id FROM ReadPosts_Old;
+
+DROP TABLE ReadPosts_Old;

database/impl/src/test/kotlin/dev/msfjarvis/claw/database/local/ReadPostsQueriesTest.kt

@@ -20,10 +20,13 @@ class ReadPostsQueriesTest {
   }
 
   @Test
-  fun `mark post as read`() {
+  fun `mark post as read returns an ID only once`() {
     val id = UUID.randomUUID().toString()
-    postQueries.markRead(id)
+
+    assertThat(postQueries.markRead(id).executeAsOne()).isEqualTo(id)
+    assertThat(postQueries.markRead(id).executeAsOneOrNull()).isNull()
     assertThat(postQueries.selectAllPosts().executeAsList()).contains(id)
+
     postQueries.markUnread(id)
     assertThat(postQueries.selectAllPosts().executeAsList()).doesNotContain(id)
   }

database/impl/src/test/kotlin/dev/msfjarvis/claw/database/local/SavedPostQueriesTest.kt

@@ -87,7 +87,7 @@ class SavedPostQueriesTest {
     posts.forEach { postQueries.insertOrReplacePost(it) }
 
     // Delete 2nd post
-    postQueries.deletePost("test_id_2")
+    postQueries.deletePost("test_id_2").executeAsOne()
 
     val postsFromDB = postQueries.selectAllPosts().executeAsList()
 
@@ -124,28 +124,35 @@ class SavedPostQueriesTest {
     postQueries.insertOrReplacePost(oldPost)
     postQueries.insertOrReplacePost(veryRecentPost)
 
-    val postsFromLast30Days = postQueries.selectPostsFromLastNDays("30").executeAsList()
+    val postIdsFromLast30Days = postQueries.selectPostIdsFromLastNDays("30").executeAsList()
 
-    assertThat(postsFromLast30Days).hasSize(2)
-    assertThat(postsFromLast30Days.map { it.shortId }).containsExactly("recent_1", "recent_2")
-    assertThat(postsFromLast30Days.map { it.shortId }).doesNotContain("old_1")
+    assertThat(postIdsFromLast30Days).containsExactly("recent_1", "recent_2")
+    assertThat(postIdsFromLast30Days).doesNotContain("old_1")
   }
 
   @Test
-  fun `postExists returns true when post exists`() {
-    val post = createTestData(1).first()
-    postQueries.insertOrReplacePost(post)
+  fun `select posts in id range`() {
+    createTestData(5).forEach { postQueries.insertOrReplacePost(it) }
 
-    val exists = postQueries.postExists("test_id_1").executeAsOne()
+    val posts = postQueries.selectPostsInIdRange("test_id_2", "test_id_4").executeAsList()
 
-    assertThat(exists).isTrue()
+    assertThat(posts.map { it.shortId })
+      .containsExactly("test_id_2", "test_id_3", "test_id_4")
+      .inOrder()
   }
 
   @Test
-  fun `postExists returns false when post does not exist`() {
-    val exists = postQueries.postExists("nonexistent_id").executeAsOne()
+  fun `select post IDs returns only saved post IDs`() {
+    createTestData(2).forEach { postQueries.insertOrReplacePost(it) }
 
-    assertThat(exists).isFalse()
+    assertThat(postQueries.selectPostIds().executeAsList())
+      .containsExactly("test_id_1", "test_id_2")
+      .inOrder()
+  }
+
+  @Test
+  fun `delete post returns nothing when it does not exist`() {
+    assertThat(postQueries.deletePost("nonexistent_id").executeAsOneOrNull()).isNull()
   }
 
   @Test