From 90413cd97faff4d4727b50eca997dec941edb547 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Nicolas <6371790+pynicolas@users.noreply.github.com> Date: Tue, 26 May 2026 14:42:03 +0200 Subject: [PATCH] OCR: basic integration of Tesseract --- app/build.gradle.kts | 1 + .../main/java/org/fairscan/app/FairScanApp.kt | 4 +- .../org/fairscan/app/domain/OcrService.kt | 72 +++++++++++++++++++ .../fairscan/app/platform/AndroidPdfWriter.kt | 41 ++++++++++- .../app/ui/screens/export/ExportViewModel.kt | 7 ++ gradle/libs.versions.toml | 3 +- .../java/org/fairscan/imageprocessing/Ocr.kt | 56 +++++++++++++++ .../org/fairscan/imageprocessing/OcrTest.kt | 48 +++++++++++++ settings.gradle.kts | 1 + 9 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 app/src/main/java/org/fairscan/app/domain/OcrService.kt create mode 100644 imageprocessing/src/main/java/org/fairscan/imageprocessing/Ocr.kt create mode 100644 imageprocessing/src/test/java/org/fairscan/imageprocessing/OcrTest.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 322eb6d..2b43212 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -136,6 +136,7 @@ dependencies { implementation(libs.reorderable) implementation(libs.aboutlibraries.compose.m3) implementation(libs.kotlinx.serialization.json) + implementation(libs.tesseract4android) testImplementation(libs.junit) testImplementation(libs.assertj) diff --git a/app/src/main/java/org/fairscan/app/FairScanApp.kt b/app/src/main/java/org/fairscan/app/FairScanApp.kt index c9cfa4c..dc003dc 100644 --- a/app/src/main/java/org/fairscan/app/FairScanApp.kt +++ b/app/src/main/java/org/fairscan/app/FairScanApp.kt @@ -24,6 +24,7 @@ import org.fairscan.app.data.FileLogger import org.fairscan.app.data.FileManager import org.fairscan.app.data.LogRepository import org.fairscan.app.domain.ImageSegmentationService +import org.fairscan.app.domain.OcrService import org.fairscan.app.platform.AndroidImageLoader import org.fairscan.app.platform.AndroidPdfWriter import org.fairscan.app.ui.screens.camera.CameraViewModel @@ -46,10 +47,11 @@ const val THUMBNAIL_SIZE_DP = 120 class AppContainer(context: Context) { private val cacheDir = context.cacheDir val preparationDir = File(context.cacheDir, "pdfs") + val ocrService = OcrService(context) val fileManager = FileManager( preparationDir, Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), - AndroidPdfWriter() + AndroidPdfWriter(ocrService) ) val logRepository = LogRepository(File(context.filesDir, "logs.txt")) val logger = FileLogger(logRepository) diff --git a/app/src/main/java/org/fairscan/app/domain/OcrService.kt b/app/src/main/java/org/fairscan/app/domain/OcrService.kt new file mode 100644 index 0000000..24d7fc7 --- /dev/null +++ b/app/src/main/java/org/fairscan/app/domain/OcrService.kt @@ -0,0 +1,72 @@ +package org.fairscan.app.domain + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Rect +import com.googlecode.tesseract.android.TessBaseAPI +import com.googlecode.tesseract.android.TessBaseAPI.PageIteratorLevel +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.fairscan.imageprocessing.ImageRect +import org.fairscan.imageprocessing.OcrTextBox +import java.io.File + +class OcrService(private val context: Context) { + + private var tess: TessBaseAPI? = null + private val mutex = Mutex() + + fun initialize() { + prepareTessdata(context) + + val tess = TessBaseAPI() + + val dataPath: String = File(context.filesDir, "tesseract").absolutePath + + // Initialize API for specified language + // (can be called multiple times during Tesseract lifetime) + if (!tess.init(dataPath, "eng")) { // could be multiple languages, like "eng+deu+fra" + tess.recycle() + return + } + + this.tess = tess + } + + // FIXME: Tesseract language-specific data should be downloaded from the SettingsScreen + fun prepareTessdata(context: Context) { + val destDir = File(context.filesDir, "tesseract/tessdata") + val destFile = File(destDir, "eng.traineddata") + if (destFile.exists()) return + + destDir.mkdirs() + context.assets.open("tesseract/tessdata_fast/eng.traineddata").use { input -> + destFile.outputStream().use { output -> + input.copyTo(output) + } + } + } + + suspend fun runOcr(bitmap: Bitmap): List { + mutex.withLock { + val tess = this.tess ?: return listOf() + val textBoxes = mutableListOf() + tess.setImage(bitmap) + tess.getUTF8Text() // Trigger text recognition + val iterator = tess.resultIterator + iterator.begin() + do { + val word = iterator.getUTF8Text(PageIteratorLevel.RIL_WORD) ?: continue + val boundingBox = iterator.getBoundingRect(PageIteratorLevel.RIL_WORD) + val confidence = iterator.confidence(PageIteratorLevel.RIL_WORD) + if (confidence > 50) { + textBoxes.add(OcrTextBox(word, boundingBox.toImageRect())) + } + } while (iterator.next(PageIteratorLevel.RIL_WORD)) + iterator.delete() + return textBoxes + } + } + + private fun Rect.toImageRect(): ImageRect = ImageRect(left, top, right, bottom) +} diff --git a/app/src/main/java/org/fairscan/app/platform/AndroidPdfWriter.kt b/app/src/main/java/org/fairscan/app/platform/AndroidPdfWriter.kt index 6cd2769..1bc8f4a 100644 --- a/app/src/main/java/org/fairscan/app/platform/AndroidPdfWriter.kt +++ b/app/src/main/java/org/fairscan/app/platform/AndroidPdfWriter.kt @@ -14,28 +14,35 @@ */ package org.fairscan.app.platform +import android.graphics.Bitmap import com.tom_roush.pdfbox.pdmodel.PDDocument import com.tom_roush.pdfbox.pdmodel.PDPage import com.tom_roush.pdfbox.pdmodel.PDPageContentStream import com.tom_roush.pdfbox.pdmodel.PDPageContentStream.AppendMode import com.tom_roush.pdfbox.pdmodel.common.PDRectangle +import com.tom_roush.pdfbox.pdmodel.font.PDType1Font import com.tom_roush.pdfbox.pdmodel.graphics.image.JPEGFactory +import com.tom_roush.pdfbox.pdmodel.graphics.image.PDImageXObject +import com.tom_roush.pdfbox.pdmodel.graphics.state.RenderingMode import org.fairscan.app.BuildConfig import org.fairscan.app.data.PdfWriter import org.fairscan.app.domain.PageToExport +import org.fairscan.app.domain.OcrService import org.fairscan.imageprocessing.EstimatedDimensions +import org.fairscan.imageprocessing.OcrCoordinateConverter import org.fairscan.imageprocessing.PaperFormats import java.io.OutputStream import java.util.Calendar -class AndroidPdfWriter : PdfWriter { +class AndroidPdfWriter(val ocrService: OcrService) : PdfWriter { override suspend fun writePdfFromJpegs(pages: List, outputStream: OutputStream): Int { val doc = PDDocument() doc.documentInformation.creationDate = Calendar.getInstance() doc.documentInformation.creator = "FairScan ${BuildConfig.VERSION_NAME}" doc.use { document -> for (page in pages) { - val image = JPEGFactory.createFromByteArray(document, page.jpeg.get().bytes) + val jpeg = page.jpeg.get() + val image = JPEGFactory.createFromByteArray(document, jpeg.bytes) // PDF has 72 points (units) per inch, 1 inch = 25.4 mm val pointsPerMm = 72f / 25.4f @@ -62,6 +69,9 @@ class AndroidPdfWriter : PdfWriter { val contentStream = PDPageContentStream(document, page, AppendMode.OVERWRITE, false) contentStream.drawImage(image, 0f, 0f, widthPoints, heightPoints) + + createText(jpeg.toBitmap(), image, widthPoints, heightPoints, contentStream) + contentStream.close() } // TODO So the whole document is in memory before this line... @@ -69,6 +79,33 @@ class AndroidPdfWriter : PdfWriter { } return doc.numberOfPages } + + private suspend fun createText( + bitmap: Bitmap, + image: PDImageXObject, + widthPoints: Float, + heightPoints: Float, + contentStream: PDPageContentStream, + ) { + val ocr = ocrService.runOcr(bitmap) + val ocrConverter = OcrCoordinateConverter( + imageWidth = image.width, + imageHeight = image.height, + pageWidth = widthPoints, + pageHeight = heightPoints + ) + val font = PDType1Font.HELVETICA + for (textBox in ocr) { + val pdfRect = ocrConverter.convert(textBox.box) + val fontSize = pdfRect.height * 0.8f + contentStream.beginText() + contentStream.setFont(font, fontSize) + contentStream.setRenderingMode(RenderingMode.NEITHER) + contentStream.newLineAtOffset(pdfRect.x, pdfRect.y) + contentStream.showText(textBox.text) + contentStream.endText() + } + } } fun constrainToMaxFormat(widthMm: Double, heightMm: Double): Pair { diff --git a/app/src/main/java/org/fairscan/app/ui/screens/export/ExportViewModel.kt b/app/src/main/java/org/fairscan/app/ui/screens/export/ExportViewModel.kt index 26a1407..c87be7c 100644 --- a/app/src/main/java/org/fairscan/app/ui/screens/export/ExportViewModel.kt +++ b/app/src/main/java/org/fairscan/app/ui/screens/export/ExportViewModel.kt @@ -73,6 +73,13 @@ class ExportViewModel(container: AppContainer, val imageRepository: ImageReposit private val _events = MutableSharedFlow() val events = _events.asSharedFlow() + // FIXME ocrService is initialized here but used in AndroidPdfWriter + init { + viewModelScope.launch { + container.ocrService.initialize() + } + } + private suspend fun generatePdf( exportQuality: ExportQuality ): ExportResult.Pdf = withContext(Dispatchers.IO) { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3a27a38..04ceeee 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,6 +23,7 @@ kotlinSerialization = "1.10.0" reorderable = "3.0.0" jetbrainsKotlinJvm = "2.3.10" coroutines-test = "1.10.2" +tesseract4android = "4.9.0" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } @@ -60,7 +61,7 @@ reorderable = { module = "sh.calvin.reorderable:reorderable", version.ref = "reo aboutlibraries-compose-m3 = { module = "com.mikepenz:aboutlibraries-compose-m3", version.ref = "aboutLibraries" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinSerialization" } kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "coroutines-test" } - +tesseract4android = { group = "cz.adaptech.tesseract4android", name = "tesseract4android-openmp", version.ref = "tesseract4android" } assertj = { group="org.assertj", name="assertj-core", version.ref = "assertj" } diff --git a/imageprocessing/src/main/java/org/fairscan/imageprocessing/Ocr.kt b/imageprocessing/src/main/java/org/fairscan/imageprocessing/Ocr.kt new file mode 100644 index 0000000..b063cda --- /dev/null +++ b/imageprocessing/src/main/java/org/fairscan/imageprocessing/Ocr.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2025-2026 The FairScan authors + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package org.fairscan.imageprocessing + +data class OcrTextBox( + val text: String, + val box: ImageRect, +) + +data class ImageRect( + val left: Int, + val top: Int, + val right: Int, // coordinate of the right side (from the left of the image) + val bottom: Int, // coordinate of the bottom side (from the top of the image) +) { + val width get() = right - left + val height get() = bottom - top +} + +data class PdfRect( + val x: Float, // in points, from left side + val y: Float, // in points, from bottom side (PDF convention) + val width: Float, + val height: Float +) + +class OcrCoordinateConverter( + private val imageWidth: Int, + private val imageHeight: Int, + private val pageWidth: Float, // in PDF points + private val pageHeight: Float // in PDF points +) { + fun convert(rect: ImageRect): PdfRect { + val scaleX = pageWidth / imageWidth + val scaleY = pageHeight / imageHeight + + val x = rect.left * scaleX + val y = pageHeight - (rect.bottom * scaleY) // Y axis is inverted + val width = rect.width * scaleX + val height = rect.height * scaleY + + return PdfRect(x, y, width, height) + } +} diff --git a/imageprocessing/src/test/java/org/fairscan/imageprocessing/OcrTest.kt b/imageprocessing/src/test/java/org/fairscan/imageprocessing/OcrTest.kt new file mode 100644 index 0000000..a4dc749 --- /dev/null +++ b/imageprocessing/src/test/java/org/fairscan/imageprocessing/OcrTest.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2025-2026 The FairScan authors + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package org.fairscan.imageprocessing + +import org.assertj.core.api.Assertions.assertThat +import org.junit.Test + +class OcrTest { + + @Test + fun `top left in image becomes bottom left in PDF`() { + val converter = OcrCoordinateConverter( + imageWidth = 1000, imageHeight = 2000, + pageWidth = 500f, pageHeight = 1000f + ) + val result = converter.convert(ImageRect(0, 0, 100, 200)) + assertThat(result.x).isEqualTo(0f) + assertThat(result.y).isEqualTo(1000f - 100f) + assertThat(result.width).isEqualTo(50f) + assertThat(result.height).isEqualTo(100f) + } + + @Test + fun `bottom right in image becomes top right in PDF`() { + val converter = OcrCoordinateConverter( + imageWidth = 1000, imageHeight = 2000, + pageWidth = 500f, pageHeight = 1000f + ) + val result = converter.convert(ImageRect(900, 1800, 1000, 2000)) + assertThat(result.x).isEqualTo(450f) + assertThat(result.y).isEqualTo(0f) + assertThat(result.width).isEqualTo(50f) + assertThat(result.height).isEqualTo(100f) + } + +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 79f66cc..4142624 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -16,6 +16,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() + maven { url = uri("https://jitpack.io") } } }