dispaly ocr

apps/documentViewer/src/main/kotlin/de/itkl/documentViewer/DocumentViewer.kt

@@ -1,8 +1,6 @@
 package de.itkl.documentViewer
 
-import androidx.compose.desktop.ui.tooling.preview.Preview
 import androidx.compose.foundation.*
-import androidx.compose.foundation.gestures.transformable
 import androidx.compose.foundation.layout.*
 import androidx.compose.material.Text
 import androidx.compose.runtime.*
@@ -10,31 +8,22 @@ import androidx.compose.ui.Alignment
 import androidx.compose.ui.ExperimentalComposeUiApi
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.draw.clip
-import androidx.compose.ui.draw.scale
 import androidx.compose.ui.geometry.Offset
 import androidx.compose.ui.geometry.Size
 import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.ImageBitmap
 import androidx.compose.ui.graphics.RectangleShape
 import androidx.compose.ui.graphics.drawscope.Stroke
-import androidx.compose.ui.graphics.drawscope.rotate
-import androidx.compose.ui.graphics.drawscope.scale
-import androidx.compose.ui.graphics.drawscope.withTransform
-import androidx.compose.ui.graphics.graphicsLayer
 import androidx.compose.ui.graphics.painter.BitmapPainter
 import androidx.compose.ui.graphics.painter.Painter
-import androidx.compose.ui.input.pointer.PointerEventType
-import androidx.compose.ui.input.pointer.onPointerEvent
 import androidx.compose.ui.layout.ContentScale
 import androidx.compose.ui.res.loadImageBitmap
-import androidx.compose.ui.unit.Dp
 import androidx.compose.ui.unit.DpSize
 import androidx.compose.ui.unit.IntOffset
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.window.WindowPlacement
 import androidx.compose.ui.window.WindowPosition
 import androidx.compose.ui.window.rememberWindowState
-import androidx.compose.ui.zIndex
 import com.github.panpf.zoomimage.ZoomImage
 import com.github.panpf.zoomimage.compose.ZoomState
 import com.github.panpf.zoomimage.compose.rememberZoomState
@@ -45,6 +34,7 @@ import de.itkl.httpClient.clients.MsOcr
 import de.itkl.httpClient.httpClientModule
 import de.itkl.textprocessing.CorpusFactory
 import de.itkl.textprocessing.Document
+import de.itkl.textprocessing.OcrPage
 import de.itkl.textprocessing.textProcessingModule
 import de.itkl.tui.tuiModule
 import kotlinx.coroutines.Dispatchers
@@ -67,11 +57,12 @@ private val Log = KotlinLogging.logger { }
 
 
 class DocumentViewer : KoinComponent {
-    suspend fun loadTestDocument() {
+    suspend fun loadTestDocument(): Document {
         val corpus = CorpusFactory().load("assets/xs-reg")
         val document = corpus.document("00001.jpg")
         val ocrExtractor: MsOcr by inject()
         document.process(ocrExtractor)
+        return document
     }
 }
 
@@ -85,7 +76,7 @@ fun main() = auroraApplication {
             httpClientModule)
     }
 
-    runBlocking {
+    val document = runBlocking {
         DocumentViewer().loadTestDocument()
     }
 
@@ -101,13 +92,14 @@ fun main() = auroraApplication {
         windowTitlePaneConfiguration = AuroraWindowTitlePaneConfigurations.AuroraPlain(),
         onCloseRequest = ::exitApplication
     ) {
-        viewImage()
+            viewImage(document)
     }
 }
 
 
 @Composable
-fun viewImage() {
+fun viewImage(document: Document) {
+    val ocr = remember {  runBlocking {  document.retrieveOcrPages().first() } }
     Column (
         modifier = Modifier.fillMaxSize().auroraBackground()
     ) {
@@ -123,7 +115,7 @@ fun viewImage() {
                 contentDescription = "Sample",
                 modifier = Modifier.fillMaxSize()
             )
-            canvas(state.zoomable)
+            canvas(state.zoomable, ocr)
 //            shapes(state.zoomable)
         }
     }
@@ -214,7 +206,7 @@ fun shapes(zoomableState: ZoomableState) {
 }
 @OptIn(ExperimentalComposeUiApi::class)
 @Composable
-fun canvas(zoomableState: ZoomableState) {
+fun canvas(zoomableState: ZoomableState, first: OcrPage) {
     Canvas(modifier = Modifier
         .fillMaxSize()
 //        .onPointerEvent(PointerEventType.Move) {
@@ -229,29 +221,15 @@ fun canvas(zoomableState: ZoomableState) {
             canvasWidth = this.size.width,
             canvasHeight = this.size.height
         )
-//        val width = 2481
-//        val height = 3507
-
-//        val x =
-//        "width": 2481,
-//        "height": 3507,
-//        "boundingBox": [
-//        288,
-//        697,
-//        793,
-//        700,
-//        793,
-//        744,
-//        288,
-//        741
-//        ],
-
 
+        first.words.forEach { word ->
+            val rect = word.rectangle
             drawRect(
                 Color.Blue,
-                topLeft = zoomableState.transform.offset + (Offset(288 * zoomableState.transform.scaleX,697 * zoomableState.transform.scaleY)),
-                size = Size( (793 - 288)* zoomableState.transform.scaleX, (741 - 697) * zoomableState.transform.scaleY),
+                topLeft = zoomableState.transform.offset + (Offset(rect.x.toFloat() * zoomableState.transform.scaleX,rect.y.toFloat() * zoomableState.transform.scaleY)),
+                size = Size(rect.width.toFloat() * zoomableState.transform.scaleX, rect.height.toFloat() * zoomableState.transform.scaleY),
                 style = Stroke(width = 5f)
             )
+        }
     }
 }

libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemAssetManager.kt

@@ -61,6 +61,7 @@ class FilesystemAssets(private val baseDir: Path) : Assets, KoinComponent {
         if (!destination.exists()) {
             return null
         }
+        Log.debug { "Loading file at $destination" }
         val resource = resourceFactory.file(destination)
         return resource
     }

libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/Resource.kt

@@ -21,7 +21,8 @@ interface Resource {
     fun read(): InputStream
 
     fun <T: Any> json(deserializer: DeserializationStrategy<T>): T {
-        return Json.decodeFromString(deserializer, read().readAllBytes().contentToString())
+        val string =  String(read().readAllBytes())
+        return Json.decodeFromString(deserializer, string)
     }
 
 }

libraries/textprocessing/build.gradle.kts

@@ -3,6 +3,7 @@ dependencies {
     api("org.apache.lucene:lucene-analysis-common:9.9.0")
     api("io.github.piruin:geok:1.2.2")
     api(project(":libraries:assetmanager"))
+    api("com.soywiz.korge:korge-foundation:5.1.0")
     implementation("com.github.doyaaaaaken:kotlin-csv-jvm:1.9.2")
     implementation("com.google.guava:guava:32.1.3-jre")
 }

libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Document.kt

@@ -8,6 +8,7 @@ import de.itkl.core_api.interfaces.Resource
 import de.itkl.core_api.interfaces.assets.Assets
 import de.itkl.core_api.interfaces.assets.FileProcessorBackend
 import de.itkl.core_api.interfaces.data.Processable
+import korlibs.math.geom.Rectangle
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.filter
@@ -33,11 +34,10 @@ class Document(
     suspend fun retrieveOcrPages(): List<OcrPage> {
         // TODO: How to identify the assets independently from their name?
         val resource = checkNotNull(assets()
-            .retrieve("ms-ocr")) {
+            .retrieve("ms-ocr.json")) {
             "Ocr for $name is not yet created"
         }
         val msOcrResponse = resource.json(MsOcrResponse.serializer())
-
         return msOcrResponse.analyzeResult.readResults.map { toOcrPage(it) }
     }
     override suspend fun process(fileProcessor: FileProcessor2) {
@@ -59,12 +59,17 @@ class Document(
     private fun toOcrWord(word: MsOcrResponse.AnalyzeResult.ReadResult.Line.Word): OcrPage.OcrWord {
         val box = word.boundingBox
         return OcrPage.OcrWord(
-            polygon = Polygon(listOf(
-                LatLng(box[0].toDouble(), box[1].toDouble()),
-                LatLng(box[2].toDouble(), box[3].toDouble()),
-                LatLng(box[4].toDouble(), box[5].toDouble()),
-                LatLng(box[6].toDouble(), box[7].toDouble()),
-            )),
+            Rectangle(
+                x = box[0],
+                y = box[1],
+                width = box[2] - box[0],
+                height = box[7] - box[1]),
+//            polygon = Polygon(listOf(
+//                LatLng(box[0].toDouble(), box[1].toDouble()),
+//                LatLng(box[2].toDouble(), box[3].toDouble()),
+//                LatLng(box[4].toDouble(), box[5].toDouble()),
+//                LatLng(box[6].toDouble(), box[7].toDouble()),
+//            )),
             text = word.text
         )
     }
@@ -75,24 +80,24 @@ class OcrPage(
     val height: Int,
     val pageNumber: Int,
     val words: List<OcrWord>,
-    val regions: List<DocumentRegion> = emptyList()
+//    val regions: List<DocumentRegion> = emptyList()
 ) {
-    inner class DocumentRegion(
-        private val polygon: Polygon,
-        private val type: String,
-    ) {
-        fun words(): Flow<OcrWord> {
-            return words
-                .asFlow()
-                .filter { word -> word.polygon.intersectionWith(polygon) != null }
-        }
-    }
+//    inner class DocumentRegion(
+//        private val polygon: Polygon,
+//        private val type: String,
+//    ) {
+//        fun words(): Flow<OcrWord> {
+//            return words
+//                .asFlow()
+//                .filter { word -> word.polygon.intersectionWith(polygon) != null }
+//        }
+//    }
 
-    fun addOcrWord(polygon: Polygon, text: String): OcrWord {
-        return OcrWord(polygon, text)
+    fun addOcrWord(rectangle: Rectangle, text: String): OcrWord {
+        return OcrWord(rectangle, text)
     }
     class OcrWord(
-        val polygon: Polygon,
+        val rectangle: Rectangle,
         val text: String
     )
 }