diff --git a/apps/documentViewer/src/main/kotlin/de/itkl/documentViewer/DocumentViewer.kt b/apps/documentViewer/src/main/kotlin/de/itkl/documentViewer/DocumentViewer.kt index 5987092..9310ed8 100644 --- a/apps/documentViewer/src/main/kotlin/de/itkl/documentViewer/DocumentViewer.kt +++ b/apps/documentViewer/src/main/kotlin/de/itkl/documentViewer/DocumentViewer.kt @@ -1,8 +1,6 @@ package de.itkl.documentViewer -import androidx.compose.desktop.ui.tooling.preview.Preview import androidx.compose.foundation.* -import androidx.compose.foundation.gestures.transformable import androidx.compose.foundation.layout.* import androidx.compose.material.Text import androidx.compose.runtime.* @@ -10,31 +8,22 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.ExperimentalComposeUiApi import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip -import androidx.compose.ui.draw.scale import androidx.compose.ui.geometry.Offset import androidx.compose.ui.geometry.Size import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.RectangleShape import androidx.compose.ui.graphics.drawscope.Stroke -import androidx.compose.ui.graphics.drawscope.rotate -import androidx.compose.ui.graphics.drawscope.scale -import androidx.compose.ui.graphics.drawscope.withTransform -import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.graphics.painter.BitmapPainter import androidx.compose.ui.graphics.painter.Painter -import androidx.compose.ui.input.pointer.PointerEventType -import androidx.compose.ui.input.pointer.onPointerEvent import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.res.loadImageBitmap -import androidx.compose.ui.unit.Dp import androidx.compose.ui.unit.DpSize import androidx.compose.ui.unit.IntOffset import androidx.compose.ui.unit.dp import androidx.compose.ui.window.WindowPlacement import androidx.compose.ui.window.WindowPosition import androidx.compose.ui.window.rememberWindowState -import androidx.compose.ui.zIndex import com.github.panpf.zoomimage.ZoomImage import com.github.panpf.zoomimage.compose.ZoomState import com.github.panpf.zoomimage.compose.rememberZoomState @@ -45,6 +34,7 @@ import de.itkl.httpClient.clients.MsOcr import de.itkl.httpClient.httpClientModule import de.itkl.textprocessing.CorpusFactory import de.itkl.textprocessing.Document +import de.itkl.textprocessing.OcrPage import de.itkl.textprocessing.textProcessingModule import de.itkl.tui.tuiModule import kotlinx.coroutines.Dispatchers @@ -67,11 +57,12 @@ private val Log = KotlinLogging.logger { } class DocumentViewer : KoinComponent { - suspend fun loadTestDocument() { + suspend fun loadTestDocument(): Document { val corpus = CorpusFactory().load("assets/xs-reg") val document = corpus.document("00001.jpg") val ocrExtractor: MsOcr by inject() document.process(ocrExtractor) + return document } } @@ -85,7 +76,7 @@ fun main() = auroraApplication { httpClientModule) } - runBlocking { + val document = runBlocking { DocumentViewer().loadTestDocument() } @@ -101,13 +92,14 @@ fun main() = auroraApplication { windowTitlePaneConfiguration = AuroraWindowTitlePaneConfigurations.AuroraPlain(), onCloseRequest = ::exitApplication ) { - viewImage() + viewImage(document) } } @Composable -fun viewImage() { +fun viewImage(document: Document) { + val ocr = remember { runBlocking { document.retrieveOcrPages().first() } } Column ( modifier = Modifier.fillMaxSize().auroraBackground() ) { @@ -123,7 +115,7 @@ fun viewImage() { contentDescription = "Sample", modifier = Modifier.fillMaxSize() ) - canvas(state.zoomable) + canvas(state.zoomable, ocr) // shapes(state.zoomable) } } @@ -214,7 +206,7 @@ fun shapes(zoomableState: ZoomableState) { } @OptIn(ExperimentalComposeUiApi::class) @Composable -fun canvas(zoomableState: ZoomableState) { +fun canvas(zoomableState: ZoomableState, first: OcrPage) { Canvas(modifier = Modifier .fillMaxSize() // .onPointerEvent(PointerEventType.Move) { @@ -229,29 +221,15 @@ fun canvas(zoomableState: ZoomableState) { canvasWidth = this.size.width, canvasHeight = this.size.height ) -// val width = 2481 -// val height = 3507 - -// val x = -// "width": 2481, -// "height": 3507, -// "boundingBox": [ -// 288, -// 697, -// 793, -// 700, -// 793, -// 744, -// 288, -// 741 -// ], - + first.words.forEach { word -> + val rect = word.rectangle drawRect( Color.Blue, - topLeft = zoomableState.transform.offset + (Offset(288 * zoomableState.transform.scaleX,697 * zoomableState.transform.scaleY)), - size = Size( (793 - 288)* zoomableState.transform.scaleX, (741 - 697) * zoomableState.transform.scaleY), + topLeft = zoomableState.transform.offset + (Offset(rect.x.toFloat() * zoomableState.transform.scaleX,rect.y.toFloat() * zoomableState.transform.scaleY)), + size = Size(rect.width.toFloat() * zoomableState.transform.scaleX, rect.height.toFloat() * zoomableState.transform.scaleY), style = Stroke(width = 5f) ) + } } } diff --git a/libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemAssetManager.kt b/libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemAssetManager.kt index f204492..afba4f0 100644 --- a/libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemAssetManager.kt +++ b/libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemAssetManager.kt @@ -61,6 +61,7 @@ class FilesystemAssets(private val baseDir: Path) : Assets, KoinComponent { if (!destination.exists()) { return null } + Log.debug { "Loading file at $destination" } val resource = resourceFactory.file(destination) return resource } diff --git a/libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/Resource.kt b/libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/Resource.kt index 7ea0d03..d7dadc9 100644 --- a/libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/Resource.kt +++ b/libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/Resource.kt @@ -21,7 +21,8 @@ interface Resource { fun read(): InputStream fun json(deserializer: DeserializationStrategy): T { - return Json.decodeFromString(deserializer, read().readAllBytes().contentToString()) + val string = String(read().readAllBytes()) + return Json.decodeFromString(deserializer, string) } } diff --git a/libraries/textprocessing/build.gradle.kts b/libraries/textprocessing/build.gradle.kts index ea3641e..21351ac 100644 --- a/libraries/textprocessing/build.gradle.kts +++ b/libraries/textprocessing/build.gradle.kts @@ -3,6 +3,7 @@ dependencies { api("org.apache.lucene:lucene-analysis-common:9.9.0") api("io.github.piruin:geok:1.2.2") api(project(":libraries:assetmanager")) + api("com.soywiz.korge:korge-foundation:5.1.0") implementation("com.github.doyaaaaaken:kotlin-csv-jvm:1.9.2") implementation("com.google.guava:guava:32.1.3-jre") } diff --git a/libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Document.kt b/libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Document.kt index 4f3fb37..f7a6dd8 100644 --- a/libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Document.kt +++ b/libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Document.kt @@ -8,6 +8,7 @@ import de.itkl.core_api.interfaces.Resource import de.itkl.core_api.interfaces.assets.Assets import de.itkl.core_api.interfaces.assets.FileProcessorBackend import de.itkl.core_api.interfaces.data.Processable +import korlibs.math.geom.Rectangle import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.filter @@ -33,11 +34,10 @@ class Document( suspend fun retrieveOcrPages(): List { // TODO: How to identify the assets independently from their name? val resource = checkNotNull(assets() - .retrieve("ms-ocr")) { + .retrieve("ms-ocr.json")) { "Ocr for $name is not yet created" } val msOcrResponse = resource.json(MsOcrResponse.serializer()) - return msOcrResponse.analyzeResult.readResults.map { toOcrPage(it) } } override suspend fun process(fileProcessor: FileProcessor2) { @@ -59,12 +59,17 @@ class Document( private fun toOcrWord(word: MsOcrResponse.AnalyzeResult.ReadResult.Line.Word): OcrPage.OcrWord { val box = word.boundingBox return OcrPage.OcrWord( - polygon = Polygon(listOf( - LatLng(box[0].toDouble(), box[1].toDouble()), - LatLng(box[2].toDouble(), box[3].toDouble()), - LatLng(box[4].toDouble(), box[5].toDouble()), - LatLng(box[6].toDouble(), box[7].toDouble()), - )), + Rectangle( + x = box[0], + y = box[1], + width = box[2] - box[0], + height = box[7] - box[1]), +// polygon = Polygon(listOf( +// LatLng(box[0].toDouble(), box[1].toDouble()), +// LatLng(box[2].toDouble(), box[3].toDouble()), +// LatLng(box[4].toDouble(), box[5].toDouble()), +// LatLng(box[6].toDouble(), box[7].toDouble()), +// )), text = word.text ) } @@ -75,24 +80,24 @@ class OcrPage( val height: Int, val pageNumber: Int, val words: List, - val regions: List = emptyList() +// val regions: List = emptyList() ) { - inner class DocumentRegion( - private val polygon: Polygon, - private val type: String, - ) { - fun words(): Flow { - return words - .asFlow() - .filter { word -> word.polygon.intersectionWith(polygon) != null } - } - } +// inner class DocumentRegion( +// private val polygon: Polygon, +// private val type: String, +// ) { +// fun words(): Flow { +// return words +// .asFlow() +// .filter { word -> word.polygon.intersectionWith(polygon) != null } +// } +// } - fun addOcrWord(polygon: Polygon, text: String): OcrWord { - return OcrWord(polygon, text) + fun addOcrWord(rectangle: Rectangle, text: String): OcrWord { + return OcrWord(rectangle, text) } class OcrWord( - val polygon: Polygon, + val rectangle: Rectangle, val text: String ) } \ No newline at end of file