Kinch
/
docthor
1
0
Fork 0
Code Issues 15 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: Kinch:main
Branches Tags
Kinch:18
Kinch:16
Kinch:17
Kinch:main
Kinch:3
Kinch:1
Kinch:7
Kinch:develop
..
compare: Kinch:7
Branches Tags
Kinch:18
Kinch:16
Kinch:17
Kinch:main
Kinch:3
Kinch:1
Kinch:7
Kinch:develop

No commits in common. "main" and "7" have entirely different histories.
main ... 7

73 changed files with 85 additions and 1796 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
.gradle
build
.idea
/assets
assets

41
.run/DocViewer KORGE.run.xml
View File

@ -1,41 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="DocViewer KORGE" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="env">
<map>
<entry key="GRADLE_USER_HOME" value="$USER_HOME$/.gradle" />
</map>
</option>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":apps:documentViewerKorge:runJvmAutoreload" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<EXTENSION ID="com.intellij.execution.ExternalSystemRunConfigurationJavaExtension">
<extension name="net.ashald.envfile">
<option name="IS_ENABLED" value="false" />
<option name="IS_SUBST" value="false" />
<option name="IS_PATH_MACRO_SUPPORTED" value="false" />
<option name="IS_IGNORE_MISSING_FILES" value="false" />
<option name="IS_ENABLE_EXPERIMENTAL_INTEGRATIONS" value="false" />
<ENTRIES>
<ENTRY IS_ENABLED="true" PARSER="runconfig" IS_EXECUTABLE="false" />
</ENTRIES>
</extension>
</EXTENSION>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>false</RunAsTest>
<method v="2" />
</configuration>
</component>

25
.run/DocumentViewerKt.run.xml
View File

@ -1,25 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="DocumentViewerKt" type="JetRunConfigurationType" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="de.itkl.documentViewer.DocumentViewerKt" />
<module name="docthor.apps.documentViewer.main" />
<shortenClasspath name="NONE" />
<extension name="net.ashald.envfile">
<option name="IS_ENABLED" value="false" />
<option name="IS_SUBST" value="false" />
<option name="IS_PATH_MACRO_SUPPORTED" value="false" />
<option name="IS_IGNORE_MISSING_FILES" value="false" />
<option name="IS_ENABLE_EXPERIMENTAL_INTEGRATIONS" value="false" />
<ENTRIES>
<ENTRY IS_ENABLED="true" PARSER="runconfig" IS_EXECUTABLE="false" />
</ENTRIES>
</extension>
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>

1
Writerside/d.tree
View File

@ -7,5 +7,4 @@
start-page="docthor.md">
<toc-element topic="docthor.md"/>
<toc-element topic="Snippets.md"/>
</instance-profile>

36
Writerside/topics/Snippets.md
View File

@ -1,36 +0,0 @@
# Snippets
## Scale a Shape alongside ZoomImage
```kotlin
@Composable
fun shapes(zoomableState: ZoomableState) {
Box(modifier = Modifier.fillMaxSize()) {
val scaleX = zoomableState.transform.scaleX
val scaleY = zoomableState.transform.scaleY
Box(
modifier = Modifier
.offset { IntOffset(
((zoomableState.transform.offset.x + (288 * scaleX)) ).toInt(),
((zoomableState.transform.offset.y + (697 * scaleY)) ).toInt()
) }
.clip(RectangleShape)
.size(100.dp * scaleX)
.background(Color.Red)
)
}
}
```
### Scale a Canvas alongside Zoomimage
```kotlin
drawRect(
Color.Blue,
topLeft = zoomableState.transform.offset + (Offset(288 * zoomableState.transform.scaleX,697 * zoomableState.transform.scaleY)),
size = Size( (793 - 288)* zoomableState.transform.scaleX, (741 - 697) * zoomableState.transform.scaleY),
style = Stroke(width = 5f)
)
```

9
Writerside/topics/docthor.md
View File

@ -11,15 +11,6 @@ Asset can be found under <path>memento:/mnt/wd/export/data</path>
<def title="PDF Renderer for Compose">
<a href="https://github.com/GRizzi91/bouquet">bouquet</a>
</def>
<def title="Moko Resource">
<a href="https://github.com/icerockdev/moko-resources">Resource Management für Compose</a>
</def>
<def title="Aurora">
<a href="https://github.com/kirill-grouchnikov/aurora">Building modern, elegant and fast desktop Compose applications</a>
</def>
<def title="Zoomimage">
<a href="https://github.com/panpf/zoomimage">Zooming an Image</a>
</def>
</deflist>
## Modules - Libraries

1
app/build.gradle.kts
View File

@ -1,4 +1,3 @@
plugins {
id("docthor.kotlin-application-conventions")
}

2
app/src/main/kotlin/docthor/app/App.kt
View File

@ -23,7 +23,7 @@ class ComputeIdf : CliktCommand() {
.required()
override fun run() = runBlocking {
TfIdfPipeline(force = false)
TfIdfPipeline(force = true)
.input(corpus)
}
}

31
apps/documentViewer/build.gradle.kts
View File

@ -1,31 +0,0 @@
plugins {
id("org.jetbrains.compose") version "1.5.11"
}
repositories {
google()
}
dependencies {
fun addProjects(vararg names: String) {
names.forEach {
implementation(project(":libraries:$it"))
}
}
addProjects(
"assetmanager",
"core-api",
"textprocessing",
"httpClient",
"tui",
)
implementation("org.pushing-pixels:aurora-theming:1.3.0")
implementation("org.pushing-pixels:aurora-component:1.3.0")
implementation("org.pushing-pixels:aurora-window:1.3.0")
implementation(compose.desktop.currentOs)
implementation("io.github.panpf.zoomimage:zoomimage-compose:1.0.0-beta11")
implementation("io.github.panpf.zoomimage:zoomimage-compose-desktop:1.0.0-beta11")
}

206
apps/documentViewer/src/main/kotlin/de/itkl/documentViewer/DocumentViewer.kt
View File

@ -1,206 +0,0 @@
package de.itkl.documentViewer
import androidx.compose.foundation.*
import androidx.compose.foundation.layout.*
import androidx.compose.material.Text
import androidx.compose.runtime.*
import androidx.compose.ui.Alignment
import androidx.compose.ui.ExperimentalComposeUiApi
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.RectangleShape
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.graphics.painter.BitmapPainter
import androidx.compose.ui.graphics.painter.Painter
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.res.loadImageBitmap
import androidx.compose.ui.unit.DpSize
import androidx.compose.ui.unit.IntOffset
import androidx.compose.ui.unit.dp
import androidx.compose.ui.window.WindowPlacement
import androidx.compose.ui.window.WindowPosition
import androidx.compose.ui.window.rememberWindowState
import com.github.panpf.zoomimage.ZoomImage
import com.github.panpf.zoomimage.compose.ZoomState
import com.github.panpf.zoomimage.compose.rememberZoomState
import com.github.panpf.zoomimage.compose.zoom.*
import de.itkl.assetmanager.assetManagerModule
import de.itkl.core_api.coreApiModule
import de.itkl.httpClient.clients.MsOcr
import de.itkl.httpClient.httpClientModule
import de.itkl.textprocessing.CorpusFactory
import de.itkl.textprocessing.Document
import de.itkl.textprocessing.OcrPage
import de.itkl.textprocessing.textProcessingModule
import de.itkl.tui.tuiModule
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.pushingpixels.aurora.theming.auroraBackground
import org.pushingpixels.aurora.theming.marinerSkin
import org.pushingpixels.aurora.window.AuroraWindow
import org.pushingpixels.aurora.window.AuroraWindowTitlePaneConfigurations
import org.pushingpixels.aurora.window.auroraApplication
import java.io.File
import java.io.IOException
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.runBlocking
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.koin.core.context.startKoin
import com.github.panpf.zoomimage.util.Logger as ZoomLogger
private val Log = KotlinLogging.logger { }
class DocumentViewer : KoinComponent {
suspend fun loadTestDocument(): Document {
val corpus = CorpusFactory().load("assets/xs-reg")
val document = corpus.document("00001.jpg")
val ocrExtractor: MsOcr by inject()
document.process(ocrExtractor)
return document
}
}
fun main() = auroraApplication {
startKoin {
modules(
coreApiModule,
textProcessingModule,
tuiModule,
assetManagerModule,
httpClientModule)
}
val document = runBlocking {
DocumentViewer().loadTestDocument()
}
val state = rememberWindowState(
placement = WindowPlacement.Floating,
position = WindowPosition.Aligned(Alignment.Center),
size = DpSize(1000. dp, 800.dp)
)
AuroraWindow(
skin = marinerSkin(),
title = "Document Viewer",
state = state,
windowTitlePaneConfiguration = AuroraWindowTitlePaneConfigurations.AuroraPlain(),
onCloseRequest = ::exitApplication
) {
viewImage(document)
}
}
@Composable
fun viewImage(document: Document) {
val ocr = remember { runBlocking { document.retrieveOcrPages().first() } }
Column (
modifier = Modifier.fillMaxSize().auroraBackground()
) {
val state = rememberZoomState(logger = ZoomLogger("zoom", level = ZoomLogger.INFO))
Text("${state.zoomable.transform.scale} ${state.zoomable.transform.offset}")
Box(
modifier = Modifier.fillMaxSize()
) {
ZoomedImage(
state = state,
load = { loadImageBitmap(File("assets/xs-reg/00001.jpg")) },
painterFor = { remember { BitmapPainter(it) } },
contentDescription = "Sample",
modifier = Modifier.fillMaxSize()
)
canvas(state.zoomable, ocr)
// shapes(state.zoomable)
}
}
}
@Composable
fun <T> ZoomedImage(
state: ZoomState,
load: suspend () -> T,
painterFor: @Composable (T) -> Painter,
contentDescription: String,
modifier: Modifier = Modifier,
contentScale: ContentScale = ContentScale.Fit,
) {
val image: T? by produceState<T?>(null) {
value = withContext(Dispatchers.IO) {
try {
load()
} catch (e: IOException) {
// instead of printing to console, you can also write this to log,
// or show some error placeholder
e.printStackTrace()
null
}
}
}
if (image != null) {
val scrollBar = remember {
ScrollBarSpec(
color = Color.Red,
size = 6.dp,
margin = 12.dp,
)
}
ZoomImage(
painter = painterFor(image!!),
contentDescription = contentDescription,
contentScale = contentScale,
modifier = modifier,
scrollBar = scrollBar,
state = state
)
}
}
fun loadImageBitmap(file: File): ImageBitmap =
file.inputStream().buffered().use(::loadImageBitmap)
@OptIn(ExperimentalFoundationApi::class)
@Composable
fun shapes(zoomableState: ZoomableState) {
Box(modifier = Modifier.fillMaxSize()) {
val scaleX = zoomableState.transform.scaleX
val scaleY = zoomableState.transform.scaleY
Box(
modifier = Modifier
.offset { IntOffset(
((zoomableState.transform.offset.x + (288 * scaleX)) ).toInt(),
((zoomableState.transform.offset.y + (697 * scaleY)) ).toInt()
) }
.clip(RectangleShape)
.size(100.dp * scaleX)
.background(Color.Red)
)
}
}
@OptIn(ExperimentalComposeUiApi::class)
@Composable
fun canvas(zoomableState: ZoomableState, first: OcrPage) {
Canvas(modifier = Modifier
.fillMaxSize()
// .onPointerEvent(PointerEventType.Move) {
// val position = it.changes.first().position
// println(position)
// }
)
{
first.words.forEach { word ->
val rect = word.rectangle
drawRect(
Color.Blue,
topLeft = zoomableState.transform.offset + (Offset(rect.x.toFloat() * zoomableState.transform.scaleX,rect.y.toFloat() * zoomableState.transform.scaleY)),
size = Size(rect.width.toFloat() * zoomableState.transform.scaleX, rect.height.toFloat() * zoomableState.transform.scaleY),
style = Stroke(width = 5f)
)
}
}
}

15
apps/documentViewerKorge/build.gradle.kts
View File

@ -1,15 +0,0 @@
import korlibs.korge.gradle.korge
plugins {
id("com.soywiz.korge") version "5.3.0"
}
dependencies {
jvmMainImplementation("ch.qos.logback:logback-classic:1.4.14")
jvmMainImplementation(project(":libraries:docthor-core"))
}
korge {
targetJvm()
serializationJson()
}

2
apps/documentViewerKorge/deps.kproject.yml
View File

@ -1,2 +0,0 @@
dependencies:
- https://github.com/korlibs/korge-box2d/tree/v0.1.2/korge-box2d##b1737ab3985c0bd3e2f002346ff2ac43ca1ebf48

108
apps/documentViewerKorge/src/jvmMain/kotlin/de/itkl/documentViewerKorge/main.kt
View File

@ -1,108 +0,0 @@
import de.itkl.docthor.core.DocumentViewer
import de.itkl.textprocessing.Document
import korlibs.event.Key
import korlibs.image.bitmap.context2d
import korlibs.korge.*
import korlibs.korge.scene.*
import korlibs.korge.view.*
import korlibs.image.color.*
import korlibs.image.format.*
import korlibs.io.file.std.*
import korlibs.korge.input.*
import korlibs.korge.tween.get
import korlibs.korge.tween.tween
import korlibs.korge.ui.tooltip
import korlibs.korge.ui.uiCheckBox
import korlibs.logger.AnsiEscape
import korlibs.math.geom.*
import korlibs.math.geom.shape.toShape2D
import korlibs.math.geom.shape.toShape2d
import korlibs.math.geom.vector.VectorPath
import korlibs.math.interpolation.Easing
import java.nio.file.Paths
import kotlin.time.Duration.Companion.seconds
suspend fun main() {
val document = DocumentViewer().loadTestDocument(Paths.get("assets/xs-reg"), "00001.jpg")
Korge(windowSize = Size(512, 512), backgroundColor = Colors["#2b2b2b"]) {
val sceneContainer = sceneContainer()
sceneContainer.changeTo { ViewDocument(document) }
}
}
class ViewDocument(private val document: Document) : Scene() {
override suspend fun SContainer.sceneMain() {
var scaleFactor = 0.5
var offset = Point(0.0,0.0)
val moveFactor = 50.0
var rotationFactor = 0.0
scale(scaleFactor)
suspend fun zoom(amount: Double) {
scaleFactor += amount
tween(
this@sceneMain::scale[scaleFactor],
time = 0.1.seconds,
easing = Easing.EASE_IN_OUT
)
}
suspend fun moveX(amount: Double) {
offset = offset.copy(x = offset.x + amount)
tween(
this@sceneMain::x[offset.x],
time = 0.1.seconds,
easing = Easing.EASE_IN_OUT)
}
suspend fun moveY(amount: Double) {
offset = offset.copy(y = offset.y + amount)
tween(
this@sceneMain::y[offset.y],
time = 0.1.seconds,
easing = Easing.EASE_IN_OUT)
}
suspend fun rotateBy(amount: Double){
rotationFactor += amount
tween(
this@sceneMain::rotation[rotationFactor.degrees],
time = 0.1.seconds,
easing = Easing.EASE_IN_OUT
)
}
keys {
down { key ->
when (key.key) {
Key.EQUAL -> zoom(0.05)
Key.MINUS -> zoom(-0.05)
Key.UP -> moveY(moveFactor)
Key.DOWN -> moveY(-moveFactor)
Key.RIGHT -> moveX(moveFactor)
Key.LEFT -> moveX(-moveFactor)
Key.E -> rotateBy(10.0)
Key.R -> rotateBy(-10.0)
else -> {}
}
}
}
val imageFile = localCurrentDirVfs["assets/xs-reg/00001.jpg"].readBitmap()
image(imageFile)
uiCheckBox { text = "foo" }
document.retrieveOcrPages().first().words.forEach { word ->
solidRect(
width = word.rectangle.width,
height = word.rectangle.height,
color = Colors.AQUAMARINE.withA(128)
) {
x = word.rectangle.x
y = word.rectangle.y
}.onClick {
println(word.text)
}
}
}
}

1
apps/documentViewerKorge/src/jvmMain/resources/clear_sans.fnt
View File

File diff suppressed because one or more lines are too long

BIN
apps/documentViewerKorge/src/jvmMain/resources/kitten_pixelated.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 4.1 KiB

3
apps/xsCli/build.gradle.kts
View File

@ -1,3 +0,0 @@
dependencies {
implementation(project(":libraries:httpClient"))
}

68
apps/xsCli/src/main/kotlin/de/itkl/xsCli/App.kt
View File

@ -1,68 +0,0 @@
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.subcommands
import com.github.ajalt.clikt.parameters.options.convert
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.options.required
import com.github.ajalt.clikt.parameters.types.int
import de.itkl.httpClient.clients.TaskWaitStatus
import de.itkl.httpClient.clients.XsClient
import de.itkl.httpClient.httpClientModule
import kotlinx.coroutines.*
import kotlinx.coroutines.Dispatchers.IO
import java.util.Random
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.koin.core.context.GlobalContext.startKoin
import java.io.File
import java.nio.file.Path
import java.nio.file.Paths
import kotlin.time.Duration.Companion.minutes
class Cli : CliktCommand() {
override fun run() {}
inner class StressAnalyse : CliktCommand(name = "stress-analyse") {
val inputDirectory: File by option(help="Input directory path").convert { File(it) }.required()
val tasks: Int by option(help="Number of tasks").int().required()
override fun run() {
}
}
init {
subcommands(StressAnalyse())
}
}
class XsCli : KoinComponent {
private val xsClient: XsClient by inject()
suspend fun run(tasks: Int, inputDirectory: Path) = coroutineScope {
val files = inputDirectory.toFile()
.listFiles()!!
.toList()
.filter { it.isFile }.filter { it.extension in listOf("pdf", "ttf", "jpg", "jpeg") }
.filter { !it.name.startsWith(".") }
val random = Random()
val (success, error) = (0..<tasks)
.map {
val file = files[random.nextInt(files.size)]
async {
val taskRef = xsClient.analyse(file.toPath())
xsClient.waitFor(taskRef)
}
}
.awaitAll()
.partition { it.status == TaskWaitStatus.SUCCESS }
println("Summary: ${success.size + error.size}: ${success.size}/${error.size}")
}
}
suspend fun main(args: Array<String>) {
startKoin {
modules(httpClientModule)
}
XsCli().run(tasks = 100, inputDirectory = Paths.get("assets/xs-reg"))
}

15
apps/xsViewer/build.gradle.kts
View File

@ -1,15 +0,0 @@
import korlibs.korge.gradle.korge
plugins {
id("com.soywiz.korge") version "5.3.0"
}
dependencies {
jvmMainImplementation("ch.qos.logback:logback-classic:1.4.14")
jvmMainImplementation(project(":libraries:docthor-core"))
}
korge {
targetJvm()
serializationJson()
}

7
build.gradle.kts
View File

@ -1,10 +1,3 @@
project(":libraries").subprojects {
apply(plugin = "docthor.kotlin-library-conventions")
}
project(":apps").subprojects {
if(name != "documentViewerKorge" && name != "xsViewer") {
apply(plugin = "docthor.kotlin-application-conventions")
}
}

2
buildSrc/build.gradle.kts
View File

@ -7,5 +7,5 @@ repositories {
}
dependencies {
implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$embeddedKotlinVersion")
implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.20")
}

13
buildSrc/src/main/kotlin/docthor.kotlin-common-conventions.gradle.kts
View File

@ -1,6 +1,3 @@
import gradle.kotlin.dsl.accessors._6ebf0b5d05ec1eff67605a516c5db18b.implementation
import gradle.kotlin.dsl.accessors._d9dcfd1a467b0b6fe90c5571a57aa558.api
import gradle.kotlin.dsl.accessors._d9dcfd1a467b0b6fe90c5571a57aa558.testImplementation
import org.gradle.api.plugins.jvm.JvmTestSuite
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
@ -15,15 +12,7 @@ repositories {
dependencies {
val koin_version = "3.5.3"
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3")
api("io.insert-koin:koin-core:$koin_version")
implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.5.0")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.2")
api("io.github.oshai:kotlin-logging-jvm:5.1.0")
testImplementation("io.insert-koin:koin-test:$koin_version")
testImplementation("ch.qos.logback:logback-classic:1.4.14")
testImplementation("com.willowtreeapps.assertk:assertk:0.28.0")
implementation("io.insert-koin:koin-core:$koin_version")
}
java {

22
gradle/libs.versions.toml
View File

@ -1,22 +0,0 @@
[versions]
kotlin = "1.9.21"
coroutines = "1.7.3"
compose = "1.5.11"
dokka = "1.9.10"
batik = "1.17"
versionchecker = "0.50.0"
mavenpublish = "0.25.3"
[plugins]
korge = { id = "com.soywiz.korge", version = "5.3.0" }
[libraries]
compose-desktop = { module = "org.jetbrains.compose:compose-gradle-plugin", version.ref = "compose" }
kotlin-gradlePlugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }
kotlin-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "coroutines" }
dokka-gradlePlugin = { module = "org.jetbrains.dokka:dokka-gradle-plugin", version.ref = "dokka"}
batik = { module = "org.apache.xmlgraphics:batik-all", version.ref = "batik" }
versionchecker-gradlePlugin = { module = "com.github.ben-manes:gradle-versions-plugin", version.ref = "versionchecker" }
mavenpublish-gradlePlugin = { module = "com.vanniktech:gradle-maven-publish-plugin", version.ref = "mavenpublish" }

5
libraries/assetmanager/build.gradle.kts
View File

@ -1,5 +0,0 @@
dependencies {
api(project(":libraries:core-api"))
// used for contentType
api("io.ktor:ktor-http-jvm:2.3.7")
}

15
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/assetManagerModule.kt
View File

@ -1,15 +0,0 @@
package de.itkl.assetmanager
import de.itkl.assetmanager.implementation.AssetsFileProcessorBackend
import de.itkl.assetmanager.implementation.FilesystemAssetManager
import de.itkl.assetmanager.implementation.FilesystemProjectManager
import de.itkl.assetmanager.interfaces.AssetManager
import de.itkl.assetmanager.interfaces.ProjectManager
import de.itkl.core_api.interfaces.assets.FileProcessorBackend
import org.koin.dsl.module
val assetManagerModule = module {
single<ProjectManager> { FilesystemProjectManager() }
single<AssetManager> { FilesystemAssetManager() }
single<FileProcessorBackend> { AssetsFileProcessorBackend() }
}

22
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/AssetsFileProcessorBackend.kt
View File

@ -1,22 +0,0 @@
package de.itkl.assetmanager.implementation
import de.itkl.core_api.interfaces.FileProcessor2
import de.itkl.core_api.interfaces.Resource
import de.itkl.core_api.interfaces.assets.Assets
import de.itkl.core_api.interfaces.assets.FileProcessorBackend
import io.github.oshai.kotlinlogging.KotlinLogging
import org.koin.core.component.KoinComponent
private val Log = KotlinLogging.logger { }
class AssetsFileProcessorBackend : FileProcessorBackend, KoinComponent {
override suspend fun process(resource: Resource, assets: Assets, fileProcessor: FileProcessor2) {
Log.debug { "Call processor '${fileProcessor.filename}' on $resource" }
if (assets.exists(fileProcessor.filename)) {
Log.info { "${fileProcessor.filename} already exists on ${resource}. Skipping" }
} else {
Log.info { "${fileProcessor.filename} does not yet exists for $resource" }
val newResource = fileProcessor.process(resource)
assets.store(newResource)
}
}
}

84
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemAssetManager.kt
View File

@ -1,84 +0,0 @@
package de.itkl.assetmanager.implementation
import de.itkl.assetmanager.interfaces.AssetManager
import de.itkl.core_api.interfaces.assets.Assets
import de.itkl.core_api.interfaces.Resource
import de.itkl.core_api.interfaces.ResourceFactory
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.FlowCollector
import kotlinx.coroutines.flow.emitAll
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.stream.consumeAsFlow
import kotlinx.coroutines.withContext
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import kotlin.io.path.deleteExisting
import kotlin.io.path.exists
import kotlin.io.path.outputStream
private val Log = KotlinLogging.logger { }
class FilesystemAssetManager: AssetManager {
override suspend fun assets(name: String): Assets {
val path = createAssetsPath(name)
withContext(Dispatchers.IO) {
Files.createDirectories(path)
}
return FilesystemAssets(path)
}
override suspend fun delete(name: String) {
val path = createAssetsPath(name)
withContext(Dispatchers.IO) {
Files.delete(path)
}
}
private fun createAssetsPath(name: String): Path {
return Paths.get(name).parent.resolve("$name.assets.d").toAbsolutePath()
}
}
class FilesystemAssets(private val baseDir: Path) : Assets, KoinComponent {
private val resourceFactory by inject<ResourceFactory>()
override suspend fun store(resource: Resource) {
val destination = baseDir.resolve(resource.filename)
resource.read().use { source ->
destination.outputStream().use {output ->
withContext(Dispatchers.IO) {
source.copyTo(output)
}
}
}
}
override suspend fun retrieve(name: String): Resource? {
val destination = baseDir.resolve(name)
if (!destination.exists()) {
return null
}
Log.debug { "Loading file at $destination" }
val resource = resourceFactory.file(destination)
return resource
}
override suspend fun delete(name: String) {
val destination = baseDir.resolve(name)
withContext(Dispatchers.IO) {
destination.deleteExisting()
}
}
override suspend fun collect(collector: FlowCollector<Resource>) {
val flow = withContext(Dispatchers.IO) {
Files.list(baseDir).consumeAsFlow()
}
.map { path -> resourceFactory.file(path) }
collector.emitAll(flow)
}
}

64
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/implementation/FilesystemProjectManager.kt
View File

@ -1,64 +0,0 @@
package de.itkl.assetmanager.implementation
import de.itkl.assetmanager.interfaces.AssetManager
import de.itkl.core_api.interfaces.assets.Assets
import de.itkl.assetmanager.interfaces.Project
import de.itkl.assetmanager.interfaces.ProjectManager
import de.itkl.core_api.interfaces.Resource
import de.itkl.core_api.interfaces.ResourceFactory
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.nio.file.Paths
import kotlin.io.path.isDirectory
import kotlin.io.path.isRegularFile
import kotlin.io.path.listDirectoryEntries
private val Log = KotlinLogging.logger { }
class FilesystemProjectManager : ProjectManager {
override suspend fun load(name: String): Project {
val path = Paths.get(name)
check(path.isDirectory()) {
"Currently only directories as corpora are supported"
}
val documents =
withContext(Dispatchers.IO) {
path.listDirectoryEntries()
.filter { it.isRegularFile() }
.map { it.toAbsolutePath() }
.map { it.toString() }
}
return FilesystemProject(
name = name,
displayName = path.fileName.toString(),
documentNames = documents)
}
}
class FilesystemProject(
override val name: String,
override val displayName: String,
override val documentNames: List<String>
) : Project, KoinComponent {
private val basePath = Paths.get(name)
private val assetManager: AssetManager by inject()
private val resourceFactory: ResourceFactory by inject()
override fun resolveName(name: String): String {
return basePath.resolve(name).toAbsolutePath().toString()
}
override suspend fun assets(documentName: String): Assets {
return assetManager.assets(documentName)
}
override suspend fun resource(name: String): Resource? {
Log.debug { "Project: opening resource of name $name" }
return resourceFactory.file(basePath.resolve(name))
}
}

11
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/interfaces/AssetManager.kt
View File

@ -1,11 +0,0 @@
package de.itkl.assetmanager.interfaces
import de.itkl.core_api.interfaces.assets.Assets
/**
* Manage the assets for one document
*/
interface AssetManager {
suspend fun assets(name: String): Assets
suspend fun delete(name: String)
}

18
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/interfaces/Project.kt
View File

@ -1,18 +0,0 @@
package de.itkl.assetmanager.interfaces
import de.itkl.core_api.interfaces.Resource
import de.itkl.core_api.interfaces.assets.Assets
/**
* A set of documents. Each can hold its own assets
*/
interface Project {
val name: String
val displayName: String
val documentNames: List<String>
fun resolveName(name: String): String
suspend fun assets(documentName: String): Assets
suspend fun resource(name: String): Resource?
}

5
libraries/assetmanager/src/main/kotlin/de/itkl/assetmanager/interfaces/ProjectManager.kt
View File

@ -1,5 +0,0 @@
package de.itkl.assetmanager.interfaces
interface ProjectManager {
suspend fun load(name: String): Project
}

0
libraries/clients/build.gradle.kts Normal file
View File

6
libraries/clients/src/main/kotlin/de/itkl/clients/MsOcr.kt Normal file
View File

@ -0,0 +1,6 @@
package de.itkl.clients
class MsOcr {
suspend fun ocr() {}
}

4
libraries/core-api/build.gradle.kts
View File

@ -1,7 +1,3 @@
plugins {
kotlin("plugin.serialization") version embeddedKotlinVersion
}
dependencies {
// used for contentType
api("io.ktor:ktor-http-jvm:2.3.7")

3
libraries/core-api/src/main/kotlin/de/itkl/core_api/coreApiModule.kt
View File

@ -1,8 +1,11 @@
package de.itkl.core_api
import de.itkl.core_api.interfaces.NoopResourceReadDecorator
import de.itkl.core_api.interfaces.ResourceFactory
import de.itkl.core_api.interfaces.ResourceReadDecorator
import org.koin.dsl.module
val coreApiModule = module {
single<ResourceFactory> { ResourceFactory()}
single<ResourceReadDecorator> { NoopResourceReadDecorator() }
}

80
libraries/core-api/src/main/kotlin/de/itkl/core_api/dtos/MsOcrResponse.kt
View File

@ -1,80 +0,0 @@
package de.itkl.core_api.dtos
import kotlinx.datetime.Instant
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@Serializable
data class MsOcrResponse(
@SerialName("analyzeResult")
val analyzeResult: AnalyzeResult,
@SerialName("createdDateTime")
val createdDateTime: Instant, // 2023-12-29T21:02:30Z
@SerialName("lastUpdatedDateTime")
val lastUpdatedDateTime: Instant, // 2023-12-29T21:02:31Z
@SerialName("status")
val status: String // succeeded
) {
@Serializable
data class AnalyzeResult(
@SerialName("modelVersion")
val modelVersion: String, // 2022-04-30
@SerialName("readResults")
val readResults: List<ReadResult>,
@SerialName("version")
val version: String // 3.2.0
) {
@Serializable
data class ReadResult(
@SerialName("angle")
val angle: Int, // 0
@SerialName("height")
val height: Int, // 3507
@SerialName("lines")
val lines: List<Line>,
@SerialName("page")
val page: Int, // 1
@SerialName("unit")
val unit: String, // pixel
@SerialName("width")
val width: Int // 2481
) {
@Serializable
data class Line(
@SerialName("appearance")
val appearance: Appearance,
@SerialName("boundingBox")
val boundingBox: List<Int>,
@SerialName("text")
val text: String, // Franz Mustermann
@SerialName("words")
val words: List<Word>
) {
@Serializable
data class Appearance(
@SerialName("style")
val style: Style
) {
@Serializable
data class Style(
@SerialName("confidence")
val confidence: Double, // 0.972
@SerialName("name")
val name: String // other
)
}
@Serializable
data class Word(
@SerialName("boundingBox")
val boundingBox: List<Int>,
@SerialName("confidence")
val confidence: Double, // 0.998
@SerialName("text")
val text: String // Franz
)
}
}
}
}

34
libraries/core-api/src/main/kotlin/de/itkl/core_api/implementation/SerializableResource.kt
View File

@ -1,34 +0,0 @@
package de.itkl.core_api.implementation
import de.itkl.core_api.interfaces.Resource
import io.ktor.http.*
import kotlinx.serialization.*
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.encodeToStream
import java.io.File
import java.io.InputStream
import java.io.UnsupportedEncodingException
import java.nio.file.Path
class SerializableResource<T : Any> @OptIn(ExperimentalSerializationApi::class) constructor(
override val filename: String,
override val contentType: ContentType,
private val obj: T,
private val serializer: SerializationStrategy<T>
) : Resource {
override val length: Long? = null
override val file: File? = null
override val path: Path? = null
override fun read(): InputStream {
return serialize().byteInputStream()
}
private fun serialize(): String {
return when(contentType) {
ContentType.Application.Json -> Json.encodeToString(serializer, obj)
else -> throw UnsupportedEncodingException("Sorry but $contentType is not supported for Resources")
}
}
}

6
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/FileProcessor.kt
View File

@ -2,14 +2,8 @@ package de.itkl.core_api.interfaces
import java.io.File
import java.nio.file.Path
import java.util.function.Consumer
interface FileProcessor {
fun willProduce(path: Path): Path
suspend fun process(resource: Resource): File
}
interface FileProcessor2 {
val filename: String
suspend fun process(resource: Resource): Resource
}

23
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/Resource.kt
View File

@ -1,15 +1,11 @@
package de.itkl.core_api.interfaces
import io.ktor.http.*
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.KSerializer
import kotlinx.serialization.json.Json
import org.koin.core.component.KoinComponent
import org.koin.core.component.get
import java.io.File
import java.io.InputStream
import java.nio.file.Path
import kotlin.reflect.KClass
interface Resource {
val filename: String
@ -19,16 +15,8 @@ interface Resource {
val file: File?
val path: Path?
fun read(): InputStream
fun <T: Any> json(deserializer: DeserializationStrategy<T>): T {
val string = String(read().readAllBytes())
return Json.decodeFromString(deserializer, string)
}
}
/**
* Automatically adds koin injectable decorators to reading/writing
* operations
@ -36,10 +24,11 @@ interface Resource {
abstract class AbstractResource : Resource, KoinComponent {
abstract fun doRead(): InputStream
final override fun read(): InputStream {
return doRead()
}
override fun toString(): String {
return filename
return length?.let { length ->
get<ResourceReadDecorator>().decorate(
length = length,
doRead()
)
} ?: doRead()
}
}

18
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/ResourceFactory.kt
View File

@ -2,31 +2,13 @@ package de.itkl.core_api.interfaces
import de.itkl.core_api.implementation.FileResource
import de.itkl.core_api.implementation.ProgressResource
import de.itkl.core_api.implementation.SerializableResource
import io.ktor.http.*
import kotlinx.serialization.SerializationStrategy
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.File
import java.nio.file.Path
import java.nio.file.Paths
class ResourceFactory : KoinComponent {
private val progressBarFactory by inject<ProgressBarFactory>()
fun <T : Any> json(name: String, obj: T, serializationStrategy: SerializationStrategy<T>): Resource {
return SerializableResource<T>(
filename = name,
contentType = ContentType.Application.Json,
obj = obj,
serializer = serializationStrategy)
}
fun file(path: String): Resource {
return file(Paths.get(path))
}
fun file(path: Path): Resource {
return file(path.toFile())
}
fun file(file: File): Resource {
val resource = FileResource(file)
return ProgressResource(resource, progressBarFactory)

12
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/ResourceReadDecorator.kt
View File

@ -1,3 +1,15 @@
package de.itkl.core_api.interfaces
import java.io.InputStream
interface ResourceReadDecorator {
fun decorate(
length: Long,
inputStream: InputStream): InputStream
}
class NoopResourceReadDecorator : ResourceReadDecorator {
override fun decorate(length: Long, inputStream: InputStream): InputStream {
return inputStream
}
}

15
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/assets/Assets.kt
View File

@ -1,15 +0,0 @@
package de.itkl.core_api.interfaces.assets
import de.itkl.core_api.interfaces.Resource
import kotlinx.coroutines.flow.Flow
import java.util.function.Consumer
interface Assets : Flow<Resource> {
suspend fun store(resource: Resource)
suspend fun retrieve(name: String): Resource?
suspend fun delete(name: String)
suspend fun exists(name: String): Boolean {
return retrieve(name) != null
}
}

16
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/assets/FileProcessorBackend.kt
View File

@ -1,16 +0,0 @@
package de.itkl.core_api.interfaces.assets
import de.itkl.core_api.interfaces.FileProcessor
import de.itkl.core_api.interfaces.FileProcessor2
import de.itkl.core_api.interfaces.Resource
/**
* Executes a [FileProcessor2] on a [Resource]. It decides if and when
* the [FileProcessor2.process] should be called and what should happen with the result
*/
interface FileProcessorBackend {
suspend fun process(
resource: Resource,
assets: Assets,
fileProcessor: FileProcessor2)
}

4
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/data/DataTable.kt
View File

@ -1,4 +0,0 @@
package de.itkl.core_api.interfaces.data
interface DataTable : Iterable<List<String>> {
val columns: List<String>
}

8
libraries/core-api/src/main/kotlin/de/itkl/core_api/interfaces/data/Processable.kt
View File

@ -1,8 +0,0 @@
package de.itkl.core_api.interfaces.data
import de.itkl.core_api.interfaces.FileProcessor
import de.itkl.core_api.interfaces.FileProcessor2
interface Processable {
suspend fun process(fileProcessor: FileProcessor2)
}

15
libraries/docthor-core/build.gradle.kts
View File

@ -1,15 +0,0 @@
dependencies {
fun addProjects(vararg names: String) {
names.forEach {
api(project(":libraries:$it"))
}
}
addProjects(
"assetmanager",
"core-api",
"textprocessing",
"httpClient",
"tui",
)
}

36
libraries/docthor-core/src/main/kotlin/de/itkl/docthor/core/DocumentViewer.kt
View File

@ -1,36 +0,0 @@
package de.itkl.docthor.core
import de.itkl.assetmanager.assetManagerModule
import de.itkl.core_api.coreApiModule
import de.itkl.httpClient.clients.MsOcr
import de.itkl.httpClient.httpClientModule
import de.itkl.textprocessing.CorpusFactory
import de.itkl.textprocessing.Document
import de.itkl.textprocessing.textProcessingModule
import de.itkl.tui.tuiModule
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.koin.core.context.startKoin
import java.nio.file.Path
class DocumentViewer : KoinComponent {
init {
startKoin {
modules(
coreApiModule,
textProcessingModule,
tuiModule,
assetManagerModule,
httpClientModule
)
}
}
suspend fun loadTestDocument(corpusPath: Path, documentName: String): Document {
val corpus = CorpusFactory().load(corpusPath.toAbsolutePath().toString())
val document = corpus.document(documentName)
val ocrExtractor: MsOcr by inject()
document.process(ocrExtractor)
return document
}
}

18
libraries/httpClient/build.gradle.kts
View File

@ -1,18 +0,0 @@
plugins {
kotlin("plugin.serialization") version embeddedKotlinVersion
}
val ktorVersion: String by project
dependencies {
api(project(":libraries:core-api"))
api("io.ktor:ktor-client-core:$ktorVersion")
api("io.ktor:ktor-client-core-jvm:$ktorVersion")
implementation("io.ktor:ktor-client-cio:$ktorVersion")
implementation("io.ktor:ktor-client-content-negotiation:$ktorVersion")
implementation("io.ktor:ktor-serialization-kotlinx-json:$ktorVersion")
implementation("com.akuleshov7:ktoml-core:0.5.1")
implementation("com.akuleshov7:ktoml-file:0.5.1")
implementation("com.github.ben-manes.caffeine:caffeine:3.1.8")
}

1
libraries/httpClient/gradle.properties
View File

@ -1 +0,0 @@
ktorVersion=2.3.7

4
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/BearerTokenCache.kt
View File

@ -1,4 +0,0 @@
package de.itkl.httpClient
class BearerTokenCache {
}

22
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/SmartCloudAuthPlugin.kt
View File

@ -1,22 +0,0 @@
package de.itkl.httpClient
import de.itkl.httpClient.implementation.SmartCloudAuthStrategy
import io.github.oshai.kotlinlogging.KotlinLogging
import io.ktor.client.plugins.api.*
import io.ktor.client.request.*
import io.ktor.http.*
import io.ktor.util.*
import org.koin.core.component.KoinComponent
private val Log = KotlinLogging.logger { }
val smartCloudAuthPlugin = createClientPlugin("SmartCloudAuth") {
val smartCloudAuthStrategy = SmartCloudAuthStrategy()
on(Send) { request ->
smartCloudAuthStrategy.login(client, request)?.let { token ->
request.headers {
set(HttpHeaders.Authorization, token.toBearer())
}
}
proceed(request)
}
}

12
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/TrustAllX509TrustManager.kt
View File

@ -1,12 +0,0 @@
package de.itkl.httpClient
import java.security.cert.X509Certificate
import javax.net.ssl.X509TrustManager
class TrustAllX509TrustManager : X509TrustManager {
override fun getAcceptedIssuers(): Array<X509Certificate?> = arrayOfNulls(0)
override fun checkClientTrusted(certs: Array<X509Certificate?>?, authType: String?) {}
override fun checkServerTrusted(certs: Array<X509Certificate?>?, authType: String?) {}
}

11
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/auth/AuthStrategy.kt
View File

@ -1,11 +0,0 @@
package de.itkl.httpClient.auth
import io.ktor.client.*
import io.ktor.client.request.*
interface AuthStrategy {
suspend fun login(
httpClient: HttpClient,
request: HttpRequestBuilder
): AuthenticationToken?
}

23
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/auth/AuthenticationToken.kt
View File

@ -1,23 +0,0 @@
package de.itkl.httpClient.auth
import kotlinx.datetime.Instant
import kotlinx.serialization.Serializable
sealed class AuthenticationToken {
abstract val expires: Expires
abstract fun toBearer(): String
}
@Serializable
data class BearerToken(val token: String, val validUntil: Instant) : AuthenticationToken() {
override val expires: Expires
get() = Expires.ExpiresAt(validUntil)
override fun toBearer(): String {
return "Bearer $token"
}
}
sealed class Expires {
data object Never : Expires()
data class ExpiresAt(val instant: Instant) : Expires()
}

17
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/auth/Authenticator.kt
View File

@ -1,17 +0,0 @@
package de.itkl.httpClient.auth
import io.ktor.http.*
import org.koin.core.component.KoinComponent
typealias UrlMatcher = (Url) -> Boolean
class Authenticator : KoinComponent {
fun addStrategy(strategy: AuthStrategy, urlMatcher: UrlMatcher) {}
fun requiresAuthentication(url: Url): Boolean {
TODO()
}
suspend fun authenticate(url: Url, username: String): AuthenticationToken? {
TODO()
}
}

8
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/auth/Credentials.kt
View File

@ -1,8 +0,0 @@
package de.itkl.httpClient.auth
import kotlinx.serialization.Serializable
sealed class Credentials {
@Serializable
data class LoginAndPassword(val username: String, val password: String) : Credentials()
}

5
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/auth/CredentialsProvider.kt
View File

@ -1,5 +0,0 @@
package de.itkl.httpClient.auth
interface CredentialsProvider {
suspend fun lookupByUsername(username: String): Credentials?
}

47
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/clients/MsOcr.kt
View File

@ -1,47 +0,0 @@
package de.itkl.httpClient.clients
import de.itkl.core_api.dtos.MsOcrResponse
import de.itkl.core_api.interfaces.FileProcessor
import de.itkl.core_api.interfaces.FileProcessor2
import de.itkl.core_api.interfaces.Resource
import de.itkl.core_api.interfaces.ResourceFactory
import io.github.oshai.kotlinlogging.KotlinLogging
import io.ktor.client.*
import io.ktor.client.call.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.client.utils.EmptyContent.contentType
import io.ktor.http.*
import kotlinx.serialization.json.Json
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.File
import java.nio.file.Path
import kotlin.io.path.nameWithoutExtension
import kotlin.io.path.writeText
private val Log = KotlinLogging.logger { }
class MsOcr: KoinComponent, FileProcessor2 {
private val httpClient: HttpClient by inject()
private val resourceFactory: ResourceFactory by inject()
suspend fun ocr(resource: Resource): MsOcrResponse {
val response = httpClient.post {
url("http://10.54.150.152:5000/vision/v3.2/read/syncAnalyze")
parameters {
append("language", "de")
append("readingOrder", "natural")
}
contentType(resource.contentType)
setBody(resource.read())
}
return response.body()
}
override val filename = "ms-ocr.json"
override suspend fun process(resource: Resource): Resource {
val result = ocr(resource)
return resourceFactory.json(filename, result, MsOcrResponse.serializer())
}
}

9
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/clients/RestClient.kt
View File

@ -1,9 +0,0 @@
package de.itkl.httpClient.clients
import io.ktor.client.*
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
abstract class RestClient : KoinComponent {
private val httpClient by inject<HttpClient>()
}

134
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/clients/XsClient.kt
View File

@ -1,134 +0,0 @@
package de.itkl.httpClient.clients
import io.github.oshai.kotlinlogging.KotlinLogging
import io.ktor.client.*
import io.ktor.client.call.*
import io.ktor.client.request.*
import io.ktor.client.request.forms.*
import io.ktor.client.statement.*
import io.ktor.client.utils.EmptyContent.contentType
import io.ktor.http.*
import io.ktor.http.content.*
import io.ktor.util.*
import io.ktor.util.cio.*
import io.ktor.utils.io.*
import io.ktor.utils.io.streams.*
import kotlinx.serialization.Serializable
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.awt.SystemColor.info
import java.nio.file.Files
import java.nio.file.Path
import java.util.UUID
import kotlin.io.path.exists
import kotlin.io.path.isRegularFile
import kotlin.io.path.name
private val Log = KotlinLogging.logger { }
class XsClient : KoinComponent {
private val httpClient by inject<HttpClient>()
suspend fun waitFor(task: XsTask): WaitForResponse {
Log.info { "Wait for competition: $task" }
val response = httpClient.get {
url("http://localhost:8080/api/v1/analyse/tasks/wait/${task.xsTaskId.taskId}")
user = "xs.dev.klara"
}
val result = response.body<WaitForResponse>()
Log.info { "Waiting done for task $task: ${response.status}: $result" }
return result
}
suspend fun analyseResult(task: XsTask) {
val response = httpClient.get {
url("http://localhost:8080/api/v1/analyse-async-result/${task.xsTaskId.taskId}")
user = "xs.dev.klara"
}
check(response.status.isSuccess()) {
"HTTP Error"
}
val text = response.bodyAsText()
println(text)
}
suspend fun analyse(image: Path): TaskReference {
Log.info { "Starting analysis for image at path: $image" }
check(image.isRegularFile()) {
"The provided path $image is not a file"
}
Log.info { "Submitting form with binary data to http://localhost:8080/api/v1/analyse-async" }
val response = httpClient.submitFormWithBinaryData(
url = "http://localhost:8080/api/v1/analyse-async",
formData {
append(
"image",
image.toChannelProvider(),
Headers.build {
append(HttpHeaders.ContentType, ContentType.defaultForFile(image))
append(HttpHeaders.ContentDisposition, """filename="${image.name}"""")
}
)
}
) {
user = "xs.dev.klara"
}
Log.info { "Received the response from the server." }
if(response.status.isSuccess()) {
Log.info { "Successful response with status: ${response.status}" }
return response.body()
} else {
val responseText = response.bodyAsText()
Log.warn { "Error creating analyse task: ${response.status}: $responseText" }
error("Could not create analyse task: $responseText")
}
}
}
var HttpRequestBuilder.user: String?
get() = this.attributes[AttributeKey(("username"))]
set(value) = value?.let { this.attributes.put(AttributeKey("username"), it) } ?: this.attributes.remove(
AttributeKey("username")
)
interface XsTask {
val xsTaskId: XsTaskId
}
@Serializable
data class TaskReference(
override val xsTaskId: XsTaskId
) : XsTask {
override fun toString(): String {
return "Task($xsTaskId)"
}
}
@Serializable
data class XsTaskId(val tenantId: String, val taskId: String) {
override fun toString(): String {
return "$tenantId/$taskId"
}
}
@Serializable
data class WaitForResponse(
val xsTaskId: XsTaskId,
val status: TaskWaitStatus,
)
@Serializable
enum class TaskWaitStatus {
ERROR,
SUCCESS
}
fun Path.toChannelProvider(): ChannelProvider {
val file = toFile()
return ChannelProvider(file.length()) { file.readChannel() }
}

41
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/createHttpClient.kt
View File

@ -1,41 +0,0 @@
package de.itkl.httpClient
import io.github.oshai.kotlinlogging.KotlinLogging
import io.ktor.client.*
import io.ktor.client.engine.*
import io.ktor.client.engine.cio.*
import io.ktor.client.plugins.*
import io.ktor.client.plugins.contentnegotiation.*
import io.ktor.http.*
import io.ktor.serialization.kotlinx.json.*
import java.net.InetSocketAddress
import java.net.Socket
import kotlin.time.Duration.Companion.minutes
private val Log = KotlinLogging.logger { }
fun createHttpClient(): HttpClient {
return HttpClient(CIO) {
install(ContentNegotiation) {
json()
}
install(smartCloudAuthPlugin)
install(HttpTimeout) {
requestTimeoutMillis = 30.minutes.inWholeMilliseconds
}
engine {
val isPortOpen = try {
Socket().use { it.connect(InetSocketAddress("localhost", 9999), 200) }
true
} catch (ex: Exception) {
false
}
if (isPortOpen) {
proxy = ProxyBuilder.http(Url("http://localhost:9999"))
}
https {
trustManager = TrustAllX509TrustManager()
}
}
}
}

20
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/httpClientModule.kt
View File

@ -1,20 +0,0 @@
package de.itkl.httpClient
import de.itkl.httpClient.auth.CredentialsProvider
import de.itkl.httpClient.clients.MsOcr
import de.itkl.httpClient.clients.XsClient
import de.itkl.httpClient.implementation.StaticCredentialsProvider
import io.ktor.client.*
import org.koin.dsl.module
import java.nio.file.Paths
val httpClientModule = module {
single<HttpClient> { createHttpClient() }
single<MsOcr> { MsOcr() }
single<CredentialsProvider> {
val homeDirectory = System.getProperty("user.home")
val credentialsFilePath = Paths.get(homeDirectory, ".auth.toml")
StaticCredentialsProvider.load(credentialsFilePath)
}
single<XsClient> { XsClient() }
}

72
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/implementation/SmartCloudAuthStrategy.kt
View File

@ -1,72 +0,0 @@
package de.itkl.httpClient.implementation
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.Cache
import de.itkl.httpClient.auth.*
import de.itkl.httpClient.clients.user
import io.github.oshai.kotlinlogging.KotlinLogging
import io.ktor.client.*
import io.ktor.client.call.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.util.*
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.util.concurrent.TimeUnit
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
private val Log = KotlinLogging.logger { }
class SmartCloudAuthStrategy : AuthStrategy, KoinComponent {
private val credentialsProvider: CredentialsProvider by inject()
private val loginMutex = Mutex()
// Cache Helpers
private val tokenCache: Cache<String, AuthenticationToken> = Caffeine.newBuilder()
.expireAfterWrite(1, TimeUnit.HOURS)
.build()
override suspend fun login(httpClient: HttpClient, request: HttpRequestBuilder): AuthenticationToken? {
Log.debug { "Attempting login..." }
val user = request.attributes.getOrNull(AttributeKey<String>("username")) ?: run {
Log.info { "No username is specified for this request" }
return null
}
tokenCache.getIfPresent(user)?.let {
Log.info { "Returning cached token for user: $user" }
return it
}
return loginMutex.withLock {
tokenCache.getIfPresent(user)?.let {
Log.info { "Returning cached token for user: $user" }
return it
}
val credentials: Credentials = credentialsProvider.lookupByUsername(user) ?: error("No credentials found for user: $user")
val loginAndPassword: Credentials.LoginAndPassword = credentials as? Credentials.LoginAndPassword ?: error("Only username and password is supported by smartcloud auth login")
Log.debug { "User: $user, using smartcloud auth login" }
val response = httpClient.post {
url("https://api.internal.insiders.cloud/1/rest/accounts/authentication/requesttoken")
contentType(ContentType.Application.Json)
setBody(loginAndPassword)
}
check(response.status.isSuccess()) {
"could not login into smart cloud: ${response.bodyAsText()}"
}
val token = response.body<BearerToken>()
Log.info { "Login successful. Valid until: ${token.validUntil}" }
// Cache the token after successful login
tokenCache.put(user, token)
token
}
}
}

49
libraries/httpClient/src/main/kotlin/de/itkl/httpClient/implementation/StaticCredentialsProvider.kt
View File

@ -1,49 +0,0 @@
package de.itkl.httpClient.implementation
import com.akuleshov7.ktoml.Toml
import de.itkl.httpClient.auth.Credentials
import de.itkl.httpClient.auth.CredentialsProvider
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromString
import java.nio.file.Files
import java.nio.file.Path
private val Log = KotlinLogging.logger { }
class StaticCredentialsProvider(private val credentials: Map<String, Credentials.LoginAndPassword>) : CredentialsProvider {
companion object {
fun load(path: Path): StaticCredentialsProvider {
Log.info { "Loading credentials from path: $path" }
val content = Files.readString(path)
val credentials = Toml.decodeFromString<CredentialsTable>(content).toCredentialsMap()
Log.info { "Credentials loaded successfully" }
return StaticCredentialsProvider(credentials)
}
}
override suspend fun lookupByUsername(username: String): Credentials? {
Log.info { "Looking up credentials by username: $username" }
val credentialsEntry = credentials[username]
if (credentialsEntry != null) {
Log.info { "Credentials found for username: $username" }
return credentialsEntry
}
Log.info { "No credentials found for username: $username" }
return null
}
}
@Serializable
data class CredentialsTable(
val credentials: Map<String, Map<String, String>>
) {
fun toCredentialsMap(): Map<String, Credentials.LoginAndPassword> {
return credentials.mapValues {
val login = it.value["Login"] ?: throw IllegalArgumentException("Missing login")
val password = it.value["Password"] ?: throw IllegalArgumentException("Missing password")
Credentials.LoginAndPassword(login, password)
}
}
}

36
libraries/httpClient/src/test/kotlin/de/itkl/httpClient/clients/MsOcrTest.kt
View File

@ -1,36 +0,0 @@
package de.itkl.httpClient.clients
import de.itkl.core_api.coreApiModule
import de.itkl.core_api.implementation.FileResource
import de.itkl.core_api.interfaces.Resource
import de.itkl.httpClient.httpClientModule
import kotlinx.coroutines.runBlocking
import org.junit.Rule
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.koin.core.component.inject
import org.koin.core.context.startKoin
import org.koin.test.KoinTest
import java.nio.file.Paths
class MsOcrTest : KoinTest {
@BeforeEach
fun start() {
startKoin {
printLogger()
modules(
coreApiModule,
httpClientModule)
}
}
@Test
fun `can create a request`() = runBlocking {
val msOcrClient: MsOcr by inject()
val resource = FileResource(Paths.get("../../assets/xs-reg/00001.jpg").toAbsolutePath())
val response = msOcrClient.ocr(resource)
println(response)
Unit
}
}

25
libraries/httpClient/src/test/kotlin/de/itkl/httpClient/implementation/StaticCredentialsProviderTest.kt
View File

@ -1,25 +0,0 @@
package de.itkl.httpClient.implementation
import assertk.assertThat
import assertk.assertions.isDataClassEqualTo
import assertk.assertions.isNotNull
import de.itkl.httpClient.auth.Credentials
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.Test
import java.nio.file.Paths
class StaticCredentialsProviderTest {
@Test
fun `can load credentials file`() = runBlocking {
val target = StaticCredentialsProvider.load(Paths.get("../../assets/credentials.toml"))
val credentials = target.lookupByUsername("test user")
assertThat(credentials)
.isNotNull()
.isDataClassEqualTo(Credentials.LoginAndPassword("TestiTest", "Secret"))
Unit
}
}

3
libraries/io/build.gradle.kts Normal file
View File

@ -0,0 +1,3 @@
dependencies {
api(project(":libraries:core-api"))
}

19
libraries/io/src/main/kotlin/de/itkl/io/implementation/FileSystemResource.kt Normal file
View File

@ -0,0 +1,19 @@
package de.itkl.io.implementation
import de.itkl.core_api.interfaces.Resource
import io.ktor.http.*
import java.io.File
import java.io.InputStream
class FileSystemResource(private val file: File) : Resource() {
override val filename: String
get() = file.name
override val contentType: ContentType
get() = ContentType.fromFilePath(file.path).first()
override val length: Long
get() = file.length()
override fun doRead(): InputStream {
return file.inputStream()
}
}

9
libraries/io/src/main/kotlin/de/itkl/io/ioModule.kt Normal file
View File

@ -0,0 +1,9 @@
package de.itkl.io
import de.itkl.core_api.interfaces.NoopResourceReadDecorator
import de.itkl.core_api.interfaces.ResourceReadDecorator
import org.koin.dsl.module
val ioModule = module {
single<ResourceReadDecorator> { NoopResourceReadDecorator() }
}

3
libraries/textprocessing/build.gradle.kts
View File

@ -1,9 +1,6 @@
dependencies {
api(project(":libraries:core-api"))
api("org.apache.lucene:lucene-analysis-common:9.9.0")
api("io.github.piruin:geok:1.2.2")
api(project(":libraries:assetmanager"))
api("com.soywiz.korge:korge-foundation:5.1.0")
implementation("com.github.doyaaaaaken:kotlin-csv-jvm:1.9.2")
implementation("com.google.guava:guava:32.1.3-jre")
}

37
libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Corpus.kt
View File

@ -1,37 +0,0 @@
package de.itkl.textprocessing
import de.itkl.assetmanager.interfaces.Project
import de.itkl.assetmanager.interfaces.ProjectManager
import de.itkl.core_api.interfaces.FileProcessor
import de.itkl.core_api.interfaces.ResourceFactory
import de.itkl.core_api.interfaces.data.Processable
import io.github.oshai.kotlinlogging.KotlinLogging
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.koin.java.KoinJavaComponent.inject
import java.nio.file.Paths
private val Log = KotlinLogging.logger { }
class CorpusFactory : KoinComponent {
private val projectManager: ProjectManager by inject()
suspend fun load(name: String): Corpus {
Log.info { "Open corpus at ${Paths.get(name).toAbsolutePath()}" }
return Corpus(projectManager.load(name)).apply {
Log.debug { "Found documents in corpus: ${this.documentNames.joinToString("\n")}" }
}
}
}
class Corpus(private val project: Project): KoinComponent {
val displayName get() = project.displayName
val documentNames get() = project.documentNames
private val resourceFactory: ResourceFactory by inject()
suspend fun document(name: String): Document {
return Document(
project.resolveName(name),
listOf(project.resource(name)!!)
)
}
}

101
libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Document.kt
View File

@ -1,103 +1,4 @@
package de.itkl.textprocessing
import de.itkl.assetmanager.interfaces.AssetManager
import de.itkl.core_api.dtos.MsOcrResponse
import de.itkl.core_api.interfaces.FileProcessor
import de.itkl.core_api.interfaces.FileProcessor2
import de.itkl.core_api.interfaces.Resource
import de.itkl.core_api.interfaces.assets.Assets
import de.itkl.core_api.interfaces.assets.FileProcessorBackend
import de.itkl.core_api.interfaces.data.Processable
import korlibs.math.geom.Rectangle
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.filter
import me.piruin.geok.LatLng
import me.piruin.geok.geometry.Polygon
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
class Document(
val name: String,
val resources: List<Resource>
) : Processable, KoinComponent {
private val assetManager: AssetManager by inject()
private val fileProcessorBackend: FileProcessorBackend by inject()
suspend fun assets(): Assets {
return assetManager.assets(name)
}
/**
* Loads the extracted ocr pages. Note that not every pages
* needs to have ocr
*/
suspend fun retrieveOcrPages(): List<OcrPage> {
// TODO: How to identify the assets independently from their name?
val resource = checkNotNull(assets()
.retrieve("ms-ocr.json")) {
"Ocr for $name is not yet created"
}
val msOcrResponse = resource.json(MsOcrResponse.serializer())
return msOcrResponse.analyzeResult.readResults.map { toOcrPage(it) }
}
override suspend fun process(fileProcessor: FileProcessor2) {
fileProcessorBackend.process(
resources.first(),
assets(),
fileProcessor
)
}
private fun toOcrPage(readResult: MsOcrResponse.AnalyzeResult.ReadResult): OcrPage {
return OcrPage(
pageNumber = readResult.page,
width = readResult.width,
height = readResult.height,
words = readResult.lines.flatMap { line -> line.words.map { toOcrWord(it) } }
)
}
private fun toOcrWord(word: MsOcrResponse.AnalyzeResult.ReadResult.Line.Word): OcrPage.OcrWord {
val box = word.boundingBox
return OcrPage.OcrWord(
Rectangle(
x = box[0],
y = box[1],
width = box[2] - box[0],
height = box[7] - box[1]),
// polygon = Polygon(listOf(
// LatLng(box[0].toDouble(), box[1].toDouble()),
// LatLng(box[2].toDouble(), box[3].toDouble()),
// LatLng(box[4].toDouble(), box[5].toDouble()),
// LatLng(box[6].toDouble(), box[7].toDouble()),
// )),
text = word.text
)
}
}
class OcrPage(
val width: Int,
val height: Int,
val pageNumber: Int,
val words: List<OcrWord>,
// val regions: List<DocumentRegion> = emptyList()
) {
// inner class DocumentRegion(
// private val polygon: Polygon,
// private val type: String,
// ) {
// fun words(): Flow<OcrWord> {
// return words
// .asFlow()
// .filter { word -> word.polygon.intersectionWith(polygon) != null }
// }
// }
fun addOcrWord(rectangle: Rectangle, text: String): OcrWord {
return OcrWord(rectangle, text)
}
class OcrWord(
val rectangle: Rectangle,
val text: String
)
class DocumentContainer {
}

20
libraries/textprocessing/src/main/kotlin/de/itkl/textprocessing/Histogram.kt
View File

@ -2,16 +2,30 @@ package de.itkl.textprocessing
import kotlinx.coroutines.flow.*
class Histogram(
private val histo: MutableMap<String,UInt> = mutableMapOf()
) : Iterable<Pair<String, UInt>>{
class Histogram(private val histo: MutableMap<String,UInt> = mutableMapOf()) : Iterable<Pair<String, UInt>>{
companion object {
suspend fun from(flow: Flow<String>): Histogram {
return Histogram().apply {
flow.collect(this::add)
}
}
fun fromBagOfWords(bagOfWords: BagOfWords): Histogram {
val result = Histogram()
bagOfWords.forEach(result::add)
return result
}
suspend fun fromBagOfWords(flow: Flow<BagOfWords>): Histogram {
val result = Histogram()
flow.collect() { value ->
value.forEach(result::add)
}
return result
}
fun from(sequence: Sequence<Map<String, String>>): Histogram {
val histo = sequence.associate { map -> map["word"]!! to map["count"]!!.toUInt() }
.toMutableMap()

21
settings.gradle.kts
View File

@ -1,21 +1,11 @@
pluginManagement {
repositories {
mavenCentral();
google();
gradlePluginPortal() }
}
//pluginManagement {
// includeBuild("build-logic")
//}
plugins {
id("org.gradle.toolchains.foojay-resolver-convention") version "0.4.0"
}
rootProject.name = "docthor"
fun includeDirs(vararg paths: String) {
paths.forEach(this::includeDir)
}
fun includeDir(path: String) {
file(path)
.listFiles()!!
@ -28,9 +18,8 @@ fun includeDir(path: String) {
}
}
rootProject.name = "docthor"
include(
"app",
)
includeDirs(
"apps",
"libraries")
includeDir("libraries")