Eval

core/src/main/kotlin/by/jprof/telegram/bot/core/UpdateProcessingPipeline.kt

@@ -2,8 +2,9 @@ package by.jprof.telegram.bot.core
 
 import dev.inmo.tgbotapi.types.update.abstracts.Update
 import kotlinx.coroutines.CoroutineExceptionHandler
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.joinAll
-import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.supervisorScope
 import org.apache.logging.log4j.LogManager

eval/build.gradle.kts

@@ -4,6 +4,7 @@ plugins {
 
 dependencies {
     api(project.projects.core)
+    api(project.projects.eval.dto)
     api(libs.tgbotapi.core)
     implementation(project.projects.votes.tgbotapiExtensions)
     implementation(libs.tgbotapi.extensions.api)

eval/dto/build.gradle.kts

@@ -0,0 +1,8 @@
+plugins {
+    kotlin("jvm")
+    kotlin("plugin.serialization")
+}
+
+dependencies {
+    implementation(libs.kotlinx.serialization.core)
+}

eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalEvent.kt

@@ -0,0 +1,8 @@
+package by.jprof.telegram.bot.eval.dto
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class EvalEvent(
+    val code: String,
+)

eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/EvalResponse.kt

@@ -0,0 +1,16 @@
+package by.jprof.telegram.bot.eval.dto
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+sealed class EvalResponse {
+    @Serializable
+    data class Successful(
+        val language: Language,
+        val stdout: String? = null,
+        val stderr: String? = null,
+    ) : EvalResponse()
+
+    @Serializable
+    object Unsuccessful : EvalResponse()
+}

eval/dto/src/main/kotlin/by/jprof/telegram/bot/eval/dto/Language.kt

@@ -0,0 +1,10 @@
+package by.jprof.telegram.bot.eval.dto
+
+enum class Language {
+    KOTLIN,
+    JAVA,
+    JAVASCRIPT,
+    TYPESCRIPT,
+    PYTHON,
+    GO,
+}

eval/evaluator/Dockerfile

@@ -0,0 +1,36 @@
+FROM amazon/aws-lambda-java:latest
+
+RUN yum -y install unzip tar gzip xz && \
+    yum -y clean all  && \
+    rm -rf /var/cache
+
+RUN curl -L https://github.com/JetBrains/kotlin/releases/download/v1.5.31/kotlin-compiler-1.5.31.zip --output /tmp/kotlin.zip --silent && \
+    unzip /tmp/kotlin.zip -d /tmp && \
+	mv /tmp/kotlinc /opt/kotlin && \
+    rm /tmp/kotlin.zip
+
+RUN curl -L https://download.java.net/java/GA/jdk17/0d483333a00540d886896bac774ff48b/35/GPL/openjdk-17_linux-x64_bin.tar.gz --output /tmp/java.tar.gz --silent && \
+    mkdir /opt/java && \
+    tar -xf /tmp/java.tar.gz -C /opt/java --strip-components 1 && \
+    rm /tmp/java.tar.gz
+
+RUN curl -L https://nodejs.org/dist/v16.11.0/node-v16.11.0-linux-x64.tar.xz --output /tmp/node.tar.xz --silent && \
+    mkdir /opt/node && \
+    tar -xf /tmp/node.tar.xz -C /opt/node --strip-components 1 && \
+    rm /tmp/node.tar.xz
+
+ENV PATH="/opt/node/bin:${PATH}"
+RUN npm install -g [email protected]
+
+RUN curl -L https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tar.xz --output /tmp/python.tar.xz --silent && \
+    mkdir /opt/python && \
+    tar -xf /tmp/python.tar.xz -C /opt/python --strip-components 1 && \
+    rm /tmp/python.tar.xz
+
+RUN curl -L https://golang.org/dl/go1.17.2.linux-amd64.tar.gz --output /tmp/go.tar.gz --silent && \
+    mkdir /opt/go && \
+    tar -xf /tmp/go.tar.gz -C /opt/go --strip-components 1 && \
+    rm /tmp/go.tar.gz
+
+COPY build/libs/jprof_by_bot-eval-evaluator-all.jar ${LAMBDA_TASK_ROOT}/lib/
+CMD [ "by.jprof.telegram.bot.eval.evaluator.Evaluator::handleRequest" ]

eval/evaluator/build.gradle.kts

@@ -0,0 +1,18 @@
+plugins {
+    kotlin("jvm")
+    id("com.github.johnrengelman.shadow")
+}
+
+dependencies {
+    api(project.projects.eval.dto)
+    implementation(libs.bundles.aws.lambda)
+    implementation(libs.koin.core)
+    implementation(libs.kotlinx.coroutines.core)
+    implementation(libs.kotlinx.serialization.json)
+    implementation(libs.bundles.log4j)
+
+    testImplementation(libs.junit.jupiter.api)
+    testImplementation(libs.junit.jupiter.params)
+    testRuntimeOnly(libs.junit.jupiter.engine)
+    testRuntimeOnly(libs.log4j.core)
+}

eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/Evaluator.kt

@@ -0,0 +1,53 @@
+package by.jprof.telegram.bot.eval.evaluator
+
+import by.jprof.telegram.bot.eval.dto.EvalEvent
+import by.jprof.telegram.bot.eval.evaluator.config.jsonModule
+import by.jprof.telegram.bot.eval.evaluator.config.pipelineModule
+import by.jprof.telegram.bot.eval.evaluator.middleware.EvalPipeline
+import com.amazonaws.services.lambda.runtime.Context
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.encodeToStream
+import org.apache.logging.log4j.LogManager
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import org.koin.core.context.startKoin
+import java.io.InputStream
+import java.io.OutputStream
+
+@ExperimentalSerializationApi
+@Suppress("unused")
+class Evaluator : RequestStreamHandler, KoinComponent {
+    companion object {
+        private val logger = LogManager.getLogger(Evaluator::class.java)
+    }
+
+    init {
+        startKoin {
+            modules(
+                jsonModule,
+                pipelineModule,
+            )
+        }
+    }
+
+    private val json: Json by inject()
+    private val pipeline: EvalPipeline by inject()
+
+    override fun handleRequest(input: InputStream, output: OutputStream, context: Context) = runBlocking {
+        val payload = input.bufferedReader().use { it.readText() }
+
+        logger.debug("Payload: {}", payload)
+
+        val evalEvent = json.decodeFromString<EvalEvent>(payload)
+
+        logger.debug("Parsed event: {}", evalEvent)
+
+        val evalResponse = pipeline.process(evalEvent)
+
+        output.buffered().use { json.encodeToStream(evalResponse, it) }
+    }
+}

eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/json.kt

@@ -0,0 +1,13 @@
+package by.jprof.telegram.bot.eval.evaluator.config
+
+import kotlinx.serialization.json.Json
+import org.koin.dsl.module
+
+val jsonModule = module {
+    single {
+        Json {
+            encodeDefaults = false
+            ignoreUnknownKeys = true
+        }
+    }
+}

eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/config/pipeline.kt

@@ -0,0 +1,17 @@
+package by.jprof.telegram.bot.eval.evaluator.config
+
+import by.jprof.telegram.bot.eval.evaluator.middleware.Eval
+import by.jprof.telegram.bot.eval.evaluator.middleware.EvalPipeline
+import by.jprof.telegram.bot.eval.evaluator.middleware.JavaScriptEval
+import org.koin.core.qualifier.named
+import org.koin.dsl.module
+
+val pipelineModule = module {
+    single {
+        EvalPipeline(getAll())
+    }
+
+    single<Eval>(named("JavaScriptEval")) {
+        JavaScriptEval()
+    }
+}

eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/Eval.kt

@@ -0,0 +1,8 @@
+package by.jprof.telegram.bot.eval.evaluator.middleware
+
+import by.jprof.telegram.bot.eval.dto.EvalEvent
+import by.jprof.telegram.bot.eval.dto.EvalResponse
+
+interface Eval {
+    suspend fun eval(payload: EvalEvent): EvalResponse?
+}

eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipeline.kt

@@ -0,0 +1,32 @@
+package by.jprof.telegram.bot.eval.evaluator.middleware
+
+import by.jprof.telegram.bot.eval.dto.EvalEvent
+import by.jprof.telegram.bot.eval.dto.EvalResponse
+import kotlinx.coroutines.CoroutineExceptionHandler
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.supervisorScope
+import org.apache.logging.log4j.LogManager
+
+class EvalPipeline(
+    private val evals: List<Eval>
+) {
+    companion object {
+        private val logger = LogManager.getLogger(EvalPipeline::class.java)!!
+    }
+
+    fun process(evalEvent: EvalEvent): EvalResponse = runBlocking {
+        supervisorScope {
+            evals
+                .map { async(exceptionHandler(it)) { it.eval(evalEvent) } }
+                .awaitAll()
+                .filterNotNull()
+                .firstOrNull() ?: EvalResponse.Unsuccessful
+        }
+    }
+
+    private fun exceptionHandler(eval: Eval) = CoroutineExceptionHandler { _, exception ->
+        logger.error("{} failed!", eval::class.simpleName, exception)
+    }
+}

eval/evaluator/src/main/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/JavaScriptEval.kt

@@ -0,0 +1,45 @@
+package by.jprof.telegram.bot.eval.evaluator.middleware
+
+import by.jprof.telegram.bot.eval.dto.EvalEvent
+import by.jprof.telegram.bot.eval.dto.EvalResponse
+import by.jprof.telegram.bot.eval.dto.Language
+import org.apache.logging.log4j.LogManager
+import java.io.IOException
+import java.util.concurrent.TimeUnit
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.writeText
+
+class JavaScriptEval : Eval {
+    companion object {
+        private val logger = LogManager.getLogger(JavaScriptEval::class.java)
+    }
+
+    override suspend fun eval(payload: EvalEvent): EvalResponse? {
+        val file = kotlin.io.path.createTempFile(prefix = "JavaScriptEval", suffix = ".js")
+
+        logger.info("Created temp file: {}", file)
+
+        file.writeText(payload.code)
+
+        try {
+            val proc = ProcessBuilder("node", file.absolutePathString())
+                .directory(file.parent.toFile())
+                .redirectOutput(ProcessBuilder.Redirect.PIPE)
+                .redirectError(ProcessBuilder.Redirect.PIPE)
+                .start()
+
+            proc.waitFor(30, TimeUnit.SECONDS)
+
+            val result = proc.exitValue()
+            val stdout = proc.inputStream.bufferedReader().use { it.readText() }
+            val stderr = proc.errorStream.bufferedReader().use { it.readText() }
+
+            logger.info("Process finished with status: {}. stdout: {}, stderr: {}", result, stdout, stderr)
+
+            return EvalResponse.Successful(Language.JAVASCRIPT, stdout, stderr)
+        } catch (e: IOException) {
+            e.printStackTrace()
+            return null
+        }
+    }
+}

eval/evaluator/src/main/resources/log4j2.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN" packages="com.amazonaws.services.lambda.runtime.log4j2">
+    <Appenders>
+        <Lambda name="Lambda">
+            <JsonLayout>
+                <Complete>false</Complete>
+                <Compact>true</Compact>
+                <KeyValuePair key="AWSRequestId" value="$${ctx:AWSRequestId}"/>
+            </JsonLayout>
+        </Lambda>
+    </Appenders>
+    <Loggers>
+        <Logger name="by.jprof.telegram.bot" level="${env:LOG_THRESHOLD:-INFO}" additivity="false">
+            <AppenderRef ref="Lambda"/>
+        </Logger>
+        <Root level="${env:LOG_ROOT_THRESHOLD:-WARN}">
+            <AppenderRef ref="Lambda"/>
+        </Root>
+    </Loggers>
+</Configuration>

eval/evaluator/src/test/kotlin/by/jprof/telegram/bot/eval/evaluator/middleware/EvalPipelineTest.kt

@@ -0,0 +1,42 @@
+package by.jprof.telegram.bot.eval.evaluator.middleware
+
+import by.jprof.telegram.bot.eval.dto.EvalEvent
+import by.jprof.telegram.bot.eval.dto.EvalResponse
+import kotlinx.coroutines.delay
+import org.junit.jupiter.api.Assertions
+import org.junit.jupiter.api.Test
+import java.time.Duration
+
+internal class EvalPipelineTest {
+    @Test
+    fun processWithBroken() {
+        val sut = EvalPipeline(
+            (1..5).map { index ->
+                when (index % 2) {
+                    0 -> Delay((index + 1) * 1000L)
+                    else -> Fail()
+                }
+            }
+        )
+
+        Assertions.assertTimeout(Duration.ofMillis(6000)) {
+            sut.process(EvalEvent(""))
+        }
+    }
+}
+
+internal class Delay(
+    private val delay: Long,
+) : Eval {
+    override suspend fun eval(payload: EvalEvent): EvalResponse? {
+        delay(delay)
+
+        return EvalResponse.Unsuccessful
+    }
+}
+
+internal class Fail : Eval {
+    override suspend fun eval(payload: EvalEvent): EvalResponse? {
+        throw IllegalArgumentException()
+    }
+}

gradle/libs.versions.toml

@@ -26,6 +26,7 @@ aws-junit5 = "6.0.1"
 mockk = "1.12.0"
 
 [libraries]
+kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines" }
 kotlinx-coroutines-jdk8 = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-jdk8", version.ref = "coroutines" }
 
 aws-lambda-java-events = { group = "com.amazonaws", name = "aws-lambda-java-events", version.ref = "aws-lambda-java-events" }

settings.gradle.kts

@@ -28,4 +28,6 @@ include(":pins:dto")
 include(":pins:unpin")
 include(":pins:dynamodb")
 include(":pins:sfn")
+include(":eval:dto")
+include(":eval:evaluator")
 include(":runners:lambda")