allow getting native libs dir from android app, add tflite logs on screen

Acs176 · Acs176 · commit a769dbc0ce7c · 2025-03-10T15:13:07.000+01:00
diff --git a/README.md b/README.md
@@ -75,12 +75,13 @@ make env
 
 2. Inside the Docker environment, build the `whisperkit-cli` CLI using (for Android and Linux):
 ```
-make build [linux | qnn | gpu]
+make build [linux | qnn | gpu | jni]
 ```
 
 The QNN option builds WhisperKit with Qualcomm AI NPU support and the QNN TFLite delegate.
 The 'gpu' option is the generic GPU backend for all Android devices from TFLite GPU delegate.
 Linux builds are currently CPU-only.
+The 'jni' option builds the .so file with JNI library to use on android (using QNN support).
 
 3. Back on the host machine (outside Docker shell), push dependencies to the Android device:
 ```
@@ -111,7 +112,16 @@ For Linux:
 make build linux
 ```
 
-2. Manually run `whisperkit-cli`:
+2. Run on Android with `run_on_android.sh` script:
+
+Log in via adb shell:
+```
+adb shell
+cd /sdcard/argmax/tflite
+sh run_on_android.sh
+```
+
+3. Manually run `whisperkit-cli`:
 
 Usage: 
 
@@ -130,7 +140,7 @@ export LD_LIBRARY_PATH=/data/local/tmp/lib
 whisperkit-cli transcribe --model-path  /path/to/openai_whisper-base --audio-path /path/to/inputs/jfk_441khz.m4a
 ```
 
-3. Sample execution output:
+4. Sample execution output:
 ```
 root@cf40510e9b93:/src/AXIE# ./build/linux/whisperkit-cli transcribe --model-path /src/AXIE/models/openai_whisper-small --audio-path /src/AXIE/test/jfk_441khz.m4a 
 SoC: 	generic CPU (x86, arm64, etc) 
diff --git a/WhisperKit/src/TranscribeTask.cpp b/WhisperKit/src/TranscribeTask.cpp
@@ -30,7 +30,7 @@ extern "C" {
 #define TRANSCRIBE_TASK_DEFAULT_LIB_DIR     "/data/local/tmp/lib"
 #define TRANSCRIBE_TASK_DEFAULT_CACHE_DIR   "/data/user/0/com.whispertflite/cache"
 #elif (QNN_DELEGATE || GPU_DELEGATE) 
-// for Android QNN or GPU delegatea
+// for Android QNN or GPU delegate
 #define TRANSCRIBE_TASK_TFLITE_ROOT_PATH    "/sdcard/argmax/tflite"
 #define TRANSCRIBE_TASK_DEFAULT_LIB_DIR     "/data/local/tmp/lib"
 #define TRANSCRIBE_TASK_DEFAULT_CACHE_DIR   "/data/local/tmp/cache"
@@ -199,7 +199,7 @@ std::unique_ptr<std::string> Runtime::cmdexec(const char* cmd) {
 
 bool Runtime::check_qcom_soc() {
     vector<string> supported_socs{
-        "SM8650", "SM8550", "SM8450","SM8350"
+        "SM8650", "SM8550", "SM8450","SM8350", "SM7450"
     };
 
     auto soc = *cmdexec("getprop ro.soc.model");
diff --git a/androidApp/app/src/main/java/com/whispertflite/MainActivity.kt b/androidApp/app/src/main/java/com/whispertflite/MainActivity.kt
@@ -28,11 +28,15 @@ import androidx.compose.foundation.background
 import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.PaddingValues
 import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.fillMaxHeight
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.layout.width
+import androidx.compose.foundation.lazy.LazyColumn
+import androidx.compose.foundation.lazy.items
 import androidx.compose.foundation.rememberScrollState
 import androidx.compose.foundation.shape.RoundedCornerShape
 import androidx.compose.foundation.verticalScroll
@@ -46,8 +50,10 @@ import androidx.compose.material3.Surface
 import androidx.compose.material3.Text
 import androidx.compose.material3.TextButton
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.MutableState
 import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateListOf
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
@@ -58,6 +64,11 @@ import androidx.compose.ui.text.font.FontWeight
 import androidx.compose.ui.unit.dp
 import androidx.core.content.ContextCompat
 import androidx.lifecycle.ViewModelProvider
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.withContext
+import java.io.BufferedReader
+import java.io.InputStreamReader
 
 /** Sample activity to test the TFLite C API. */
 class MainActivity : ComponentActivity() {
@@ -155,6 +166,8 @@ class MainActivity : ComponentActivity() {
             StatusSection()
             Divider(modifier = Modifier.padding(vertical = 16.dp))
             ResultsSection()
+            Divider(modifier = Modifier.padding(vertical = 16.dp))
+            LogcatScreen()
         }
     }
 
@@ -279,8 +292,10 @@ class MainActivity : ComponentActivity() {
         val result by mainViewModel.resultState
         Column(
             modifier = Modifier
+                .fillMaxHeight(0.5f)
                 .fillMaxWidth()
                 .verticalScroll(rememberScrollState())
+
         ) {
             Text(
                 text = result,
@@ -378,114 +393,45 @@ class MainActivity : ComponentActivity() {
         }
     }
 
-//    @Composable
-//    fun ModelSelectionSection() {
-//        val modelList by mainViewModel.tfliteFileNamesState
-//        Surface(
-//            modifier = Modifier
-//                .fillMaxWidth()
-//                .padding(horizontal = 16.dp, vertical = 8.dp),
-//            color = Color(0xFFC4C4C4),
-//            shape = RoundedCornerShape(8.dp),
-//        ) {
-//            Row(
-//                modifier = Modifier
-//                    .fillMaxWidth()
-//                    .padding(horizontal = 16.dp, vertical = 8.dp), // Content padding
-//                verticalAlignment = Alignment.CenterVertically
-//            ) {
-//                Text(
-//                    text = "SELECT MODEL",
-//                    modifier = Modifier
-//                        .width(130.dp),
-//                    color = Color(0xFFFFFFFF),
-//                    style = MaterialTheme.typography.bodyMedium.copy(
-//                        fontWeight = FontWeight.Bold
-//                    )
-//                )
-//                DropdownMenuComponent(
-//                    items = modelList,
-//                    selectedItemState = mainViewModel.selectedTfLiteFilename,
-//                    modifier = Modifier.weight(5f)
-//                )
-//            }
-//        }
-//
-//    }
-
-//    @Composable
-//    fun LogViewer() {
-//        val items by mainViewModel.eventsFlow.collectAsStateWithLifecycle()
-//        LazyColumn (
-//            modifier = Modifier
-//                .fillMaxSize()
-//                .background(Color.LightGray)
-//        ) {
-//            item {
-//                Text(
-//                    text="Hellooo",
-//                    modifier = Modifier.padding(vertical = 4.dp)
-//                )
-//            }
-//            items(items) { item ->
-//                Text(
-//                    text=item,
-//                    modifier = Modifier.padding(vertical = 4.dp)
-//                )
-//            }
-//
-//        }
-//    }
-
-//    private fun logEvent(message: String, logMessages: MutableList<String>, throwable: Throwable? = null) {
-//        Log.e(TAG, message, throwable)
-//        logMessages.add(message)
-//        throwable?.let {
-//            logMessages.add("${it.javaClass.canonicalName}: ${it.message}")
-//            logMessages.addAll(it.stackTrace.map { stackElement -> stackElement.toString() })
-//        }
-//    }
-//
-//    private fun getFileArrayAdapter(files: ArrayList<File>): ArrayAdapter<File> {
-//        return object : ArrayAdapter<File>(this, android.R.layout.simple_spinner_item, files) {
-//            override fun getView(position: Int, convertView: View?, parent: ViewGroup): View {
-//                val view = super.getView(position, convertView, parent)
-//                val textView: TextView = view.findViewById(android.R.id.text1)
-//                textView.text = getItem(position)?.name
-//                return view
-//            }
-//
-//            override fun getDropDownView(position: Int, convertView: View?, parent: ViewGroup): View {
-//                val view = super.getDropDownView(position, convertView, parent)
-//                val textView: TextView = view.findViewById(android.R.id.text1)
-//                textView.text = getItem(position)?.name
-//                return view
-//            }
-//        }.apply {
-//            setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item)
-//        }
-//    }
-
-//    class SharedResource {
-//        private val lock = Object()
-//
-//        fun waitForSignalWithTimeout(timeoutMillis: Long): Boolean {
-//            val startTime = System.currentTimeMillis()
-//            synchronized(lock) {
-//                try {
-//                    lock.wait(timeoutMillis) // Wait for the signal or timeout
-//                } catch (e: InterruptedException) {
-//                    Thread.currentThread().interrupt() // Restore interrupt status
-//                    return false
-//                }
-//                return (System.currentTimeMillis() - startTime) < timeoutMillis
-//            }
-//        }
-//
-//        fun sendSignal() {
-//            synchronized(lock) {
-//                lock.notify() // Notify the waiting thread
-//            }
-//        }
-//    }
+    @Composable
+    fun LogcatScreen() {
+        // State list to hold log lines
+        val logs = remember { mutableStateListOf<String>() }
+
+        // Launch a coroutine to read from logcat
+        LaunchedEffect(Unit) {
+            withContext(Dispatchers.IO) {
+                try {
+                    // Execute the logcat command
+                    val process = Runtime.getRuntime().exec("logcat tflite:V *:S")
+                    val reader = BufferedReader(InputStreamReader(process.inputStream))
+                    // Continuously read new lines
+                    while (isActive) {
+                        val line = reader.readLine() ?: break
+                        // Post the line to the UI thread
+                        withContext(Dispatchers.Main) {
+                            logs.add(line)
+                            // Optionally, limit the size of the logs list
+                            if (logs.size > 1000) { // for example, keep only the latest 1000 lines
+                                logs.removeAt(0)
+                            }
+                        }
+                    }
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                }
+            }
+        }
+
+        // Display the logs in a scrollable list
+        LazyColumn(
+            modifier = Modifier.fillMaxWidth(),
+            contentPadding = PaddingValues(16.dp)
+        ) {
+            items(logs) { log ->
+                Text(text = log)
+            }
+        }
+    }
+
 }
diff --git a/androidApp/app/src/main/java/com/whispertflite/MainViewModel.kt b/androidApp/app/src/main/java/com/whispertflite/MainViewModel.kt
@@ -41,6 +41,9 @@ class MainViewModel : ViewModel() {
     private lateinit var whisperKit: WhisperKitNative
 
     fun initialize(context: Context) {
+        val cacheDir = context.cacheDir
+        Log.d("Cache", cacheDir.absolutePath)
+        val nativeLibsDir = context.applicationInfo.nativeLibraryDir
         copyDataToSdCardFolder(context)
         loadAudioFileNames(sdcardDataFolder!!.absolutePath)
 
@@ -55,7 +58,7 @@ class MainViewModel : ViewModel() {
         waveFile = File(sdcardDataFolder!!.absolutePath + "/" + microphoneInputFileName)
 
         viewModelScope.launch(Dispatchers.IO) {
-            whisperKit = WhisperKitNative(modelDestFolder.absolutePath, audioPath, ".", 4)
+            whisperKit = WhisperKitNative(modelDestFolder.absolutePath, audioPath, ".", nativeLibsDir,  4)
             statusState.value = "IDLE"
         }
     }
diff --git a/androidApp/app/src/main/java/com/whispertflite/WhisperKitNative.java b/androidApp/app/src/main/java/com/whispertflite/WhisperKitNative.java
@@ -3,8 +3,8 @@
 public class WhisperKitNative {
     private long nativePtr;
 
-    public WhisperKitNative(String modelPath, String audioPath, String reportPath, int concurrentWorkers) {
-        nativePtr = init(modelPath, audioPath, reportPath, false, concurrentWorkers);
+    public WhisperKitNative(String modelPath, String audioPath, String reportPath, String nativeLibsDir, int concurrentWorkers) {
+        nativePtr = init(modelPath, audioPath, reportPath, nativeLibsDir, false, concurrentWorkers);
     }
 
     public String transcribe(String audioPath) {
@@ -15,7 +15,7 @@ public void release() {
         release(nativePtr);
     }
 
-    private native long init(String modelPath, String audioPath, String reportPath, boolean enableReport, int concurrentWorkers);
+    private native long init(String modelPath, String audioPath, String reportPath, String nativeLibsDir, boolean enableReport, int concurrentWorkers);
     private native String transcribe(long nativePtr, String audioPath);
     private native void release(long nativePtr);
 }
diff --git a/androidApp/app/src/main/jniLibs/arm64-v8a/libnative-whisper.so b/androidApp/app/src/main/jniLibs/arm64-v8a/libnative-whisper.so
diff --git a/androidApp/app/src/main/jniLibs/arm64-v8a/libwhisperkit.so b/androidApp/app/src/main/jniLibs/arm64-v8a/libwhisperkit.so
diff --git a/cli/whisperkit_cli.cpp b/cli/whisperkit_cli.cpp
@@ -5,6 +5,9 @@
 #include <string>
 #include "whisperkit_cli.h"
 
+#define TRANSCRIBE_TASK_DEFAULT_LIB_DIR     "/data/local/tmp/lib"
+
+
 WhisperKitConfig::WhisperKitConfig()  {
     audioPath = "";
     modelPath = "";
@@ -32,7 +35,7 @@ void CHECK_WHISPERKIT_STATUS(whisperkit_status_t status) {
     }
 }
 
-WhisperKitRunner::WhisperKitRunner(WhisperKitConfig& config) : config(config) {
+WhisperKitRunner::WhisperKitRunner(WhisperKitConfig& config, std::string libDir) : config(config) {
 
     whisperkit_status_t status = WHISPERKIT_STATUS_SUCCESS;
     status = whisperkit_configuration_create(&configuration);
@@ -41,6 +44,9 @@ WhisperKitRunner::WhisperKitRunner(WhisperKitConfig& config) : config(config) {
     status = whisperkit_pipeline_create(&pipeline);
     CHECK_WHISPERKIT_STATUS(status);
 
+    status = whisperkit_configuration_set_lib_dir(configuration, libDir.c_str());
+    CHECK_WHISPERKIT_STATUS(status);                                                                    
+
 }
 
 void WhisperKitRunner::buildPipeline() {
@@ -146,7 +152,7 @@ int main(int argc, char* argv[]) {
         std::cerr << "Error parsing options: " << e.what() << std::endl;
         return 1;
     }
-    WhisperKitRunner runner(config);
+    WhisperKitRunner runner(config, std::string(TRANSCRIBE_TASK_DEFAULT_LIB_DIR));
 
     try {
         runner.buildPipeline();
diff --git a/cli/whisperkit_cli.h b/cli/whisperkit_cli.h
@@ -34,7 +34,7 @@ void CHECK_WHISPERKIT_STATUS(whisperkit_status_t status);
 
 class WhisperKitRunner {
 public:
-    explicit WhisperKitRunner(WhisperKitConfig& config);
+    explicit WhisperKitRunner(WhisperKitConfig& config, std::string libDir);
     ~WhisperKitRunner();
     void buildPipeline();
     void transcribe();
diff --git a/jni/NativeWhisperKit.cpp b/jni/NativeWhisperKit.cpp

Original file line number	Diff line number	Diff line change
`@@ -3,8 +3,8 @@`
`3`	`3`	`public class WhisperKitNative {`
`4`	`4`	`private long nativePtr;`
`5`	`5`
`6`		`- public WhisperKitNative(String modelPath, String audioPath, String reportPath, int concurrentWorkers) {`
`7`		`- nativePtr = init(modelPath, audioPath, reportPath, false, concurrentWorkers);`
	`6`	`+ public WhisperKitNative(String modelPath, String audioPath, String reportPath, String nativeLibsDir, int concurrentWorkers) {`
	`7`	`+ nativePtr = init(modelPath, audioPath, reportPath, nativeLibsDir, false, concurrentWorkers);`
`8`	`8`	`}`
`9`	`9`
`10`	`10`	`public String transcribe(String audioPath) {`
`@@ -15,7 +15,7 @@ public void release() {`
`15`	`15`	`release(nativePtr);`
`16`	`16`	`}`
`17`	`17`
`18`		`- private native long init(String modelPath, String audioPath, String reportPath, boolean enableReport, int concurrentWorkers);`
	`18`	`+ private native long init(String modelPath, String audioPath, String reportPath, String nativeLibsDir, boolean enableReport, int concurrentWorkers);`
`19`	`19`	`private native String transcribe(long nativePtr, String audioPath);`
`20`	`20`	`private native void release(long nativePtr);`
`21`	`21`	`}`