Fix dependencies build.

Activate Linux builds.

Author: jurmous

README.md

@@ -8,7 +8,7 @@ storage of data on disk. RocksDB is widely used in many industries for various a
 systems, big data systems, and storage engines for other distributed systems.
 
 The aim of this project is to provide a multi-platform RocksDB implementation that can be used across different platforms,
-including JVM, Android, and native macOS/iOS. This allows developers to write applications that can run on different platforms 
+including JVM, Android, and native Linux/macOS/iOS. This allows developers to write applications that can run on different platforms 
 without having to rewrite the codebase.
 
 This project is useful for developers who want to build multi-platform applications that require high-performance disk-based
@@ -22,7 +22,9 @@ This common library includes most common operations, but if you need additional
 them by creating an issue or submitting a merge request.
 
 ## Native platforms
-Currently, only iOS and macOS (+arm64 simulators) are supported for native compilations. Since the implementation is based on the RocksDB C API,
+Currently, only Linux, iOS and macOS (+arm64 simulators) are supported for native compilations. 
+
+Since the implementation is based on the RocksDB C API,
 other native platforms should be able to be added in the future if RocksDB can be compiled to that platform.
 Unfortunately tvOS and watchOS cannot be supported. No tvOS because it blocks certain calls RocksDB needs, and no watchOS
 because it needs specific arm64_32 support.
@@ -32,6 +34,8 @@ Kotlin Native targets:
 - macosX64
 - iosArm64
 - iosSimulatorArm64
+- linuxX64
+- linuxArm64
 
 ## Reference
 

build.gradle.kts

@@ -27,26 +27,6 @@ val rocksDBAndroidVersion = "9.6.1"
 
 val rocksdbBuildPath = "./rocksdb/build"
 
-val buildMacOS by tasks.creating(Exec::class) {
-    workingDir = projectDir
-    commandLine("./buildRocksdbMac.sh")
-}
-
-val buildLinux by tasks.creating(Exec::class) {
-    workingDir = projectDir
-    commandLine("./buildRocksdbLinux.sh")
-}
-
-val buildIOS by tasks.creating(Exec::class) {
-    workingDir = projectDir
-    commandLine("./buildRocksdbiOS.sh")
-}
-
-val buildIOSSimulator by tasks.creating(Exec::class) {
-    workingDir = projectDir
-    commandLine("./buildRocksdbiOS.sh")
-}
-
 android {
     namespace = "io.maryk.rocksdb"
     buildToolsVersion = "34.0.0"
@@ -132,7 +112,7 @@ kotlin {
             )
         }
 
-        val task = tasks.create("buildDependencies_$buildName", Exec::class) {
+        val dependencyTask = tasks.create("buildDependencies_$buildName", Exec::class) {
             workingDir = projectDir
             val options = buildList<String> {
                 if (buildName.startsWith("linux") && isMacOs) {
@@ -155,7 +135,7 @@ kotlin {
                     defFile("src/nativeInterop/cinterop/rocksdb.def")
                     includeDirs("rocksdb/include/rocksdb")
                     tasks[interopProcessingTaskName].dependsOn(buildTask)
-                    tasks[interopProcessingTaskName].dependsOn(task)
+                    tasks[interopProcessingTaskName].dependsOn(dependencyTask)
                 }
             }
         }
@@ -174,7 +154,7 @@ kotlin {
             workingDir = projectDir
             commandLine("./buildRocksdbApple.sh", "--platform=ios", "--arch=arm64")
         }
-        setupTarget("ios_arm64", buildTask, "-arch arm64 -target arm64-apple-ios13.0-simulator -isysroot $sdkPath", "-DPLATFORM=OS64")
+        setupTarget("ios_arm64", buildTask, "-arch arm64 -target arm64-apple-ios13.0 -isysroot $sdkPath", "-DPLATFORM=OS64")
     }
     iosSimulatorArm64 {
         val sdkPathProvider = providers.exec {
@@ -205,9 +185,20 @@ kotlin {
         }
         setupTarget("macos_arm64", buildTask, "-arch arm64 -target arm64-apple-macos11.0", "-DPLATFORM=MAC")
     }
-//    linuxX64 {
-//        setupTarget("linux_x86_64", buildLinux, "-march=x86-64")
-//    }
+    linuxX64 {
+        val buildTask = tasks.create("buildLib-"+this.name, Exec::class) {
+            workingDir = projectDir
+            commandLine("./buildRocksdbLinux.sh", "--arch=x86-64")
+        }
+        setupTarget("linux_x86_64", buildTask, "-march=x86-64")
+    }
+    linuxArm64 {
+        val buildTask = tasks.create("buildLib-"+this.name, Exec::class) {
+            workingDir = projectDir
+            commandLine("./buildRocksdbLinux.sh", "--arch=arm64")
+        }
+        setupTarget("linux_arm64", buildTask, "-march=armv8-a")
+    }
 }
 
 // Creates the folders for the database

buildDependencies.sh

@@ -23,6 +23,7 @@ set -euo pipefail
 # Default Values
 # ---------------------------------------------------------
 DOWNLOAD_DIR="lib"
+TOOLCHAIN_FILE=null
 
 # iOS Toolchain
 IOS_TOOLCHAIN_URL="https://github.com/leetal/ios-cmake/archive/refs/tags/4.5.0.tar.gz"
@@ -92,6 +93,7 @@ while [[ $# -gt 0 ]]; do
   case "$1" in
     --extra-cflags)
       EXTRA_CFLAGS="$2"
+      EXTRA_CXXFLAGS="$2"
       shift 2
       ;;
     --extra-cmakeflags)
@@ -115,6 +117,16 @@ done
 # Set OUTPUT_DIR to DEFAULT_OUTPUT_DIR if not provided
 OUTPUT_DIR="${OUTPUT_DIR:-$DEFAULT_OUTPUT_DIR}"
 
+set +u
+if [[ "$OUTPUT_DIR" == *linux_x86_64* ]]; then
+  export CC="$HOME/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/bin/x86_64-unknown-linux-gnu-gcc"
+  export CXX="$HOME/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/bin/x86_64-unknown-linux-gnu-g++"
+elif [[ "$OUTPUT_DIR" == *linux_arm64* ]]; then
+  export CC="$HOME/.konan/dependencies/aarch64-unknown-linux-gnu-gcc-8.3.0-glibc-2.25-kernel-4.9-2/bin/aarch64-unknown-linux-gnu-gcc"
+  export CXX="$HOME/.konan/dependencies/aarch64-unknown-linux-gnu-gcc-8.3.0-glibc-2.25-kernel-4.9-2/bin/aarch64-unknown-linux-gnu-g++"
+fi
+set -u
+
 # Create the output directory if it doesn't exist
 mkdir -p "$OUTPUT_DIR"
 mkdir -p "$DOWNLOAD_DIR/include"
@@ -163,7 +175,7 @@ download_ios_toolchain() {
   rm -rf "${DOWNLOAD_DIR}/ios-cmake-4.5.0"
 }
 
-if [[ "${EXTRA_CFLAGS}" == *"-isysroot"* ]]; then
+if [[ "${EXTRA_CFLAGS}" == *"-isysroot"* || "${EXTRA_CFLAGS}" == *"apple-macos"* ]]; then
   download_ios_toolchain
   TOOLCHAIN_FILE="${IOS_TOOLCHAIN_DIR}/ios.toolchain.cmake"
 fi
@@ -244,7 +256,7 @@ build_zstd() {
   local src_dir="${DOWNLOAD_DIR}/zstd-${ZSTD_VER}"
   tar xzf "${tarball}" -C "${DOWNLOAD_DIR}"  > /dev/null
   pushd "${src_dir}/lib" > /dev/null
-  CFLAGS="${EXTRA_CFLAGS} -fPIC -O2" make clean > /dev/null
+  make clean > /dev/null
   CFLAGS="${EXTRA_CFLAGS} -fPIC -O2" make libzstd.a > /dev/null
   popd > /dev/null
   cp "${src_dir}/lib/zstd.h" "${src_dir}/lib/zdict.h" "${DOWNLOAD_DIR}/include/"
@@ -264,8 +276,16 @@ build_snappy() {
   tar xzf "${tarball}" -C "${DOWNLOAD_DIR}"  > /dev/null
   pushd "${src_dir}" > /dev/null
 
-  if [ -n "${TOOLCHAIN_FILE}" ]; then
+  if [ "${TOOLCHAIN_FILE}" != null ]; then
     EXTRA_CMAKEFLAGS+=""" -DCMAKE_TOOLCHAIN_FILE="../../${TOOLCHAIN_FILE}""""
+  else
+    local toolchain_file="toolchain.cmake"
+    echo "set(CMAKE_C_FLAGS \"\${CMAKE_CXX_FLAGS} ${EXTRA_CFLAGS}\")" > "${toolchain_file}"
+    echo "set(CMAKE_CXX_FLAGS \"\${CMAKE_CXX_FLAGS} ${EXTRA_CXXFLAGS}\")" >> "${toolchain_file}"
+    echo "set(CMAKE_POSITION_INDEPENDENT_CODE ON)" >> "${toolchain_file}"
+    echo "set(CMAKE_BUILD_TYPE Release)" >> "${toolchain_file}"
+
+    EXTRA_CMAKEFLAGS+=""" -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_TOOLCHAIN_FILE=$toolchain_file"""
   fi
 
   cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
@@ -295,7 +315,7 @@ build_lz4() {
   echo "Building LZ4 version ${LZ4_VER}..."
   tar xzf "${tarball}" -C "${DOWNLOAD_DIR}" > /dev/null
   pushd "${src_dir}/lib" > /dev/null
-  CFLAGS="${EXTRA_CFLAGS} -fPIC -O2" LDFLAGS="${EXTRA_LDFLAGS}" make clean > /dev/null
+  make clean > /dev/null
   CFLAGS="${EXTRA_CFLAGS} -fPIC -O2" LDFLAGS="${EXTRA_LDFLAGS}" make > /dev/null
   cp "lz4.h" "lz4hc.h" "../../../${DOWNLOAD_DIR}/include"
   cp "liblz4.a" "../../../${OUTPUT_DIR}/"

buildRocksdbApple.sh

@@ -159,6 +159,12 @@ check_build() {
 # Build (make)
 ###############################################################################
 echo "Starting build for: $BUILD_DIR"
+
+if [ -f "${BUILD_DIR}/librocksdb.a" ]; then
+  echo "** BUILD SKIPPED: ${BUILD_DIR}/librocksdb.a already exists **"
+  exit 0
+fi
+
 BUILD_OUTPUT=$(
   make -j"$(sysctl -n hw.ncpu)" \
     LIB_MODE=static \

buildRocksdbLinux.sh

@@ -1,97 +1,84 @@
 #!/usr/bin/env bash
 
-echo "Building RocksDB..."
+ARCH="" # e.g. arm64, x86_64, etc.
+
+for arg in "$@"; do
+  case $arg in
+    --arch=*)
+      ARCH="${arg#*=}"
+      ;;
+    *)
+      echo "Unknown option: $arg"
+      exit 1
+      ;;
+  esac
+done
+
+# Validate
+if [ -z "$ARCH" ]; then
+  echo "Usage: $0 --arch=<arm64|x86_64>"
+  exit 1
+fi
+
+echo "Building RocksDB for $ARCH..."
 
 # Optional: navigate to the rocksdb directory
 cd "rocksdb" || { echo "Failed to navigate to rocksdb"; exit 1; }
 
 # Simple function to check build output
 check_build() {
   local output="$1"
-  local arch="$2"
+  local folder="$2"
 
-  if echo "$output" | grep -q "AR       build/linux_${arch}/librocksdb.a"; then
-      echo "** BUILD SUCCEEDED for $arch **"
+  if echo "$output" | grep -q "AR       build/$folder/librocksdb.a"; then
+      echo "** BUILD SUCCEEDED for $ARCH **"
   elif echo "$output" | grep -q "make: Nothing to be done for 'static_lib'."; then
-      echo "** BUILD NOT NEEDED for $arch (Already up to date) **"
+      echo "** BUILD NOT NEEDED for $ARCH (Already up to date) **"
   else
-      echo "** BUILD FAILED for $arch **"
+      echo "** BUILD FAILED for $ARCH **"
       echo "$output"
       exit 1
   fi
 }
 
-build_for_arch() {
-  local arch="$1"
+EXTRA_FLAGS="-I../lib/include -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD "
 
-  # Set up arch-specific variables
-  local pkgs
-  local cc
-  local cxx
-  local extra_cflags
-  local extra_cxxflags
+if [[ "$ARCH" == "arm64" ]]; then
+  folder="linux_arm64"
+  cc="$HOME/.konan/dependencies/aarch64-unknown-linux-gnu-gcc-8.3.0-glibc-2.25-kernel-4.9-2/bin/aarch64-unknown-linux-gnu-gcc"
+  cxx="$HOME/.konan/dependencies/aarch64-unknown-linux-gnu-gcc-8.3.0-glibc-2.25-kernel-4.9-2/bin/aarch64-unknown-linux-gnu-g++"
+  EXTRA_FLAGS+="-march=armv8-a"
+else
+  folder="linux_x86_64"
+  cc="$HOME/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/bin/x86_64-unknown-linux-gnu-gcc"
+  cxx="$HOME/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/bin/x86_64-unknown-linux-gnu-g++"
+  EXTRA_FLAGS+="-march=x86-64"
+fi
 
-  if [[ "$arch" == "arm64" ]]; then
-    pkgs="gcc-aarch64-linux-gnu g++-aarch64-linux-gnu"
-    cc="aarch64-linux-gnu-gcc"
-    cxx="aarch64-linux-gnu-g++"
-    extra_cflags="-march=armv8-a"
-    extra_cxxflags="-march=armv8-a"
-  else
-    pkgs="gcc-x86-64-linux-gnu g++-x86-64-linux-gnu"
-    cc="x86_64-linux-gnu-gcc"
-    cxx="x86_64-linux-gnu-g++"
-    extra_cflags="-march=x86-64"
-    extra_cxxflags="-march=x86-64"
-  fi
+# Check if the build output already exists
+if [ -f "build/$folder/librocksdb.a" ]; then
+  echo "** BUILD SKIPPED: build/$folder/librocksdb.a already exists **"
+  exit 0
+fi
 
-  echo "Building for $arch..."
-
-  # Run either in Docker (non-Linux) or natively (Linux)
-  local output
-  if [[ "$(uname -s)" == "Linux" ]]; then
-    echo "Detected Linux — building $arch directly..."
-    output=$(
-      apt-get update &&
-      apt-get install -y $pkgs &&
-      make -j"$(nproc)" \
-        LIB_MODE=static \
-        LIBNAME="build/linux_${arch}/librocksdb" \
-        DEBUG_LEVEL=0 \
-        OBJ_DIR="build/linux_${arch}" \
-        EXTRA_CXXFLAGS="$extra_cxxflags" \
-        EXTRA_CFLAGS="$extra_cflags" \
-        LD_FLAGS="-lbz2 -lz" \
-        static_lib
-    )
-  else
-    echo "Detected non-Linux system — building $arch via Docker..."
-    output=$(
-      docker run --rm \
-        -v "$PWD":/rocks \
-        -w /rocks \
-        buildpack-deps \
-        bash -c "
-          apt-get update &&
-          apt-get install -y $pkgs &&
-          make -j\$(nproc) \
-               LIB_MODE=static \
-               LIBNAME=build/linux_${arch}/librocksdb \
-               DEBUG_LEVEL=0 \
-               OBJ_DIR=build/linux_${arch} \
-               EXTRA_CXXFLAGS='$extra_cxxflags' \
-               EXTRA_CFLAGS='$extra_cflags' \
-               LD_FLAGS='-lbz2 -lz' \
-               CC=$cc \
-               CXX=$cxx \
-               static_lib
-        "
-    )
-  fi
-
-  # Check build results
-  check_build "$output" "$arch"
-}
+if [[ "$(uname -s)" == "Linux" ]]; then
+  output=$(
+    make -j"$(nproc)" \
+      LIB_MODE=static \
+      LIBNAME="build/$folder/librocksdb" \
+      DEBUG_LEVEL=0 \
+      CC=$cc \
+      CXX=$cxx \
+      OBJ_DIR="build/$folder" \
+      EXTRA_CXXFLAGS="$EXTRA_FLAGS" \
+      EXTRA_CFLAGS="$EXTRA_FLAGS" \
+      PORTABLE=1 \
+      LD_FLAGS="-lbz2 -lz -lz4 -lsnappy" \
+      static_lib
+  )
 
-build_for_arch "x86_64"
-build_for_arch "aarch64"
+  check_build "$output" "$folder"
+else
+  echo "Should only build on linux"
+  exit 1
+fi