diff --git a/.gitignore b/.gitignore index e876e032c..c40f86573 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ Merged/ # Folder created by the buildAll scripts buildAllJars/ +.venv +relocate_natives/cache/ + # file from notepad++ *.bak diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 06deaa984..5cadda9b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,9 @@ variables: # These can be extended so code is a bit less duplicated .build_java: #image: eclipse-temurin:17 + before_script: + - apt-get update + - apt-get install python3 python3-pip python-is-python3 python3-venv -y --no-install-recommends cache: key: "gradleCache_$CI_JOB_NAME_SLUG" policy: pull-push diff --git a/build.gradle b/build.gradle index 618bc8e97..fa29fc8bb 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,6 @@ import org.apache.tools.zip.ZipEntry import javax.annotation.Nonnull import org.apache.tools.zip.ZipOutputStream -import java.nio.charset.StandardCharsets plugins { id "java" @@ -320,21 +319,10 @@ subprojects { p -> // Sqlite Database relocate("org.xerial", "dh_sqlite.org.xerial") relocate("org.sqlite", "dh_sqlite") { - // this is the exact list of classes referenced by native code -// exclude("org.sqlite.core.NativeDB") -// exclude("org.sqlite.Function") -// exclude("org.sqlite.Collation") -// exclude("org.sqlite.Function$Aggregate") -// exclude("org.sqlite.Function$Window") -// exclude("org.sqlite.core.DB$ProgressObserver") -// exclude("org.sqlite.ProgressHandler") -// exclude("org.sqlite.BusyHandlerh") - exclude("org/sqlite/native/**") } + relocate("jdbc:sqlite", "jdbc:dh_sqlite") - // "dh_sqlite" leaves just enough room for rewriting "org_sqlite" to "dh_1sqlite" in export tables - // https://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/design.html // Should be pretty much enough for Linux & Windows // Probably won't work for Mac, because it requires signing native libs transform(new Transformer() { @@ -342,40 +330,6 @@ subprojects { p -> private HashMap rewrittenFiles = new HashMap() - // region thank you chatgpt - private static void replaceInByteArray(byte[] byteArray, String target, String replacement) { - if (target.length() < replacement.length()) { - throw new IllegalArgumentException("Replacement must be the same length or shorter than the target.") - } - - byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII) - byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII) - - int nullByte = 0 - - for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++) { - int startPos = endPos - int targetPos = 0 - while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos]) { - targetPos++ - endPos++ - } - - if (targetPos == targetBytes.length) { - System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length) - - startPos = startPos + replacementBytes.length - while (byteArray[endPos] != nullByte) { - byteArray[startPos] = byteArray[endPos] - endPos++ - startPos++ - } - byteArray[startPos] = nullByte - } - } - } - // endregion - @Override boolean canTransformResource(@Nonnull FileTreeElement element) { System.out.println("canTransformResource " + element.name) @@ -386,10 +340,11 @@ subprojects { p -> void transform(@Nonnull TransformerContext context) { System.out.println("transform " + context.path) + String path = context.path.replace("org/sqlite", "dh_sqlite") byte[] content = context.is.readAllBytes() - replaceInByteArray(content, "org_sqlite", "dh_1sqlite") - replaceInByteArray(content, "org/sqlite", "dh_sqlite") - rewrittenFiles.put(context.path.replace("org/sqlite", "dh_sqlite"), content) + + content = RelocateNatives.processNative(path, content) + rewrittenFiles.put(path, content) } @Override diff --git a/buildSrc/src/main/java/RelocateNatives.java b/buildSrc/src/main/java/RelocateNatives.java new file mode 100644 index 000000000..688d00f99 --- /dev/null +++ b/buildSrc/src/main/java/RelocateNatives.java @@ -0,0 +1,150 @@ +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; + +class RelocateNatives +{ + static boolean prepared = false; + + private static void ensurePrepared() throws Exception + { + if (prepared) + { + return; + } + prepared = true; + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(new File(System.getProperty("user.dir"))); + + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) + { + processBuilder.command("powershell", "./relocate_natives/prepare.ps1"); + } + else if (os.contains("nix") || os.contains("nux") || os.contains("mac")) + { + processBuilder.command("./relocate_natives/prepare.sh"); + } + else + { + throw new IllegalStateException("Unsupported operating system: " + os); + } + + Process process = processBuilder.start(); + process.getInputStream().transferTo(System.out); + process.getErrorStream().transferTo(System.err); + + int exitCode = process.waitFor(); + if (exitCode != 0) + { + throw new Exception("Prepare failed: " + exitCode); + } + } + + @SuppressWarnings({"ResultOfMethodCallIgnored", "BusyWait"}) + public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + { + ensurePrepared(); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(new File(System.getProperty("user.dir"))); + processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString()); + + Process process = processBuilder.start(); + CompletableFuture outputFuture = CompletableFuture.runAsync(() -> { + try + { + while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0) + { + if (process.getInputStream().available() > 0) + { + byte[] data = new byte[process.getInputStream().available()]; + process.getInputStream().read(data); + System.out.write(data); + } + if (process.getErrorStream().available() > 0) + { + byte[] data = new byte[process.getErrorStream().available()]; + process.getErrorStream().read(data); + System.err.write(data); + } + Thread.sleep(100); + } + } + catch (Exception ignored) + { + } + }); + + process.getOutputStream().write(content); + process.getOutputStream().close(); + + int exitCode = process.waitFor(); + outputFuture.get(); + + if (exitCode != 0) + { + throw new Exception("Process failed: " + exitCode); + } + + return Files.readAllBytes(outputFilePath); + } + + private static void replaceInByteArray(byte[] byteArray, String target, String replacement) + { + if (target.length() < replacement.length()) + { + throw new IllegalArgumentException("Replacement must be the same length or shorter than the target."); + } + + byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII); + byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII); + + byte nullByte = 0; + + for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++) + { + int startPos = endPos; + int targetPos = 0; + while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos]) + { + targetPos++; + endPos++; + } + + if (targetPos == targetBytes.length) + { + System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length); + + startPos = startPos + replacementBytes.length; + while (byteArray[endPos] != nullByte) + { + byteArray[startPos] = byteArray[endPos]; + endPos++; + startPos++; + } + byteArray[startPos] = nullByte; + } + } + } + + public static byte[] processNative(String path, byte[] content) throws Exception + { + Path cacheRoot = Path.of(System.getProperty("user.dir"), "relocate_natives/cache/"); + Path outputFilePath = cacheRoot.resolve(path); + outputFilePath.getParent().toFile().mkdirs(); + + if (outputFilePath.toFile().exists()) + { + return Files.readAllBytes(outputFilePath); + } + + replaceInByteArray(content, "org_sqlite", "dh_1sqlite"); + replaceInByteArray(content, "org/sqlite", "dh_sqlite"); + return updateUsingLief(outputFilePath, content); + } + +} diff --git a/relocate_natives/prepare.ps1 b/relocate_natives/prepare.ps1 new file mode 100644 index 000000000..36dcfc186 --- /dev/null +++ b/relocate_natives/prepare.ps1 @@ -0,0 +1,5 @@ +$ErrorActionPreference = "Stop" + +python -m venv .venv +.\.venv\Scripts\activate +pip install lief diff --git a/relocate_natives/prepare.sh b/relocate_natives/prepare.sh new file mode 100755 index 000000000..5bfd06a29 --- /dev/null +++ b/relocate_natives/prepare.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +python -m venv .venv +. ./.venv/bin/activate +pip install lief diff --git a/relocate_natives/process.py b/relocate_natives/process.py new file mode 100644 index 000000000..1c6c1c103 --- /dev/null +++ b/relocate_natives/process.py @@ -0,0 +1,8 @@ +import sys +import lief + +# print("imported", file=sys.stderr) + +binary = lief.parse(sys.stdin.buffer.read()) +# print([func.name for func in binary.exported_functions], file=sys.stderr) +binary.write(sys.argv[1])