Process elf correctly

.gitignore

@@ -26,6 +26,9 @@ Merged/
 # Folder created by the buildAll scripts
 buildAllJars/
 
+.venv
+relocate_natives/cache/
+
 # file from notepad++
 *.bak
 

.gitlab-ci.yml

@@ -17,6 +17,9 @@ variables:
 # These can be extended so code is a bit less duplicated
 .build_java:
   #image: eclipse-temurin:17
+  before_script:
+    - apt-get update
+    - apt-get install python3 python3-pip python-is-python3 python3-venv -y --no-install-recommends
   cache:
     key: "gradleCache_$CI_JOB_NAME_SLUG"
     policy: pull-push

build.gradle

@@ -5,7 +5,6 @@ import org.apache.tools.zip.ZipEntry
 import javax.annotation.Nonnull
 import org.apache.tools.zip.ZipOutputStream
 
-import java.nio.charset.StandardCharsets
 
 plugins {
     id "java"
@@ -320,21 +319,10 @@ subprojects { p ->
         // Sqlite Database
         relocate("org.xerial", "dh_sqlite.org.xerial")
         relocate("org.sqlite", "dh_sqlite") {
-            // this is the exact list of classes referenced by native code
-//            exclude("org.sqlite.core.NativeDB")
-//            exclude("org.sqlite.Function")
-//            exclude("org.sqlite.Collation")
-//            exclude("org.sqlite.Function$Aggregate")
-//            exclude("org.sqlite.Function$Window")
-//            exclude("org.sqlite.core.DB$ProgressObserver")
-//            exclude("org.sqlite.ProgressHandler")
-//            exclude("org.sqlite.BusyHandlerh")
-
             exclude("org/sqlite/native/**")
         }
+        relocate("jdbc:sqlite", "jdbc:dh_sqlite")
 
-        // "dh_sqlite" leaves just enough room for rewriting "org_sqlite" to "dh_1sqlite" in export tables
-        // https://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/design.html
         // Should be pretty much enough for Linux & Windows
         // Probably won't work for Mac, because it requires signing native libs
         transform(new Transformer() {
@@ -342,40 +330,6 @@ subprojects { p ->
 
             private HashMap<String, byte[]> rewrittenFiles = new HashMap()
 
-            // region thank you chatgpt
-            private static void replaceInByteArray(byte[] byteArray, String target, String replacement) {
-                if (target.length() < replacement.length()) {
-                    throw new IllegalArgumentException("Replacement must be the same length or shorter than the target.")
-                }
-
-                byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII)
-                byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII)
-
-                int nullByte = 0
-
-                for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++) {
-                    int startPos = endPos
-                    int targetPos = 0
-                    while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos]) {
-                        targetPos++
-                        endPos++
-                    }
-
-                    if (targetPos == targetBytes.length) {
-                        System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length)
-
-                        startPos = startPos + replacementBytes.length
-                        while (byteArray[endPos] != nullByte) {
-                            byteArray[startPos] = byteArray[endPos]
-                            endPos++
-                            startPos++
-                        }
-                        byteArray[startPos] = nullByte
-                    }
-                }
-            }
-            // endregion
-
             @Override
             boolean canTransformResource(@Nonnull FileTreeElement element) {
                 System.out.println("canTransformResource " + element.name)
@@ -386,10 +340,11 @@ subprojects { p ->
             void transform(@Nonnull TransformerContext context) {
                 System.out.println("transform " + context.path)
 
+                String path = context.path.replace("org/sqlite", "dh_sqlite")
                 byte[] content = context.is.readAllBytes()
-                replaceInByteArray(content, "org_sqlite", "dh_1sqlite")
+
-                replaceInByteArray(content, "org/sqlite", "dh_sqlite")
+                content = RelocateNatives.processNative(path, content)
-                rewrittenFiles.put(context.path.replace("org/sqlite", "dh_sqlite"), content)
+                rewrittenFiles.put(path, content)
             }
 
             @Override

buildSrc/src/main/java/RelocateNatives.java

@@ -0,0 +1,150 @@
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.CompletableFuture;
+
+class RelocateNatives
+{
+	static boolean prepared = false;
+	
+	private static void ensurePrepared() throws Exception
+	{
+		if (prepared)
+		{
+			return;
+		}
+		prepared = true;
+		
+		ProcessBuilder processBuilder = new ProcessBuilder();
+		processBuilder.directory(new File(System.getProperty("user.dir")));
+		
+		String os = System.getProperty("os.name").toLowerCase();
+		if (os.contains("win"))
+		{
+			processBuilder.command("powershell", "./relocate_natives/prepare.ps1");
+		}
+		else if (os.contains("nix") || os.contains("nux") || os.contains("mac"))
+		{
+			processBuilder.command("./relocate_natives/prepare.sh");
+		}
+		else
+		{
+			throw new IllegalStateException("Unsupported operating system: " + os);
+		}
+		
+		Process process = processBuilder.start();
+		process.getInputStream().transferTo(System.out);
+		process.getErrorStream().transferTo(System.err);
+		
+		int exitCode = process.waitFor();
+		if (exitCode != 0)
+		{
+			throw new Exception("Prepare failed: " + exitCode);
+		}
+	}
+	
+	@SuppressWarnings({"ResultOfMethodCallIgnored", "BusyWait"})
+	public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception
+	{
+		ensurePrepared();
+		
+		ProcessBuilder processBuilder = new ProcessBuilder();
+		processBuilder.directory(new File(System.getProperty("user.dir")));
+		processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString());
+		
+		Process process = processBuilder.start();
+		CompletableFuture<Void> outputFuture = CompletableFuture.runAsync(() -> {
+			try
+			{
+				while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0)
+				{
+					if (process.getInputStream().available() > 0)
+					{
+						byte[] data = new byte[process.getInputStream().available()];
+						process.getInputStream().read(data);
+						System.out.write(data);
+					}
+					if (process.getErrorStream().available() > 0)
+					{
+						byte[] data = new byte[process.getErrorStream().available()];
+						process.getErrorStream().read(data);
+						System.err.write(data);
+					}
+					Thread.sleep(100);
+				}
+			}
+			catch (Exception ignored)
+			{
+			}
+		});
+		
+		process.getOutputStream().write(content);
+		process.getOutputStream().close();
+		
+		int exitCode = process.waitFor();
+		outputFuture.get();
+		
+		if (exitCode != 0)
+		{
+			throw new Exception("Process failed: " + exitCode);
+		}
+		
+		return Files.readAllBytes(outputFilePath);
+	}
+	
+	private static void replaceInByteArray(byte[] byteArray, String target, String replacement)
+	{
+		if (target.length() < replacement.length())
+		{
+			throw new IllegalArgumentException("Replacement must be the same length or shorter than the target.");
+		}
+		
+		byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII);
+		byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII);
+		
+		byte nullByte = 0;
+		
+		for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++)
+		{
+			int startPos = endPos;
+			int targetPos = 0;
+			while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos])
+			{
+				targetPos++;
+				endPos++;
+			}
+			
+			if (targetPos == targetBytes.length)
+			{
+				System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length);
+				
+				startPos = startPos + replacementBytes.length;
+				while (byteArray[endPos] != nullByte)
+				{
+					byteArray[startPos] = byteArray[endPos];
+					endPos++;
+					startPos++;
+				}
+				byteArray[startPos] = nullByte;
+			}
+		}
+	}
+	
+	public static byte[] processNative(String path, byte[] content) throws Exception
+	{
+		Path cacheRoot = Path.of(System.getProperty("user.dir"), "relocate_natives/cache/");
+		Path outputFilePath = cacheRoot.resolve(path);
+		outputFilePath.getParent().toFile().mkdirs();
+		
+		if (outputFilePath.toFile().exists())
+		{
+			return Files.readAllBytes(outputFilePath);
+		}
+		
+		replaceInByteArray(content, "org_sqlite", "dh_1sqlite");
+		replaceInByteArray(content, "org/sqlite", "dh_sqlite");
+		return updateUsingLief(outputFilePath, content);
+	}
+	
+}

relocate_natives/prepare.ps1

@@ -0,0 +1,5 @@
+$ErrorActionPreference = "Stop"
+
+python -m venv .venv
+.\.venv\Scripts\activate
+pip install lief

relocate_natives/prepare.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+python -m venv .venv
+. ./.venv/bin/activate
+pip install lief

relocate_natives/process.py

@@ -0,0 +1,8 @@
+import sys
+import lief
+
+# print("imported", file=sys.stderr)
+
+binary = lief.parse(sys.stdin.buffer.read())
+# print([func.name for func in binary.exported_functions], file=sys.stderr)
+binary.write(sys.argv[1])