Merge branch 'experimental/relocate_sqlite'

.gitignore

@@ -26,6 +26,10 @@ Merged/
 # Folder created by the buildAll scripts
 buildAllJars/
 
+relocate_natives/.venv/
+relocate_natives/apple-codesign/
+relocate_natives/cache/
+
 # file from notepad++
 *.bak
 

.gitlab-ci.yml

@@ -17,6 +17,9 @@ variables:
 # These can be extended so code is a bit less duplicated
 .build_java:
   #image: eclipse-temurin:17
+  before_script:
+    - apt-get update
+    - apt-get install python3 python3-pip python-is-python3 python3-venv -y --no-install-recommends
   cache:
     key: "gradleCache_$CI_JOB_NAME_SLUG"
     policy: pull-push

build.gradle

@@ -1,3 +1,11 @@
+import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
+import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
+import org.apache.tools.zip.ZipEntry
+
+import javax.annotation.Nonnull
+import org.apache.tools.zip.ZipOutputStream
+
+
 plugins {
     id "java"
 
@@ -96,6 +104,82 @@ forgix {
     removeDuplicate "com.seibel.distanthorizons"
 }
 
+
+class NativeTransformer implements Transformer {
+    private boolean enabled = false
+    private final HashMap<String, String> replacements = new HashMap()
+    private final HashMap<String, byte[]> rewrittenFiles = new HashMap()
+    private var nativeRelocator
+
+
+    NativeTransformer() {
+        try {
+            int exitCode = Runtime.getRuntime().exec(new String[]{"python", "--version"}).waitFor();
+            if (exitCode == 0) {
+                enabled = true
+            }
+        } catch (IOException e) {
+            println(e)
+        }
+    }
+
+    void relocateNative(String target, String replacement) {
+        if (replacement.length() > target.length()) {
+            throw new GradleException("Length of value \"${replacement}\" exceeds the length of \"${target}\": ${replacement.length()} > ${target.length()}")
+        }
+
+        replacements.put(target, replacement)
+    }
+
+    void before(Closure closure) {
+        if (enabled)
+            closure.run()
+    }
+
+
+    @Override
+    boolean canTransformResource(@Nonnull FileTreeElement element) {
+        return enabled && replacements.keySet().stream().anyMatch {
+            element.name.startsWith(it as String)
+        }
+    }
+
+    @Override
+    void transform(@Nonnull TransformerContext context) {
+        println("Transforming $context.path...")
+        byte[] content = context.is.readAllBytes()
+
+        if (nativeRelocator == null) {
+            nativeRelocator = new NativeRelocator()
+        }
+
+        try {
+            Map.Entry<String, String> pathReplacement = replacements.entrySet().stream().filter {
+                context.path.startsWith(it.key as String)
+            }.findFirst().orElseThrow()
+
+            String path = context.path.replace(pathReplacement.key as String, pathReplacement.value as String)
+            content = nativeRelocator.processBinary(path, content, replacements)
+
+            rewrittenFiles.put(path, content)
+        }
+        catch (Throwable e) {
+            throw new GradleException("Failed to relocate", e)
+        }
+    }
+
+    @Override
+    boolean hasTransformedResource() { return !rewrittenFiles.isEmpty() }
+
+    @Override
+    void modifyOutputStream(@Nonnull ZipOutputStream os, boolean preserveFileTimestamps) {
+        for (Map.Entry<String, byte[]> rewrittenFile : rewrittenFiles.entrySet()) {
+            os.putNextEntry(new ZipEntry(rewrittenFile.key))
+            os.write(rewrittenFile.value)
+        }
+    }
+}
+
 subprojects { p ->
     // Does the same as "p == project(":common") || p == project(":fabric") || p == project(":quilt")  || p == project(":forge") || p == project("WhateverWeAddLaterOn")"
     // Useful later on so we dont have duplicated code
@@ -306,26 +390,21 @@ subprojects { p ->
         
         // Logging
         relocate "org.slf4j", "${librariesLocation}.slf4j"
-        
-//        // Sqlite Database
-//        // James can't determine how to relocate the library correctly so this is commented out
-//        relocate ("org.sqlite", "${librariesLocation}.sqlite") {
-//            exclude("org/sqlite/core/NativeDB/**")
-//
-//            exclude("org/sqlite/native/FreeBSD/**")
-//            exclude("org/sqlite/native/Linux-Android/**")
-//            exclude("org/sqlite/native/Linux-Musl/**")
-//            exclude("org/sqlite/native/Linux/arm/**")
-//            exclude("org/sqlite/native/Linux/aarch64/**")
-//            exclude("org/sqlite/native/Linux/armv6/**")
-//            exclude("org/sqlite/native/Linux/x86/**")
-//            exclude("org/sqlite/native/Linux/armv7/**")
-//            exclude("org/sqlite/native/Linux/ppc64/**")
-//            exclude("org/sqlite/native/Linux/riscv64/**")
-//            exclude("org/sqlite/native/Windows/armv7/**")
-//            exclude("org/sqlite/native/Windows/aarch64/**")
-//            exclude("org/sqlite/native/Windows/armv7/**")
-//        }
+
+        // Sqlite Database
+        // librariesLocation isn't used because it's too long for replacing paths in native libraries
+        // Allowing strings larger than the original string would require shifting the entire binary's contents
+        transform(NativeTransformer) {
+            before {
+                relocate "org.sqlite", "dh_sqlite", {
+                    exclude "org/sqlite/native/**"
+                }
+                relocate "jdbc:sqlite", "jdbc:dh_sqlite"
+            }
+
+            relocateNative "org/sqlite", "dh_sqlite"
+            relocateNative "org_sqlite", "dh_1sqlite"
+        }
         
         
         // JOML

buildSrc/src/main/java/NativeRelocator.java

@@ -0,0 +1,204 @@
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+class NativeRelocator
+{
+	private static final Path rootDirectory = Path.of(System.getProperty("user.dir"), "relocate_natives");
+	private static final Path cacheRoot = rootDirectory.resolve("cache");
+	
+	/**
+	 * Initializes the NativeRelocator by preparing the environment if necessary.
+	 * Executes the appropriate preparation script based on the OS.
+	 *
+	 * @throws Exception if the preparation script fails or an unsupported OS is detected.
+	 */
+	NativeRelocator() throws Exception
+	{
+		if (rootDirectory.resolve(".venv").toFile().exists())
+		{
+			return;
+		}
+		
+		ProcessBuilder processBuilder = new ProcessBuilder();
+		processBuilder.directory(rootDirectory.toFile());
+		
+		String os = System.getProperty("os.name").toLowerCase();
+		if (os.contains("win"))
+		{
+			processBuilder.command("powershell", "./prepare.ps1");
+		}
+		else if (os.contains("nix") || os.contains("nux") || os.contains("mac"))
+		{
+			processBuilder.command("./prepare.sh");
+		}
+		else
+		{
+			throw new IllegalStateException("Unsupported operating system: " + os);
+		}
+		
+		Process process = processBuilder.start();
+		CompletableFuture<Void> outputFuture = readOutputStreams(process);
+		
+		int exitCode = process.waitFor();
+		outputFuture.get();
+		
+		if (exitCode != 0)
+		{
+			throw new Exception("Prepare failed: " + exitCode);
+		}
+	}
+	
+	/**
+	 * Reads and prints the output and error streams of a process asynchronously.
+	 *
+	 * @param process The process whose streams should be read.
+	 * @return A CompletableFuture that completes once all output has been processed.
+	 */
+	private static CompletableFuture<Void> readOutputStreams(Process process)
+	{
+		return CompletableFuture.runAsync(() -> {
+			try
+			{
+				while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0)
+				{
+					if (process.getInputStream().available() > 0)
+					{
+						byte[] data = new byte[process.getInputStream().available()];
+						//noinspection ResultOfMethodCallIgnored
+						process.getInputStream().read(data);
+						System.out.write(data);
+					}
+					if (process.getErrorStream().available() > 0)
+					{
+						byte[] data = new byte[process.getErrorStream().available()];
+						//noinspection ResultOfMethodCallIgnored
+						process.getErrorStream().read(data);
+						System.err.write(data);
+					}
+					
+					//noinspection BusyWait
+					Thread.sleep(100);
+				}
+			}
+			catch (Throwable ignored)
+			{
+			}
+		});
+	}
+	
+	/**
+	 * Replaces occurrences of a target string in a byte array, ensuring null termination.
+	 *
+	 * @param byteArray The byte array where replacements should occur.
+	 * @param target The string to replace.
+	 * @param replacement The replacement string (must not be longer than the target).
+	 * @throws IllegalArgumentException if the replacement is longer than the target.
+	 */
+	private void replaceInNullTerminatedStrings(byte[] byteArray, String target, String replacement)
+	{
+		if (target.length() < replacement.length())
+		{
+			throw new IllegalArgumentException("Replacement must be the same length or shorter than the target.");
+		}
+		
+		byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII);
+		byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII);
+		
+		byte nullByte = 0;
+		
+		for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++)
+		{
+			int startPos = endPos;
+			int targetPos = 0;
+			while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos])
+			{
+				targetPos++;
+				endPos++;
+			}
+			
+			if (targetPos == targetBytes.length)
+			{
+				System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length);
+				
+				startPos = startPos + replacementBytes.length;
+				while (byteArray[endPos] != nullByte)
+				{
+					byteArray[startPos] = byteArray[endPos];
+					endPos++;
+					startPos++;
+				}
+				byteArray[startPos] = nullByte;
+			}
+		}
+	}
+	
+	/**
+	 * Runs an external script to fix a modified binary and returns the processed content.
+	 *
+	 * @param outputFilePath Path to store the processed binary.
+	 * @param content The original binary content.
+	 * @return The modified binary content.
+	 * @throws Exception if the process execution fails.
+	 */
+	public byte[] fixModifiedBinary(Path outputFilePath, byte[] content) throws Exception
+	{
+		ProcessBuilder processBuilder = new ProcessBuilder();
+		processBuilder.directory(rootDirectory.toFile());
+		
+		processBuilder.command(
+				rootDirectory.resolve(".venv/Scripts").toFile().exists()
+						? rootDirectory.resolve(".venv/Scripts/python.exe").toString()
+						: rootDirectory.resolve(".venv/bin/python").toString(),
+				"./fix_modified_binary.py",
+				outputFilePath.toString()
+		);
+		
+		Process process = processBuilder.start();
+		CompletableFuture<Void> outputFuture = readOutputStreams(process);
+		
+		process.getOutputStream().write(content);
+		process.getOutputStream().close();
+		
+		int exitCode = process.waitFor();
+		outputFuture.get();
+		
+		if (exitCode != 0)
+		{
+			throw new Exception("Process failed: " + exitCode);
+		}
+		
+		return Files.readAllBytes(outputFilePath);
+	}
+	
+	/**
+	 * Processes a binary file, applying string replacements and fixing modifications.
+	 *
+	 * @param outputPath The output file path relative to the cache directory.
+	 * @param content The binary content to process.
+	 * @param replacements A map of string replacements to apply.
+	 * @return The modified binary content.
+	 * @throws Exception if processing fails.
+	 */
+	public byte[] processBinary(String outputPath, byte[] content, Map<String, String> replacements) throws Exception
+	{
+		Path outputFilePath = cacheRoot.resolve(outputPath);
+		//noinspection ResultOfMethodCallIgnored
+		outputFilePath.getParent().toFile().mkdirs();
+		
+		if (outputFilePath.toFile().exists())
+		{
+			return Files.readAllBytes(outputFilePath);
+		}
+		
+		for (Map.Entry<String, String> replacement : replacements.entrySet())
+		{
+			this.replaceInNullTerminatedStrings(content, replacement.getKey(), replacement.getValue());
+		}
+		
+		return this.fixModifiedBinary(outputFilePath, content);
+	}
+	
+}

relocate_natives/download_codesign.py

@@ -0,0 +1,110 @@
+import os
+import platform
+import requests
+import tarfile
+import zipfile
+from pathlib import Path
+
+
+def get_platform_specific_filename():
+    system = platform.system()
+    machine = platform.machine()
+
+    if system == "Darwin":
+        if machine == "arm64":
+            return "apple-codesign-*-aarch64-apple-darwin.tar.gz"
+        else:
+            return "apple-codesign-*-x86_64-apple-darwin.tar.gz"
+    elif system == "Linux":
+        if machine == "aarch64":
+            return "apple-codesign-*-aarch64-unknown-linux-musl.tar.gz"
+        else:
+            return "apple-codesign-*-x86_64-unknown-linux-musl.tar.gz"
+    elif system == "Windows":
+        if machine.endswith("64"):
+            return "apple-codesign-*-x86_64-pc-windows-msvc.zip"
+        else:
+            return "apple-codesign-*-i686-pc-windows-msvc.zip"
+    else:
+        raise Exception(f"Unsupported platform: {system} {machine}")
+
+
+def download_and_unpack():
+    dest_dir = Path("./apple-codesign")
+
+    repo_url = "https://api.github.com/repos/indygreg/apple-platform-rs/releases/latest"
+    dest_dir.mkdir(exist_ok=True)
+
+    # Fetch the latest release info from GitHub
+    print("Fetching latest release information...")
+    response = requests.get(repo_url)
+    response.raise_for_status()
+    release_data = response.json()
+
+    # Ensure release data has assets
+    if "assets" not in release_data:
+        raise Exception("Release data does not contain assets.")
+
+    # Determine the correct asset
+    platform_filename = get_platform_specific_filename()
+    asset = next((asset for asset in release_data["assets"] if asset["name"].startswith("apple-codesign-") and asset["name"].endswith(platform_filename.split("*")[-1])), None)
+
+    if not asset:
+        raise Exception(f"No matching asset found for platform: {platform_filename}")
+
+    # Download the archive
+    print(f"Downloading {asset['name']}...")
+    download_url = asset["browser_download_url"]
+    archive_path = dest_dir / asset["name"]
+
+    with requests.get(download_url, stream=True) as r:
+        r.raise_for_status()
+        with open(archive_path, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+    print(f"Downloaded to {archive_path}")
+
+    # Extract the archive
+    print("Extracting archive...")
+    temp_extract_dir = dest_dir / "temp_extract"
+    temp_extract_dir.mkdir(parents=True, exist_ok=True)
+
+    if archive_path.suffix == ".zip":
+        with zipfile.ZipFile(archive_path, "r") as zip_ref:
+            zip_ref.extractall(temp_extract_dir)
+    elif archive_path.suffixes[-2:] == [".tar", ".gz"]:
+        with tarfile.open(archive_path, "r:gz") as tar_ref:
+            tar_ref.extractall(temp_extract_dir)
+    else:
+        raise Exception(f"Unknown archive format: {archive_path}")
+
+    # Move contents of the root directory inside the archive to dest_dir
+    root_dir = next(temp_extract_dir.iterdir())  # Assuming only one root directory
+    for item in root_dir.iterdir():
+        target_path = dest_dir / item.name
+        if target_path.exists():
+            if target_path.is_dir():
+                os.rmdir(target_path)
+            else:
+                os.remove(target_path)
+        item.rename(target_path)
+
+    # Clean up temporary directories
+    for item in temp_extract_dir.iterdir():
+        if item.is_dir():
+            os.rmdir(item)
+    temp_extract_dir.rmdir()
+
+    print(f"Extracted to {dest_dir}")
+
+    # Clean up the archive
+    os.remove(archive_path)
+    print(f"Removed archive {archive_path}")
+
+
+if __name__ == "__main__":
+    try:
+        download_and_unpack()
+    except Exception as e:
+        print(f"Error: {e}")

relocate_natives/fix_modified_binary.py

@@ -0,0 +1,31 @@
+import sys
+import lief
+import subprocess
+import download_codesign
+from pathlib import Path
+
+# Parse the input binary & xit if binary is invalid
+output_path = sys.argv[1]
+binary = lief.parse(sys.stdin.buffer.read())
+if binary is None:
+    exit(1)
+
+# Remove signature from Mac binaries
+if isinstance(binary, lief.MachO.Binary):
+    binary.remove_signature()
+
+# Write the modified binary to the output path
+binary.write(output_path)
+
+# Sign Mac binaries (required to make them usable because apple)
+if isinstance(binary, lief.MachO.Binary):
+    print(f"Signing {output_path}...")
+
+    # Check if the Apple code-signing files are available, if not, download them
+    if not Path("./apple-codesign/COPYING").exists():
+        download_codesign.download_and_unpack()
+
+    # Run the code-signing process
+    sign_process = subprocess.Popen(["./apple-codesign/rcodesign", "sign", output_path], shell=False,
+                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    sign_process.wait()

relocate_natives/prepare.ps1

@@ -0,0 +1,5 @@
+$ErrorActionPreference = "Stop"
+
+python -m venv .venv
+.\.venv\Scripts\activate
+pip install -r requirements.txt

relocate_natives/prepare.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+python -m venv .venv
+. ./.venv/bin/activate
+pip install -r requirements.txt