From 082b1224a856da07e6d74f29f6ad33d868bea058 Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Sun, 19 Jan 2025 03:26:53 +0500 Subject: [PATCH 1/8] Relocate sqlite library --- build.gradle | 122 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 4113e60fb..618bc8e97 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,12 @@ +import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer +import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext +import org.apache.tools.zip.ZipEntry + +import javax.annotation.Nonnull +import org.apache.tools.zip.ZipOutputStream + +import java.nio.charset.StandardCharsets + plugins { id "java" @@ -96,6 +105,7 @@ forgix { removeDuplicate "com.seibel.distanthorizons" } + subprojects { p -> // Does the same as "p == project(":common") || p == project(":fabric") || p == project(":quilt") || p == project(":forge") || p == project("WhateverWeAddLaterOn")" // Useful later on so we dont have duplicated code @@ -306,26 +316,98 @@ subprojects { p -> // Logging relocate "org.slf4j", "${librariesLocation}.slf4j" - -// // Sqlite Database -// // James can't determine how to relocate the library correctly so this is commented out -// relocate ("org.sqlite", "${librariesLocation}.sqlite") { -// exclude("org/sqlite/core/NativeDB/**") -// -// exclude("org/sqlite/native/FreeBSD/**") -// exclude("org/sqlite/native/Linux-Android/**") -// exclude("org/sqlite/native/Linux-Musl/**") -// exclude("org/sqlite/native/Linux/arm/**") -// exclude("org/sqlite/native/Linux/aarch64/**") -// exclude("org/sqlite/native/Linux/armv6/**") -// exclude("org/sqlite/native/Linux/x86/**") -// exclude("org/sqlite/native/Linux/armv7/**") -// exclude("org/sqlite/native/Linux/ppc64/**") -// exclude("org/sqlite/native/Linux/riscv64/**") -// exclude("org/sqlite/native/Windows/armv7/**") -// exclude("org/sqlite/native/Windows/aarch64/**") -// exclude("org/sqlite/native/Windows/armv7/**") -// } + + // Sqlite Database + relocate("org.xerial", "dh_sqlite.org.xerial") + relocate("org.sqlite", "dh_sqlite") { + // this is the exact list of classes referenced by native code +// exclude("org.sqlite.core.NativeDB") +// exclude("org.sqlite.Function") +// exclude("org.sqlite.Collation") +// exclude("org.sqlite.Function$Aggregate") +// exclude("org.sqlite.Function$Window") +// exclude("org.sqlite.core.DB$ProgressObserver") +// exclude("org.sqlite.ProgressHandler") +// exclude("org.sqlite.BusyHandlerh") + + exclude("org/sqlite/native/**") + } + + // "dh_sqlite" leaves just enough room for rewriting "org_sqlite" to "dh_1sqlite" in export tables + // https://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/design.html + // Should be pretty much enough for Linux & Windows + // Probably won't work for Mac, because it requires signing native libs + transform(new Transformer() { + final String name = "RelocateSqliteNatives" + + private HashMap rewrittenFiles = new HashMap() + + // region thank you chatgpt + private static void replaceInByteArray(byte[] byteArray, String target, String replacement) { + if (target.length() < replacement.length()) { + throw new IllegalArgumentException("Replacement must be the same length or shorter than the target.") + } + + byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII) + byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII) + + int nullByte = 0 + + for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++) { + int startPos = endPos + int targetPos = 0 + while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos]) { + targetPos++ + endPos++ + } + + if (targetPos == targetBytes.length) { + System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length) + + startPos = startPos + replacementBytes.length + while (byteArray[endPos] != nullByte) { + byteArray[startPos] = byteArray[endPos] + endPos++ + startPos++ + } + byteArray[startPos] = nullByte + } + } + } + // endregion + + @Override + boolean canTransformResource(@Nonnull FileTreeElement element) { + System.out.println("canTransformResource " + element.name) + return element.name.contains("sqlitejdbc") + } + + @Override + void transform(@Nonnull TransformerContext context) { + System.out.println("transform " + context.path) + + byte[] content = context.is.readAllBytes() + replaceInByteArray(content, "org_sqlite", "dh_1sqlite") + replaceInByteArray(content, "org/sqlite", "dh_sqlite") + rewrittenFiles.put(context.path.replace("org/sqlite", "dh_sqlite"), content) + } + + @Override + boolean hasTransformedResource() { + System.out.println("hasTransformedResource") + return true + } + + @Override + void modifyOutputStream(@Nonnull ZipOutputStream os, boolean preserveFileTimestamps) { + System.out.println("modifyOutputStream") + + for (Map.Entry rewrittenFile : rewrittenFiles.entrySet()) { + os.putNextEntry(new ZipEntry(rewrittenFile.key)) + os.write(rewrittenFile.value) + } + } + }) // JOML From 68793fbe8d50633d08eb8f000c184e0e84d8184b Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Tue, 21 Jan 2025 00:33:43 +0500 Subject: [PATCH 2/8] Process elf correctly --- .gitignore | 3 + .gitlab-ci.yml | 3 + build.gradle | 55 +------ buildSrc/src/main/java/RelocateNatives.java | 150 ++++++++++++++++++++ relocate_natives/prepare.ps1 | 5 + relocate_natives/prepare.sh | 7 + relocate_natives/process.py | 8 ++ 7 files changed, 181 insertions(+), 50 deletions(-) create mode 100644 buildSrc/src/main/java/RelocateNatives.java create mode 100644 relocate_natives/prepare.ps1 create mode 100755 relocate_natives/prepare.sh create mode 100644 relocate_natives/process.py diff --git a/.gitignore b/.gitignore index e876e032c..c40f86573 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ Merged/ # Folder created by the buildAll scripts buildAllJars/ +.venv +relocate_natives/cache/ + # file from notepad++ *.bak diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 06deaa984..5cadda9b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,9 @@ variables: # These can be extended so code is a bit less duplicated .build_java: #image: eclipse-temurin:17 + before_script: + - apt-get update + - apt-get install python3 python3-pip python-is-python3 python3-venv -y --no-install-recommends cache: key: "gradleCache_$CI_JOB_NAME_SLUG" policy: pull-push diff --git a/build.gradle b/build.gradle index 618bc8e97..fa29fc8bb 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,6 @@ import org.apache.tools.zip.ZipEntry import javax.annotation.Nonnull import org.apache.tools.zip.ZipOutputStream -import java.nio.charset.StandardCharsets plugins { id "java" @@ -320,21 +319,10 @@ subprojects { p -> // Sqlite Database relocate("org.xerial", "dh_sqlite.org.xerial") relocate("org.sqlite", "dh_sqlite") { - // this is the exact list of classes referenced by native code -// exclude("org.sqlite.core.NativeDB") -// exclude("org.sqlite.Function") -// exclude("org.sqlite.Collation") -// exclude("org.sqlite.Function$Aggregate") -// exclude("org.sqlite.Function$Window") -// exclude("org.sqlite.core.DB$ProgressObserver") -// exclude("org.sqlite.ProgressHandler") -// exclude("org.sqlite.BusyHandlerh") - exclude("org/sqlite/native/**") } + relocate("jdbc:sqlite", "jdbc:dh_sqlite") - // "dh_sqlite" leaves just enough room for rewriting "org_sqlite" to "dh_1sqlite" in export tables - // https://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/design.html // Should be pretty much enough for Linux & Windows // Probably won't work for Mac, because it requires signing native libs transform(new Transformer() { @@ -342,40 +330,6 @@ subprojects { p -> private HashMap rewrittenFiles = new HashMap() - // region thank you chatgpt - private static void replaceInByteArray(byte[] byteArray, String target, String replacement) { - if (target.length() < replacement.length()) { - throw new IllegalArgumentException("Replacement must be the same length or shorter than the target.") - } - - byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII) - byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII) - - int nullByte = 0 - - for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++) { - int startPos = endPos - int targetPos = 0 - while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos]) { - targetPos++ - endPos++ - } - - if (targetPos == targetBytes.length) { - System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length) - - startPos = startPos + replacementBytes.length - while (byteArray[endPos] != nullByte) { - byteArray[startPos] = byteArray[endPos] - endPos++ - startPos++ - } - byteArray[startPos] = nullByte - } - } - } - // endregion - @Override boolean canTransformResource(@Nonnull FileTreeElement element) { System.out.println("canTransformResource " + element.name) @@ -386,10 +340,11 @@ subprojects { p -> void transform(@Nonnull TransformerContext context) { System.out.println("transform " + context.path) + String path = context.path.replace("org/sqlite", "dh_sqlite") byte[] content = context.is.readAllBytes() - replaceInByteArray(content, "org_sqlite", "dh_1sqlite") - replaceInByteArray(content, "org/sqlite", "dh_sqlite") - rewrittenFiles.put(context.path.replace("org/sqlite", "dh_sqlite"), content) + + content = RelocateNatives.processNative(path, content) + rewrittenFiles.put(path, content) } @Override diff --git a/buildSrc/src/main/java/RelocateNatives.java b/buildSrc/src/main/java/RelocateNatives.java new file mode 100644 index 000000000..688d00f99 --- /dev/null +++ b/buildSrc/src/main/java/RelocateNatives.java @@ -0,0 +1,150 @@ +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; + +class RelocateNatives +{ + static boolean prepared = false; + + private static void ensurePrepared() throws Exception + { + if (prepared) + { + return; + } + prepared = true; + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(new File(System.getProperty("user.dir"))); + + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) + { + processBuilder.command("powershell", "./relocate_natives/prepare.ps1"); + } + else if (os.contains("nix") || os.contains("nux") || os.contains("mac")) + { + processBuilder.command("./relocate_natives/prepare.sh"); + } + else + { + throw new IllegalStateException("Unsupported operating system: " + os); + } + + Process process = processBuilder.start(); + process.getInputStream().transferTo(System.out); + process.getErrorStream().transferTo(System.err); + + int exitCode = process.waitFor(); + if (exitCode != 0) + { + throw new Exception("Prepare failed: " + exitCode); + } + } + + @SuppressWarnings({"ResultOfMethodCallIgnored", "BusyWait"}) + public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + { + ensurePrepared(); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(new File(System.getProperty("user.dir"))); + processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString()); + + Process process = processBuilder.start(); + CompletableFuture outputFuture = CompletableFuture.runAsync(() -> { + try + { + while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0) + { + if (process.getInputStream().available() > 0) + { + byte[] data = new byte[process.getInputStream().available()]; + process.getInputStream().read(data); + System.out.write(data); + } + if (process.getErrorStream().available() > 0) + { + byte[] data = new byte[process.getErrorStream().available()]; + process.getErrorStream().read(data); + System.err.write(data); + } + Thread.sleep(100); + } + } + catch (Exception ignored) + { + } + }); + + process.getOutputStream().write(content); + process.getOutputStream().close(); + + int exitCode = process.waitFor(); + outputFuture.get(); + + if (exitCode != 0) + { + throw new Exception("Process failed: " + exitCode); + } + + return Files.readAllBytes(outputFilePath); + } + + private static void replaceInByteArray(byte[] byteArray, String target, String replacement) + { + if (target.length() < replacement.length()) + { + throw new IllegalArgumentException("Replacement must be the same length or shorter than the target."); + } + + byte[] targetBytes = target.getBytes(StandardCharsets.US_ASCII); + byte[] replacementBytes = replacement.getBytes(StandardCharsets.US_ASCII); + + byte nullByte = 0; + + for (int endPos = 0; endPos < byteArray.length - targetBytes.length - 1; endPos++) + { + int startPos = endPos; + int targetPos = 0; + while (targetPos < targetBytes.length && byteArray[endPos] == targetBytes[targetPos]) + { + targetPos++; + endPos++; + } + + if (targetPos == targetBytes.length) + { + System.arraycopy(replacementBytes, 0, byteArray, startPos, replacementBytes.length); + + startPos = startPos + replacementBytes.length; + while (byteArray[endPos] != nullByte) + { + byteArray[startPos] = byteArray[endPos]; + endPos++; + startPos++; + } + byteArray[startPos] = nullByte; + } + } + } + + public static byte[] processNative(String path, byte[] content) throws Exception + { + Path cacheRoot = Path.of(System.getProperty("user.dir"), "relocate_natives/cache/"); + Path outputFilePath = cacheRoot.resolve(path); + outputFilePath.getParent().toFile().mkdirs(); + + if (outputFilePath.toFile().exists()) + { + return Files.readAllBytes(outputFilePath); + } + + replaceInByteArray(content, "org_sqlite", "dh_1sqlite"); + replaceInByteArray(content, "org/sqlite", "dh_sqlite"); + return updateUsingLief(outputFilePath, content); + } + +} diff --git a/relocate_natives/prepare.ps1 b/relocate_natives/prepare.ps1 new file mode 100644 index 000000000..36dcfc186 --- /dev/null +++ b/relocate_natives/prepare.ps1 @@ -0,0 +1,5 @@ +$ErrorActionPreference = "Stop" + +python -m venv .venv +.\.venv\Scripts\activate +pip install lief diff --git a/relocate_natives/prepare.sh b/relocate_natives/prepare.sh new file mode 100755 index 000000000..5bfd06a29 --- /dev/null +++ b/relocate_natives/prepare.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -e + +python -m venv .venv +. ./.venv/bin/activate +pip install lief diff --git a/relocate_natives/process.py b/relocate_natives/process.py new file mode 100644 index 000000000..1c6c1c103 --- /dev/null +++ b/relocate_natives/process.py @@ -0,0 +1,8 @@ +import sys +import lief + +# print("imported", file=sys.stderr) + +binary = lief.parse(sys.stdin.buffer.read()) +# print([func.name for func in binary.exported_functions], file=sys.stderr) +binary.write(sys.argv[1]) From 4eb20d5ce8b7fe016f35a4a7c8c449ff4d17698e Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:28:51 +0500 Subject: [PATCH 3/8] Fix using wrong path on linux --- buildSrc/src/main/java/RelocateNatives.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/RelocateNatives.java b/buildSrc/src/main/java/RelocateNatives.java index 688d00f99..ff0cb1854 100644 --- a/buildSrc/src/main/java/RelocateNatives.java +++ b/buildSrc/src/main/java/RelocateNatives.java @@ -51,7 +51,15 @@ class RelocateNatives ProcessBuilder processBuilder = new ProcessBuilder(); processBuilder.directory(new File(System.getProperty("user.dir"))); - processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString()); + + if (Path.of(System.getProperty("user.dir"), ".venv/Scripts").toFile().exists()) + { + processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString()); + } + else + { + processBuilder.command("./.venv/bin/python", "./relocate_natives/process.py", outputFilePath.toString()); + } Process process = processBuilder.start(); CompletableFuture outputFuture = CompletableFuture.runAsync(() -> { From 4a72e0255030e60578704fa6307c95eba07a1d97 Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Sat, 25 Jan 2025 18:25:58 +0500 Subject: [PATCH 4/8] Sign natives for mac --- .gitignore | 3 +- build.gradle | 7 +- buildSrc/src/main/java/RelocateNatives.java | 117 +++++++++++--------- relocate_natives/download_codesign.py | 110 ++++++++++++++++++ relocate_natives/prepare.ps1 | 2 +- relocate_natives/prepare.sh | 2 +- relocate_natives/process.py | 26 ++++- relocate_natives/requirements.txt | Bin 0 -> 66 bytes 8 files changed, 204 insertions(+), 63 deletions(-) create mode 100644 relocate_natives/download_codesign.py create mode 100644 relocate_natives/requirements.txt diff --git a/.gitignore b/.gitignore index c40f86573..373e80267 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,8 @@ Merged/ # Folder created by the buildAll scripts buildAllJars/ -.venv +relocate_natives/.venv/ +relocate_natives/apple-codesign/ relocate_natives/cache/ # file from notepad++ diff --git a/build.gradle b/build.gradle index fa29fc8bb..852f30ef2 100644 --- a/build.gradle +++ b/build.gradle @@ -343,7 +343,12 @@ subprojects { p -> String path = context.path.replace("org/sqlite", "dh_sqlite") byte[] content = context.is.readAllBytes() - content = RelocateNatives.processNative(path, content) + try { + content = RelocateNatives.processNative(path, content) + } + catch (Throwable e) { + throw new GradleException("Failed to relocate", e) + } rewrittenFiles.put(path, content) } diff --git a/buildSrc/src/main/java/RelocateNatives.java b/buildSrc/src/main/java/RelocateNatives.java index ff0cb1854..f8a5b1c4f 100644 --- a/buildSrc/src/main/java/RelocateNatives.java +++ b/buildSrc/src/main/java/RelocateNatives.java @@ -1,4 +1,3 @@ -import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -6,63 +5,16 @@ import java.util.concurrent.CompletableFuture; class RelocateNatives { - static boolean prepared = false; + private static boolean prepared = false; + + private static final Path rootDirectory = Path.of(System.getProperty("user.dir"), "relocate_natives"); + private static final Path cacheRoot = rootDirectory.resolve("cache"); - private static void ensurePrepared() throws Exception - { - if (prepared) - { - return; - } - prepared = true; - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.directory(new File(System.getProperty("user.dir"))); - - String os = System.getProperty("os.name").toLowerCase(); - if (os.contains("win")) - { - processBuilder.command("powershell", "./relocate_natives/prepare.ps1"); - } - else if (os.contains("nix") || os.contains("nux") || os.contains("mac")) - { - processBuilder.command("./relocate_natives/prepare.sh"); - } - else - { - throw new IllegalStateException("Unsupported operating system: " + os); - } - - Process process = processBuilder.start(); - process.getInputStream().transferTo(System.out); - process.getErrorStream().transferTo(System.err); - - int exitCode = process.waitFor(); - if (exitCode != 0) - { - throw new Exception("Prepare failed: " + exitCode); - } - } @SuppressWarnings({"ResultOfMethodCallIgnored", "BusyWait"}) - public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + private static CompletableFuture readOutputStreams(Process process) { - ensurePrepared(); - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.directory(new File(System.getProperty("user.dir"))); - - if (Path.of(System.getProperty("user.dir"), ".venv/Scripts").toFile().exists()) - { - processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString()); - } - else - { - processBuilder.command("./.venv/bin/python", "./relocate_natives/process.py", outputFilePath.toString()); - } - - Process process = processBuilder.start(); - CompletableFuture outputFuture = CompletableFuture.runAsync(() -> { + return CompletableFuture.runAsync(() -> { try { while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0) @@ -86,6 +38,62 @@ class RelocateNatives { } }); + } + + private static void ensurePrepared() throws Exception + { + if (prepared) + { + return; + } + prepared = true; + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(rootDirectory.toFile()); + + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) + { + processBuilder.command("powershell", "./prepare.ps1"); + } + else if (os.contains("nix") || os.contains("nux") || os.contains("mac")) + { + processBuilder.command("./prepare.sh"); + } + else + { + throw new IllegalStateException("Unsupported operating system: " + os); + } + + Process process = processBuilder.start(); + CompletableFuture outputFuture = readOutputStreams(process); + + int exitCode = process.waitFor(); + outputFuture.get(); + + if (exitCode != 0) + { + throw new Exception("Prepare failed: " + exitCode); + } + } + + public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + { + ensurePrepared(); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(rootDirectory.toFile()); + + processBuilder.command( + rootDirectory.resolve(".venv/Scripts").toFile().exists() + ? rootDirectory.resolve(".venv/Scripts/python.exe").toString() + : rootDirectory.resolve(".venv/bin/python").toString(), + "./process.py", + outputFilePath.toString() + ); + + Process process = processBuilder.start(); + CompletableFuture outputFuture = readOutputStreams(process); process.getOutputStream().write(content); process.getOutputStream().close(); @@ -141,7 +149,6 @@ class RelocateNatives public static byte[] processNative(String path, byte[] content) throws Exception { - Path cacheRoot = Path.of(System.getProperty("user.dir"), "relocate_natives/cache/"); Path outputFilePath = cacheRoot.resolve(path); outputFilePath.getParent().toFile().mkdirs(); diff --git a/relocate_natives/download_codesign.py b/relocate_natives/download_codesign.py new file mode 100644 index 000000000..b2d938c49 --- /dev/null +++ b/relocate_natives/download_codesign.py @@ -0,0 +1,110 @@ +import os +import platform +import requests +import tarfile +import zipfile +from pathlib import Path + + +def get_platform_specific_filename(): + system = platform.system() + machine = platform.machine() + + if system == "Darwin": + if machine == "arm64": + return "apple-codesign-*-aarch64-apple-darwin.tar.gz" + else: + return "apple-codesign-*-x86_64-apple-darwin.tar.gz" + elif system == "Linux": + if machine == "aarch64": + return "apple-codesign-*-aarch64-unknown-linux-musl.tar.gz" + else: + return "apple-codesign-*-x86_64-unknown-linux-musl.tar.gz" + elif system == "Windows": + if machine.endswith("64"): + return "apple-codesign-*-x86_64-pc-windows-msvc.zip" + else: + return "apple-codesign-*-i686-pc-windows-msvc.zip" + else: + raise Exception(f"Unsupported platform: {system} {machine}") + + +def download_and_unpack(): + dest_dir = Path("./apple-codesign") + + repo_url = "https://api.github.com/repos/indygreg/apple-platform-rs/releases/latest" + dest_dir.mkdir(exist_ok=True) + + # Fetch the latest release info from GitHub + print("Fetching latest release information...") + response = requests.get(repo_url) + response.raise_for_status() + release_data = response.json() + + # Ensure release data has assets + if "assets" not in release_data: + raise Exception("Release data does not contain assets.") + + # Determine the correct asset + platform_filename = get_platform_specific_filename() + asset = next((asset for asset in release_data["assets"] if asset["name"].startswith("apple-codesign-") and asset["name"].endswith(platform_filename.split("*")[-1])), None) + + if not asset: + raise Exception(f"No matching asset found for platform: {platform_filename}") + + # Download the archive + print(f"Downloading {asset['name']}...") + download_url = asset["browser_download_url"] + archive_path = dest_dir / asset["name"] + + with requests.get(download_url, stream=True) as r: + r.raise_for_status() + with open(archive_path, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + + print(f"Downloaded to {archive_path}") + + # Extract the archive + print("Extracting archive...") + temp_extract_dir = dest_dir / "temp_extract" + temp_extract_dir.mkdir(parents=True, exist_ok=True) + + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path, "r") as zip_ref: + zip_ref.extractall(temp_extract_dir) + elif archive_path.suffixes[-2:] == [".tar", ".gz"]: + with tarfile.open(archive_path, "r:gz") as tar_ref: + tar_ref.extractall(temp_extract_dir) + else: + raise Exception(f"Unknown archive format: {archive_path}") + + # Move contents of the root directory inside the archive to dest_dir + root_dir = next(temp_extract_dir.iterdir()) # Assuming only one root directory + for item in root_dir.iterdir(): + target_path = dest_dir / item.name + if target_path.exists(): + if target_path.is_dir(): + os.rmdir(target_path) + else: + os.remove(target_path) + item.rename(target_path) + + # Clean up temporary directories + for item in temp_extract_dir.iterdir(): + if item.is_dir(): + os.rmdir(item) + temp_extract_dir.rmdir() + + print(f"Extracted to {dest_dir}") + + # Clean up the archive + os.remove(archive_path) + print(f"Removed archive {archive_path}") + + +if __name__ == "__main__": + try: + download_and_unpack() + except Exception as e: + print(f"Error: {e}") diff --git a/relocate_natives/prepare.ps1 b/relocate_natives/prepare.ps1 index 36dcfc186..218ba6980 100644 --- a/relocate_natives/prepare.ps1 +++ b/relocate_natives/prepare.ps1 @@ -2,4 +2,4 @@ $ErrorActionPreference = "Stop" python -m venv .venv .\.venv\Scripts\activate -pip install lief +pip install -r requirements.txt diff --git a/relocate_natives/prepare.sh b/relocate_natives/prepare.sh index 5bfd06a29..52f9b3a66 100755 --- a/relocate_natives/prepare.sh +++ b/relocate_natives/prepare.sh @@ -4,4 +4,4 @@ set -e python -m venv .venv . ./.venv/bin/activate -pip install lief +pip install -r requirements.txt diff --git a/relocate_natives/process.py b/relocate_natives/process.py index 1c6c1c103..573581feb 100644 --- a/relocate_natives/process.py +++ b/relocate_natives/process.py @@ -1,8 +1,26 @@ import sys import lief +import subprocess +import download_codesign +from pathlib import Path -# print("imported", file=sys.stderr) - +output_path = sys.argv[1] binary = lief.parse(sys.stdin.buffer.read()) -# print([func.name for func in binary.exported_functions], file=sys.stderr) -binary.write(sys.argv[1]) + +if binary is None: + exit(1) + +if isinstance(binary, lief.MachO.Binary): + binary.remove_signature() + +binary.write(output_path) + +if isinstance(binary, lief.MachO.Binary): + print(f"Signing {output_path}...") + + if not Path("./apple-codesign/COPYING").exists(): + download_codesign.download_and_unpack() + + sign_process = subprocess.Popen(["./apple-codesign/rcodesign", "sign", output_path], shell=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sign_process.wait() diff --git a/relocate_natives/requirements.txt b/relocate_natives/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f8d132bea1bfb98e44401a5c2c7f6b11aebc1c9b GIT binary patch literal 66 zcmezWFNYzMA(bJG!4?P&81xtn8O(sB5d$v+7ef(HrjVf&NES1c05M375m43`N`lk? E0JHH5hX4Qo literal 0 HcmV?d00001 From 23a1f0b0253d0237b6b3e43914b3c5f95d58a62a Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Sun, 26 Jan 2025 17:52:30 +0500 Subject: [PATCH 5/8] Clean up code --- build.gradle | 128 ++++++++++------ ...ocateNatives.java => NativeRelocator.java} | 144 +++++++++--------- .../{process.py => fix_modified_binary.py} | 0 3 files changed, 153 insertions(+), 119 deletions(-) rename buildSrc/src/main/java/{RelocateNatives.java => NativeRelocator.java} (79%) rename relocate_natives/{process.py => fix_modified_binary.py} (100%) diff --git a/build.gradle b/build.gradle index 852f30ef2..497d065b4 100644 --- a/build.gradle +++ b/build.gradle @@ -105,6 +105,76 @@ forgix { } +class NativeTransformer implements Transformer { + private boolean enabled = false + private final HashMap replacements = new HashMap() + private final HashMap rewrittenFiles = new HashMap() + private var nativeRelocator + + + NativeTransformer() { + try { + int exitCode = Runtime.getRuntime().exec(new String[]{"python", "--version"}).waitFor(); + if (exitCode == 0) { + enabled = true + } + } catch (IOException e) { + println(e) + } + } + + void relocateNative(String target, String replacement) { + if (replacement.length() > target.length()) { + throw new GradleException("Length of value \"${replacement}\" exceeds the length of \"${target}\": ${replacement.length()} > ${target.length()}") + } + + replacements.put(target, replacement) + } + + + @Override + boolean canTransformResource(@Nonnull FileTreeElement element) { + return replacements.keySet().stream().anyMatch { + element.name.startsWith(it as String) + } + } + + @Override + void transform(@Nonnull TransformerContext context) { + println("Transforming $context.path...") + byte[] content = context.is.readAllBytes() + + if (nativeRelocator == null) { + nativeRelocator = new NativeRelocator() + } + + try { + Map.Entry pathReplacement = replacements.entrySet().stream().filter { + context.path.startsWith(it.key as String) + }.findFirst().orElseThrow() + + String path = context.path.replace(pathReplacement.key as String, pathReplacement.value as String) + content = nativeRelocator.processBinary(path, content, replacements) + + rewrittenFiles.put(path, content) + } + catch (Throwable e) { + throw new GradleException("Failed to relocate", e) + } + } + + @Override + boolean hasTransformedResource() { return !rewrittenFiles.isEmpty() } + + @Override + void modifyOutputStream(@Nonnull ZipOutputStream os, boolean preserveFileTimestamps) { + for (Map.Entry rewrittenFile : rewrittenFiles.entrySet()) { + os.putNextEntry(new ZipEntry(rewrittenFile.key)) + os.write(rewrittenFile.value) + } + } +} + subprojects { p -> // Does the same as "p == project(":common") || p == project(":fabric") || p == project(":quilt") || p == project(":forge") || p == project("WhateverWeAddLaterOn")" // Useful later on so we dont have duplicated code @@ -317,57 +387,17 @@ subprojects { p -> relocate "org.slf4j", "${librariesLocation}.slf4j" // Sqlite Database - relocate("org.xerial", "dh_sqlite.org.xerial") - relocate("org.sqlite", "dh_sqlite") { - exclude("org/sqlite/native/**") + // librariesLocation isn't used because it's too long for replacing paths in native libraries + relocate "org.xerial", "dh_sqlite.org.xerial" + relocate "org.sqlite", "dh_sqlite", { + exclude "org/sqlite/native/**" } - relocate("jdbc:sqlite", "jdbc:dh_sqlite") + relocate "jdbc:sqlite", "jdbc:dh_sqlite" - // Should be pretty much enough for Linux & Windows - // Probably won't work for Mac, because it requires signing native libs - transform(new Transformer() { - final String name = "RelocateSqliteNatives" - - private HashMap rewrittenFiles = new HashMap() - - @Override - boolean canTransformResource(@Nonnull FileTreeElement element) { - System.out.println("canTransformResource " + element.name) - return element.name.contains("sqlitejdbc") - } - - @Override - void transform(@Nonnull TransformerContext context) { - System.out.println("transform " + context.path) - - String path = context.path.replace("org/sqlite", "dh_sqlite") - byte[] content = context.is.readAllBytes() - - try { - content = RelocateNatives.processNative(path, content) - } - catch (Throwable e) { - throw new GradleException("Failed to relocate", e) - } - rewrittenFiles.put(path, content) - } - - @Override - boolean hasTransformedResource() { - System.out.println("hasTransformedResource") - return true - } - - @Override - void modifyOutputStream(@Nonnull ZipOutputStream os, boolean preserveFileTimestamps) { - System.out.println("modifyOutputStream") - - for (Map.Entry rewrittenFile : rewrittenFiles.entrySet()) { - os.putNextEntry(new ZipEntry(rewrittenFile.key)) - os.write(rewrittenFile.value) - } - } - }) + transform(NativeTransformer) { + relocateNative "org/sqlite", "dh_sqlite" + relocateNative "org_sqlite", "dh_1sqlite" + } // JOML diff --git a/buildSrc/src/main/java/RelocateNatives.java b/buildSrc/src/main/java/NativeRelocator.java similarity index 79% rename from buildSrc/src/main/java/RelocateNatives.java rename to buildSrc/src/main/java/NativeRelocator.java index f8a5b1c4f..7823db358 100644 --- a/buildSrc/src/main/java/RelocateNatives.java +++ b/buildSrc/src/main/java/NativeRelocator.java @@ -1,52 +1,21 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Map; import java.util.concurrent.CompletableFuture; -class RelocateNatives +class NativeRelocator { - private static boolean prepared = false; - private static final Path rootDirectory = Path.of(System.getProperty("user.dir"), "relocate_natives"); private static final Path cacheRoot = rootDirectory.resolve("cache"); - @SuppressWarnings({"ResultOfMethodCallIgnored", "BusyWait"}) - private static CompletableFuture readOutputStreams(Process process) + NativeRelocator() throws Exception { - return CompletableFuture.runAsync(() -> { - try - { - while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0) - { - if (process.getInputStream().available() > 0) - { - byte[] data = new byte[process.getInputStream().available()]; - process.getInputStream().read(data); - System.out.write(data); - } - if (process.getErrorStream().available() > 0) - { - byte[] data = new byte[process.getErrorStream().available()]; - process.getErrorStream().read(data); - System.err.write(data); - } - Thread.sleep(100); - } - } - catch (Exception ignored) - { - } - }); - } - - private static void ensurePrepared() throws Exception - { - if (prepared) + if (rootDirectory.resolve(".venv").toFile().exists()) { return; } - prepared = true; ProcessBuilder processBuilder = new ProcessBuilder(); processBuilder.directory(rootDirectory.toFile()); @@ -77,39 +46,40 @@ class RelocateNatives } } - public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + + private static CompletableFuture readOutputStreams(Process process) { - ensurePrepared(); - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.directory(rootDirectory.toFile()); - - processBuilder.command( - rootDirectory.resolve(".venv/Scripts").toFile().exists() - ? rootDirectory.resolve(".venv/Scripts/python.exe").toString() - : rootDirectory.resolve(".venv/bin/python").toString(), - "./process.py", - outputFilePath.toString() - ); - - Process process = processBuilder.start(); - CompletableFuture outputFuture = readOutputStreams(process); - - process.getOutputStream().write(content); - process.getOutputStream().close(); - - int exitCode = process.waitFor(); - outputFuture.get(); - - if (exitCode != 0) - { - throw new Exception("Process failed: " + exitCode); - } - - return Files.readAllBytes(outputFilePath); + return CompletableFuture.runAsync(() -> { + try + { + while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0) + { + if (process.getInputStream().available() > 0) + { + byte[] data = new byte[process.getInputStream().available()]; + //noinspection ResultOfMethodCallIgnored + process.getInputStream().read(data); + System.out.write(data); + } + if (process.getErrorStream().available() > 0) + { + byte[] data = new byte[process.getErrorStream().available()]; + //noinspection ResultOfMethodCallIgnored + process.getErrorStream().read(data); + System.err.write(data); + } + + //noinspection BusyWait + Thread.sleep(100); + } + } + catch (Throwable ignored) + { + } + }); } - private static void replaceInByteArray(byte[] byteArray, String target, String replacement) + private void replaceInNullTerminatedStrings(byte[] byteArray, String target, String replacement) { if (target.length() < replacement.length()) { @@ -147,9 +117,40 @@ class RelocateNatives } } - public static byte[] processNative(String path, byte[] content) throws Exception + public byte[] fixModifiedBinary(Path outputFilePath, byte[] content) throws Exception { - Path outputFilePath = cacheRoot.resolve(path); + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(rootDirectory.toFile()); + + processBuilder.command( + rootDirectory.resolve(".venv/Scripts").toFile().exists() + ? rootDirectory.resolve(".venv/Scripts/python.exe").toString() + : rootDirectory.resolve(".venv/bin/python").toString(), + "./fix_modified_binary.py", + outputFilePath.toString() + ); + + Process process = processBuilder.start(); + CompletableFuture outputFuture = readOutputStreams(process); + + process.getOutputStream().write(content); + process.getOutputStream().close(); + + int exitCode = process.waitFor(); + outputFuture.get(); + + if (exitCode != 0) + { + throw new Exception("Process failed: " + exitCode); + } + + return Files.readAllBytes(outputFilePath); + } + + public byte[] processBinary(String outputPath, byte[] content, Map replacements) throws Exception + { + Path outputFilePath = cacheRoot.resolve(outputPath); + //noinspection ResultOfMethodCallIgnored outputFilePath.getParent().toFile().mkdirs(); if (outputFilePath.toFile().exists()) @@ -157,9 +158,12 @@ class RelocateNatives return Files.readAllBytes(outputFilePath); } - replaceInByteArray(content, "org_sqlite", "dh_1sqlite"); - replaceInByteArray(content, "org/sqlite", "dh_sqlite"); - return updateUsingLief(outputFilePath, content); + for (Map.Entry replacement : replacements.entrySet()) + { + this.replaceInNullTerminatedStrings(content, replacement.getKey(), replacement.getValue()); + } + + return this.fixModifiedBinary(outputFilePath, content); } } diff --git a/relocate_natives/process.py b/relocate_natives/fix_modified_binary.py similarity index 100% rename from relocate_natives/process.py rename to relocate_natives/fix_modified_binary.py From 0f64df7be0c7542b488de309e3be53ca555dab96 Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Sun, 26 Jan 2025 22:08:17 +0500 Subject: [PATCH 6/8] Add missing enabled check --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 497d065b4..bc2e01020 100644 --- a/build.gradle +++ b/build.gradle @@ -134,7 +134,7 @@ class NativeTransformer implements Transformer { @Override boolean canTransformResource(@Nonnull FileTreeElement element) { - return replacements.keySet().stream().anyMatch { + return enabled && replacements.keySet().stream().anyMatch { element.name.startsWith(it as String) } } From 145182502e083285a685e7e418b08183963408af Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:01:28 +0500 Subject: [PATCH 7/8] Do not relocate when python is not installed --- build.gradle | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index bc2e01020..5ccc4621d 100644 --- a/build.gradle +++ b/build.gradle @@ -131,6 +131,11 @@ class NativeTransformer implements Transformer { replacements.put(target, replacement) } + void before(Closure closure) { + if (enabled) + closure.run() + } + @Override boolean canTransformResource(@Nonnull FileTreeElement element) { @@ -388,13 +393,14 @@ subprojects { p -> // Sqlite Database // librariesLocation isn't used because it's too long for replacing paths in native libraries - relocate "org.xerial", "dh_sqlite.org.xerial" - relocate "org.sqlite", "dh_sqlite", { - exclude "org/sqlite/native/**" - } - relocate "jdbc:sqlite", "jdbc:dh_sqlite" - transform(NativeTransformer) { + before { + relocate "org.sqlite", "dh_sqlite", { + exclude "org/sqlite/native/**" + } + relocate "jdbc:sqlite", "jdbc:dh_sqlite" + } + relocateNative "org/sqlite", "dh_sqlite" relocateNative "org_sqlite", "dh_1sqlite" } From 89ca535a6fa9dfca3d0fdc9908f8af8243b0fa51 Mon Sep 17 00:00:00 2001 From: s809 <43530948+s809@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:23:21 +0500 Subject: [PATCH 8/8] Add all the extra comments --- build.gradle | 1 + buildSrc/src/main/java/NativeRelocator.java | 39 +++++++++++++++++++-- relocate_natives/fix_modified_binary.py | 7 +++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 5ccc4621d..0bc03316c 100644 --- a/build.gradle +++ b/build.gradle @@ -393,6 +393,7 @@ subprojects { p -> // Sqlite Database // librariesLocation isn't used because it's too long for replacing paths in native libraries + // Allowing strings larger than the original string would require shifting the entire binary's contents transform(NativeTransformer) { before { relocate "org.sqlite", "dh_sqlite", { diff --git a/buildSrc/src/main/java/NativeRelocator.java b/buildSrc/src/main/java/NativeRelocator.java index 7823db358..eaa32152b 100644 --- a/buildSrc/src/main/java/NativeRelocator.java +++ b/buildSrc/src/main/java/NativeRelocator.java @@ -9,7 +9,12 @@ class NativeRelocator private static final Path rootDirectory = Path.of(System.getProperty("user.dir"), "relocate_natives"); private static final Path cacheRoot = rootDirectory.resolve("cache"); - + /** + * Initializes the NativeRelocator by preparing the environment if necessary. + * Executes the appropriate preparation script based on the OS. + * + * @throws Exception if the preparation script fails or an unsupported OS is detected. + */ NativeRelocator() throws Exception { if (rootDirectory.resolve(".venv").toFile().exists()) @@ -46,7 +51,12 @@ class NativeRelocator } } - + /** + * Reads and prints the output and error streams of a process asynchronously. + * + * @param process The process whose streams should be read. + * @return A CompletableFuture that completes once all output has been processed. + */ private static CompletableFuture readOutputStreams(Process process) { return CompletableFuture.runAsync(() -> { @@ -79,6 +89,14 @@ class NativeRelocator }); } + /** + * Replaces occurrences of a target string in a byte array, ensuring null termination. + * + * @param byteArray The byte array where replacements should occur. + * @param target The string to replace. + * @param replacement The replacement string (must not be longer than the target). + * @throws IllegalArgumentException if the replacement is longer than the target. + */ private void replaceInNullTerminatedStrings(byte[] byteArray, String target, String replacement) { if (target.length() < replacement.length()) @@ -117,6 +135,14 @@ class NativeRelocator } } + /** + * Runs an external script to fix a modified binary and returns the processed content. + * + * @param outputFilePath Path to store the processed binary. + * @param content The original binary content. + * @return The modified binary content. + * @throws Exception if the process execution fails. + */ public byte[] fixModifiedBinary(Path outputFilePath, byte[] content) throws Exception { ProcessBuilder processBuilder = new ProcessBuilder(); @@ -147,6 +173,15 @@ class NativeRelocator return Files.readAllBytes(outputFilePath); } + /** + * Processes a binary file, applying string replacements and fixing modifications. + * + * @param outputPath The output file path relative to the cache directory. + * @param content The binary content to process. + * @param replacements A map of string replacements to apply. + * @return The modified binary content. + * @throws Exception if processing fails. + */ public byte[] processBinary(String outputPath, byte[] content, Map replacements) throws Exception { Path outputFilePath = cacheRoot.resolve(outputPath); diff --git a/relocate_natives/fix_modified_binary.py b/relocate_natives/fix_modified_binary.py index 573581feb..daa2a9c31 100644 --- a/relocate_natives/fix_modified_binary.py +++ b/relocate_natives/fix_modified_binary.py @@ -4,23 +4,28 @@ import subprocess import download_codesign from pathlib import Path +# Parse the input binary & xit if binary is invalid output_path = sys.argv[1] binary = lief.parse(sys.stdin.buffer.read()) - if binary is None: exit(1) +# Remove signature from Mac binaries if isinstance(binary, lief.MachO.Binary): binary.remove_signature() +# Write the modified binary to the output path binary.write(output_path) +# Sign Mac binaries (required to make them usable because apple) if isinstance(binary, lief.MachO.Binary): print(f"Signing {output_path}...") + # Check if the Apple code-signing files are available, if not, download them if not Path("./apple-codesign/COPYING").exists(): download_codesign.download_and_unpack() + # Run the code-signing process sign_process = subprocess.Popen(["./apple-codesign/rcodesign", "sign", output_path], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sign_process.wait()