diff --git a/.gitignore b/.gitignore index 66ba8cabb..1d9b3b01b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,6 @@ buildAllJars/ relocate_natives/.venv/ relocate_natives/__pycache__/ relocate_natives/apple-codesign/ -relocate_natives/cache/ # file from notepad++ *.bak diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86dc02c1a..bd7716999 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,9 +17,6 @@ variables: # These can be extended so code is a bit less duplicated .build_java: #image: eclipse-temurin:17 - before_script: - - apt-get update - - apt-get install python3 python3-pip python-is-python3 python3-venv -y --no-install-recommends cache: key: "gradleCache_$CI_JOB_NAME_SLUG" policy: pull-push diff --git a/build.gradle b/build.gradle index 0b3d96ca8..3d4a334cb 100644 --- a/build.gradle +++ b/build.gradle @@ -106,24 +106,12 @@ forgix { class NativeTransformer implements Transformer { - private boolean enabled = false private final HashMap replacements = new HashMap() private final HashMap rewrittenFiles = new HashMap() private var nativeRelocator public File rootDir - NativeTransformer() { - try { - int exitCode = Runtime.getRuntime().exec(new String[]{"python", "--version"}).waitFor() - if (exitCode == 0) { - enabled = true - } - } catch (IOException e) { - println(e) - } - } - void relocateNative(String target, String replacement) { if (replacement.length() > target.length()) { throw new GradleException("Length of value \"${replacement}\" exceeds the length of \"${target}\": ${replacement.length()} > ${target.length()}") @@ -132,22 +120,15 @@ class NativeTransformer implements Transformer { replacements.put(target, replacement) } - void before(Closure closure) { - if (enabled) - closure.run() - } - - @Override boolean canTransformResource(@Nonnull FileTreeElement element) { - return enabled && replacements.keySet().stream().anyMatch { + return replacements.keySet().stream().anyMatch { element.name.startsWith(it as String) } } @Override void transform(@Nonnull TransformerContext context) { - println("Transforming $context.path...") byte[] content = context.is.readAllBytes() if (nativeRelocator == null) { @@ -395,16 +376,13 @@ subprojects { p -> // Sqlite Database // librariesLocation isn't used because it's too long for replacing paths in native libraries // Allowing strings larger than the original string would require shifting the entire binary's contents + relocate "org.sqlite", "dh_sqlite", { + exclude "org/sqlite/native/**" + } + relocate "jdbc:sqlite", "jdbc:dh_sqlite" + transform(NativeTransformer) { rootDir = project.rootDir - - before { - relocate "org.sqlite", "dh_sqlite", { - exclude "org/sqlite/native/**" - } - relocate "jdbc:sqlite", "jdbc:dh_sqlite" - } - relocateNative "org/sqlite", "dh_sqlite" relocateNative "org_sqlite", "dh_1sqlite" } diff --git a/buildSrc/src/main/java/NativeRelocator.java b/buildSrc/src/main/java/NativeRelocator.java index b35507cf3..f11f682d9 100644 --- a/buildSrc/src/main/java/NativeRelocator.java +++ b/buildSrc/src/main/java/NativeRelocator.java @@ -196,6 +196,8 @@ class NativeRelocator return Files.readAllBytes(outputFilePath); } + System.out.println("Relocating to " + outputPath + "..."); + for (Map.Entry replacement : replacements.entrySet()) { this.replaceInNullTerminatedStrings(content, replacement.getKey(), replacement.getValue()); diff --git a/gradle.properties b/gradle.properties index f6fd5360f..9b231f87e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,6 +23,7 @@ manifold_version=2024.1.37 nightconfig_version=3.6.6 lz4_version=1.8.0 xz_version=1.9 +# Before updating, read relocate_natives/README.md sqlite_jdbc_version=3.47.2.0 # 8.2.1 is the newest version we can use since that's the version MC 1.16.5 uses fastutil_version=8.2.1 diff --git a/relocate_natives/README.md b/relocate_natives/README.md new file mode 100644 index 000000000..9ed3da8fd --- /dev/null +++ b/relocate_natives/README.md @@ -0,0 +1,46 @@ +This directory contains the native files of libraries that were modified for relocation. They will be copied from here during the normal build steps. + +Before adding/updating a library, make sure you have Python 3.8+ installed and check the instructions below. + +How to add a library's natives: + +1. In `build.gradle`: + +- Make sure the target package is the same length or shorter (untested) than the source package. Underscores in native methods will take 2 characters so account for that as well. +- Exclude the native files and add them as `relocateNative` (see example). + +Example: + +```groovy +relocate "org.sqlite", "dh_sqlite", { + exclude "org/sqlite/native/**" +} + +transform(NativeTransformer) { + // NativeTransformer configuration + rootDir = project.rootDir + + // Replace native strings, e.g. used in calls back to Java + relocateNative "org/sqlite", "dh_sqlite" + // Rename native methods used when calling from Java + relocateNative "org_sqlite", "dh_1sqlite" +} +``` + +How to update a library's natives: + +1. Delete the library's folder in cache/. +2. It will repopulate during the next build. + +Why does this step exist? + +- Native files are not handled by the shadow plugin correctly. +- I preferred it as a more streamlined approach, although a bit hacky. +- Possible alternatives: + - Use edited libraries' source code: although more straightforward, it requires maintaining and updating the repositories for the libraries being added + - Interfacing with the necessary libraries directly: an absolute mess for technical reasons + +What are libraries used? + +- LIEF: for fixing binaries after replacing strings +- apple-codesign: for re-signing Mac binaries, since their signatures get invalidated after previous steps diff --git a/relocate_natives/cache/dh_sqlite/native/FreeBSD/aarch64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/FreeBSD/aarch64/libsqlitejdbc.so new file mode 100644 index 000000000..8fede2967 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/FreeBSD/aarch64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/FreeBSD/x86/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/FreeBSD/x86/libsqlitejdbc.so new file mode 100644 index 000000000..cb4e91736 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/FreeBSD/x86/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/FreeBSD/x86_64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/FreeBSD/x86_64/libsqlitejdbc.so new file mode 100644 index 000000000..fcf5400dd Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/FreeBSD/x86_64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Android/aarch64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Android/aarch64/libsqlitejdbc.so new file mode 100644 index 000000000..1ab6595bb Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Android/aarch64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Android/arm/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Android/arm/libsqlitejdbc.so new file mode 100644 index 000000000..4e1ceee83 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Android/arm/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Android/x86/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Android/x86/libsqlitejdbc.so new file mode 100644 index 000000000..b31057743 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Android/x86/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Android/x86_64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Android/x86_64/libsqlitejdbc.so new file mode 100644 index 000000000..e02ea2316 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Android/x86_64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Musl/aarch64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Musl/aarch64/libsqlitejdbc.so new file mode 100644 index 000000000..0062e2a3f Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Musl/aarch64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Musl/x86/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Musl/x86/libsqlitejdbc.so new file mode 100644 index 000000000..18ae520a6 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Musl/x86/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux-Musl/x86_64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux-Musl/x86_64/libsqlitejdbc.so new file mode 100644 index 000000000..fc8d7ae6a Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux-Musl/x86_64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/aarch64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/aarch64/libsqlitejdbc.so new file mode 100644 index 000000000..ed4bd93ce Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/aarch64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/arm/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/arm/libsqlitejdbc.so new file mode 100644 index 000000000..a150092c7 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/arm/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/armv6/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/armv6/libsqlitejdbc.so new file mode 100644 index 000000000..f5c77db53 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/armv6/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/armv7/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/armv7/libsqlitejdbc.so new file mode 100644 index 000000000..589a8ad3a Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/armv7/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/ppc64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/ppc64/libsqlitejdbc.so new file mode 100644 index 000000000..a83197a3a Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/ppc64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/riscv64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/riscv64/libsqlitejdbc.so new file mode 100644 index 000000000..75fbdaabd Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/riscv64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/x86/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/x86/libsqlitejdbc.so new file mode 100644 index 000000000..61df83f72 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/x86/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Linux/x86_64/libsqlitejdbc.so b/relocate_natives/cache/dh_sqlite/native/Linux/x86_64/libsqlitejdbc.so new file mode 100644 index 000000000..369039f91 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Linux/x86_64/libsqlitejdbc.so differ diff --git a/relocate_natives/cache/dh_sqlite/native/Mac/aarch64/libsqlitejdbc.dylib b/relocate_natives/cache/dh_sqlite/native/Mac/aarch64/libsqlitejdbc.dylib new file mode 100644 index 000000000..50a9b8c36 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Mac/aarch64/libsqlitejdbc.dylib differ diff --git a/relocate_natives/cache/dh_sqlite/native/Mac/x86_64/libsqlitejdbc.dylib b/relocate_natives/cache/dh_sqlite/native/Mac/x86_64/libsqlitejdbc.dylib new file mode 100644 index 000000000..dd257f341 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Mac/x86_64/libsqlitejdbc.dylib differ diff --git a/relocate_natives/cache/dh_sqlite/native/Windows/aarch64/sqlitejdbc.dll b/relocate_natives/cache/dh_sqlite/native/Windows/aarch64/sqlitejdbc.dll new file mode 100644 index 000000000..fb0a84b2c Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Windows/aarch64/sqlitejdbc.dll differ diff --git a/relocate_natives/cache/dh_sqlite/native/Windows/armv7/sqlitejdbc.dll b/relocate_natives/cache/dh_sqlite/native/Windows/armv7/sqlitejdbc.dll new file mode 100644 index 000000000..d244d21ca Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Windows/armv7/sqlitejdbc.dll differ diff --git a/relocate_natives/cache/dh_sqlite/native/Windows/x86/sqlitejdbc.dll b/relocate_natives/cache/dh_sqlite/native/Windows/x86/sqlitejdbc.dll new file mode 100644 index 000000000..cb1a1b5ae Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Windows/x86/sqlitejdbc.dll differ diff --git a/relocate_natives/cache/dh_sqlite/native/Windows/x86_64/sqlitejdbc.dll b/relocate_natives/cache/dh_sqlite/native/Windows/x86_64/sqlitejdbc.dll new file mode 100644 index 000000000..2575a5cd0 Binary files /dev/null and b/relocate_natives/cache/dh_sqlite/native/Windows/x86_64/sqlitejdbc.dll differ