diff --git a/.gitignore b/.gitignore index c40f86573..373e80267 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,8 @@ Merged/ # Folder created by the buildAll scripts buildAllJars/ -.venv +relocate_natives/.venv/ +relocate_natives/apple-codesign/ relocate_natives/cache/ # file from notepad++ diff --git a/build.gradle b/build.gradle index fa29fc8bb..852f30ef2 100644 --- a/build.gradle +++ b/build.gradle @@ -343,7 +343,12 @@ subprojects { p -> String path = context.path.replace("org/sqlite", "dh_sqlite") byte[] content = context.is.readAllBytes() - content = RelocateNatives.processNative(path, content) + try { + content = RelocateNatives.processNative(path, content) + } + catch (Throwable e) { + throw new GradleException("Failed to relocate", e) + } rewrittenFiles.put(path, content) } diff --git a/buildSrc/src/main/java/RelocateNatives.java b/buildSrc/src/main/java/RelocateNatives.java index ff0cb1854..f8a5b1c4f 100644 --- a/buildSrc/src/main/java/RelocateNatives.java +++ b/buildSrc/src/main/java/RelocateNatives.java @@ -1,4 +1,3 @@ -import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -6,63 +5,16 @@ import java.util.concurrent.CompletableFuture; class RelocateNatives { - static boolean prepared = false; + private static boolean prepared = false; + + private static final Path rootDirectory = Path.of(System.getProperty("user.dir"), "relocate_natives"); + private static final Path cacheRoot = rootDirectory.resolve("cache"); - private static void ensurePrepared() throws Exception - { - if (prepared) - { - return; - } - prepared = true; - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.directory(new File(System.getProperty("user.dir"))); - - String os = System.getProperty("os.name").toLowerCase(); - if (os.contains("win")) - { - processBuilder.command("powershell", "./relocate_natives/prepare.ps1"); - } - else if (os.contains("nix") || os.contains("nux") || os.contains("mac")) - { - processBuilder.command("./relocate_natives/prepare.sh"); - } - else - { - throw new IllegalStateException("Unsupported operating system: " + os); - } - - Process process = processBuilder.start(); - process.getInputStream().transferTo(System.out); - process.getErrorStream().transferTo(System.err); - - int exitCode = process.waitFor(); - if (exitCode != 0) - { - throw new Exception("Prepare failed: " + exitCode); - } - } @SuppressWarnings({"ResultOfMethodCallIgnored", "BusyWait"}) - public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + private static CompletableFuture readOutputStreams(Process process) { - ensurePrepared(); - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.directory(new File(System.getProperty("user.dir"))); - - if (Path.of(System.getProperty("user.dir"), ".venv/Scripts").toFile().exists()) - { - processBuilder.command("./.venv/Scripts/python", "./relocate_natives/process.py", outputFilePath.toString()); - } - else - { - processBuilder.command("./.venv/bin/python", "./relocate_natives/process.py", outputFilePath.toString()); - } - - Process process = processBuilder.start(); - CompletableFuture outputFuture = CompletableFuture.runAsync(() -> { + return CompletableFuture.runAsync(() -> { try { while (process.isAlive() || process.getInputStream().available() > 0 || process.getErrorStream().available() > 0) @@ -86,6 +38,62 @@ class RelocateNatives { } }); + } + + private static void ensurePrepared() throws Exception + { + if (prepared) + { + return; + } + prepared = true; + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(rootDirectory.toFile()); + + String os = System.getProperty("os.name").toLowerCase(); + if (os.contains("win")) + { + processBuilder.command("powershell", "./prepare.ps1"); + } + else if (os.contains("nix") || os.contains("nux") || os.contains("mac")) + { + processBuilder.command("./prepare.sh"); + } + else + { + throw new IllegalStateException("Unsupported operating system: " + os); + } + + Process process = processBuilder.start(); + CompletableFuture outputFuture = readOutputStreams(process); + + int exitCode = process.waitFor(); + outputFuture.get(); + + if (exitCode != 0) + { + throw new Exception("Prepare failed: " + exitCode); + } + } + + public static byte[] updateUsingLief(Path outputFilePath, byte[] content) throws Exception + { + ensurePrepared(); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(rootDirectory.toFile()); + + processBuilder.command( + rootDirectory.resolve(".venv/Scripts").toFile().exists() + ? rootDirectory.resolve(".venv/Scripts/python.exe").toString() + : rootDirectory.resolve(".venv/bin/python").toString(), + "./process.py", + outputFilePath.toString() + ); + + Process process = processBuilder.start(); + CompletableFuture outputFuture = readOutputStreams(process); process.getOutputStream().write(content); process.getOutputStream().close(); @@ -141,7 +149,6 @@ class RelocateNatives public static byte[] processNative(String path, byte[] content) throws Exception { - Path cacheRoot = Path.of(System.getProperty("user.dir"), "relocate_natives/cache/"); Path outputFilePath = cacheRoot.resolve(path); outputFilePath.getParent().toFile().mkdirs(); diff --git a/relocate_natives/download_codesign.py b/relocate_natives/download_codesign.py new file mode 100644 index 000000000..b2d938c49 --- /dev/null +++ b/relocate_natives/download_codesign.py @@ -0,0 +1,110 @@ +import os +import platform +import requests +import tarfile +import zipfile +from pathlib import Path + + +def get_platform_specific_filename(): + system = platform.system() + machine = platform.machine() + + if system == "Darwin": + if machine == "arm64": + return "apple-codesign-*-aarch64-apple-darwin.tar.gz" + else: + return "apple-codesign-*-x86_64-apple-darwin.tar.gz" + elif system == "Linux": + if machine == "aarch64": + return "apple-codesign-*-aarch64-unknown-linux-musl.tar.gz" + else: + return "apple-codesign-*-x86_64-unknown-linux-musl.tar.gz" + elif system == "Windows": + if machine.endswith("64"): + return "apple-codesign-*-x86_64-pc-windows-msvc.zip" + else: + return "apple-codesign-*-i686-pc-windows-msvc.zip" + else: + raise Exception(f"Unsupported platform: {system} {machine}") + + +def download_and_unpack(): + dest_dir = Path("./apple-codesign") + + repo_url = "https://api.github.com/repos/indygreg/apple-platform-rs/releases/latest" + dest_dir.mkdir(exist_ok=True) + + # Fetch the latest release info from GitHub + print("Fetching latest release information...") + response = requests.get(repo_url) + response.raise_for_status() + release_data = response.json() + + # Ensure release data has assets + if "assets" not in release_data: + raise Exception("Release data does not contain assets.") + + # Determine the correct asset + platform_filename = get_platform_specific_filename() + asset = next((asset for asset in release_data["assets"] if asset["name"].startswith("apple-codesign-") and asset["name"].endswith(platform_filename.split("*")[-1])), None) + + if not asset: + raise Exception(f"No matching asset found for platform: {platform_filename}") + + # Download the archive + print(f"Downloading {asset['name']}...") + download_url = asset["browser_download_url"] + archive_path = dest_dir / asset["name"] + + with requests.get(download_url, stream=True) as r: + r.raise_for_status() + with open(archive_path, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + + print(f"Downloaded to {archive_path}") + + # Extract the archive + print("Extracting archive...") + temp_extract_dir = dest_dir / "temp_extract" + temp_extract_dir.mkdir(parents=True, exist_ok=True) + + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path, "r") as zip_ref: + zip_ref.extractall(temp_extract_dir) + elif archive_path.suffixes[-2:] == [".tar", ".gz"]: + with tarfile.open(archive_path, "r:gz") as tar_ref: + tar_ref.extractall(temp_extract_dir) + else: + raise Exception(f"Unknown archive format: {archive_path}") + + # Move contents of the root directory inside the archive to dest_dir + root_dir = next(temp_extract_dir.iterdir()) # Assuming only one root directory + for item in root_dir.iterdir(): + target_path = dest_dir / item.name + if target_path.exists(): + if target_path.is_dir(): + os.rmdir(target_path) + else: + os.remove(target_path) + item.rename(target_path) + + # Clean up temporary directories + for item in temp_extract_dir.iterdir(): + if item.is_dir(): + os.rmdir(item) + temp_extract_dir.rmdir() + + print(f"Extracted to {dest_dir}") + + # Clean up the archive + os.remove(archive_path) + print(f"Removed archive {archive_path}") + + +if __name__ == "__main__": + try: + download_and_unpack() + except Exception as e: + print(f"Error: {e}") diff --git a/relocate_natives/prepare.ps1 b/relocate_natives/prepare.ps1 index 36dcfc186..218ba6980 100644 --- a/relocate_natives/prepare.ps1 +++ b/relocate_natives/prepare.ps1 @@ -2,4 +2,4 @@ $ErrorActionPreference = "Stop" python -m venv .venv .\.venv\Scripts\activate -pip install lief +pip install -r requirements.txt diff --git a/relocate_natives/prepare.sh b/relocate_natives/prepare.sh index 5bfd06a29..52f9b3a66 100755 --- a/relocate_natives/prepare.sh +++ b/relocate_natives/prepare.sh @@ -4,4 +4,4 @@ set -e python -m venv .venv . ./.venv/bin/activate -pip install lief +pip install -r requirements.txt diff --git a/relocate_natives/process.py b/relocate_natives/process.py index 1c6c1c103..573581feb 100644 --- a/relocate_natives/process.py +++ b/relocate_natives/process.py @@ -1,8 +1,26 @@ import sys import lief +import subprocess +import download_codesign +from pathlib import Path -# print("imported", file=sys.stderr) - +output_path = sys.argv[1] binary = lief.parse(sys.stdin.buffer.read()) -# print([func.name for func in binary.exported_functions], file=sys.stderr) -binary.write(sys.argv[1]) + +if binary is None: + exit(1) + +if isinstance(binary, lief.MachO.Binary): + binary.remove_signature() + +binary.write(output_path) + +if isinstance(binary, lief.MachO.Binary): + print(f"Signing {output_path}...") + + if not Path("./apple-codesign/COPYING").exists(): + download_codesign.download_and_unpack() + + sign_process = subprocess.Popen(["./apple-codesign/rcodesign", "sign", output_path], shell=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sign_process.wait() diff --git a/relocate_natives/requirements.txt b/relocate_natives/requirements.txt new file mode 100644 index 000000000..f8d132bea Binary files /dev/null and b/relocate_natives/requirements.txt differ