diff options
| -rw-r--r-- | Cargo.lock | 2 | ||||
| -rw-r--r-- | LICENSE.md | 4 | ||||
| -rw-r--r-- | crates/shirabe-php-rpc/Cargo.toml | 4 | ||||
| -rw-r--r-- | crates/shirabe-php-rpc/build.rs | 363 | ||||
| -rw-r--r-- | crates/shirabe-php-rpc/php/worker.php | 20 | ||||
| -rw-r--r-- | crates/shirabe-php-rpc/src/composer_runtime.rs | 181 | ||||
| -rw-r--r-- | crates/shirabe-php-rpc/src/lib.rs | 6 | ||||
| -rw-r--r-- | crates/shirabe-php-shim/src/phar.rs | 39 | ||||
| -rw-r--r-- | crates/shirabe/src/event_dispatcher/event_dispatcher.rs | 36 | ||||
| -rw-r--r-- | crates/shirabe/tests/common/php_worker.rs | 4 | ||||
| -rw-r--r-- | crates/shirabe/tests/installed_versions_test.rs | 9 | ||||
| -rw-r--r-- | docs/dev/composer-runtime-bundle.md | 40 | ||||
| -rw-r--r-- | docs/dev/php-rpc.md | 13 | ||||
| -rwxr-xr-x | scripts/linters/lint | 4 | ||||
| -rw-r--r-- | scripts/linters/src/Linters/NoHaltCompilerLiteral.php | 79 |
15 files changed, 772 insertions, 32 deletions
@@ -2154,8 +2154,10 @@ name = "shirabe-php-rpc" version = "0.0.1" dependencies = [ "anyhow", + "flate2", "indexmap", "nix", + "sha2 0.11.0", "shirabe-php-shim", "shirabe-php-src", "shirabe-symfony-process", @@ -42,6 +42,10 @@ license of the package it is ported from: | [`shirabe-symfony-process`](crates/shirabe-symfony-process/LICENSE) | symfony/process | | [`shirabe-symfony-string`](crates/shirabe-symfony-string/LICENSE) | symfony/string | +The Shirabe executable embeds these PHP sources into its own binary as the +Composer runtime bundle ([docs/dev/composer-runtime-bundle.md](docs/dev/composer-runtime-bundle.md)). +The bundle has the `LICENSE` file of every package. + ## PHP The [`shirabe-php-src`](crates/shirabe-php-src/LICENSE) crate contains a Rust diff --git a/crates/shirabe-php-rpc/Cargo.toml b/crates/shirabe-php-rpc/Cargo.toml index 18d2a695..ed62eba5 100644 --- a/crates/shirabe-php-rpc/Cargo.toml +++ b/crates/shirabe-php-rpc/Cargo.toml @@ -16,5 +16,9 @@ indexmap.workspace = true nix.workspace = true tempfile.workspace = true +[build-dependencies] +flate2.workspace = true +sha2.workspace = true + [lints] workspace = true diff --git a/crates/shirabe-php-rpc/build.rs b/crates/shirabe-php-rpc/build.rs new file mode 100644 index 00000000..07a77a07 --- /dev/null +++ b/crates/shirabe-php-rpc/build.rs @@ -0,0 +1,363 @@ +//! Builds the Composer PHP runtime phar bundle embedded in the distributed binary. See also +//! `docs/dev/composer-runtime-bundle.md`. +//! +//! ref: composer/src/Composer/Compiler.php + +use std::io::Write as _; +use std::path::Path; +use std::path::PathBuf; + +/// The directory names Symfony's `Finder::ignoreVCS(true)` excludes. +const VCS_DIRECTORIES: &[&str] = &[ + ".svn", + "_svn", + "CVS", + "_darcs", + ".arch-params", + ".monotone", + ".bzr", + ".git", + ".hg", +]; + +/// Directories `Compiler::compile` excludes from the vendor tree, on top of the VCS ones. +const VENDOR_EXCLUDED_DIRECTORIES: &[&str] = &["Tests", "tests", "docs"]; + +/// Vendor paths `Compiler::compile` expects in the archive even though they are not PHP sources +/// or licenses; a missing one means the source package changed under us. +const VENDOR_EXTRA_FILES: &[&str] = &[ + "composer/installed.json", + "composer/spdx-licenses/res/spdx-exceptions.json", + "composer/spdx-licenses/res/spdx-licenses.json", + "composer/ca-bundle/res/cacert.pem", + "symfony/console/Resources/bin/hiddeninput.exe", + "symfony/console/Resources/completion.bash", +]; + +/// The path of the entry whose read-back tells the worker that the bundle is usable. +const SENTINEL_PATH: &str = "shirabe/bundle-id"; + +struct Entry { + /// Path inside the archive, relative to the Composer checkout root. + path: String, + content: Vec<u8>, +} + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let composer = manifest_dir + .parent() + .unwrap() + .parent() + .unwrap() + .join("composer"); + for path in ["src", "res", "vendor", "bin/composer", "LICENSE"] { + println!("cargo::rerun-if-changed={}", composer.join(path).display()); + } + + if !composer.join("vendor/autoload.php").is_file() { + panic!( + "the Composer PHP runtime is missing from {}; run `git submodule update --init` \ + and `composer install` in it", + composer.display() + ); + } + + let mut entries = collect_entries(&composer); + let bundle_id = bundle_id(&entries); + entries.push(Entry { + path: SENTINEL_PATH.to_string(), + content: bundle_id.clone().into_bytes(), + }); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + std::fs::write( + out_dir.join("composer-runtime-bundle.phar"), + build_phar(&entries), + ) + .unwrap(); + println!("cargo::rustc-env=SHIRABE_COMPOSER_RUNTIME_BUNDLE_ID={bundle_id}"); +} + +/// Every file the bundle holds, in the order `Compiler::compile` adds them. +fn collect_entries(composer: &Path) -> Vec<Entry> { + let mut entries = Vec::new(); + + // Add Composer sources. Compiler.php excludes ClassLoader.php and InstalledVersions.php from + // this pass only to add them back unstripped; nothing here strips whitespace, so they are + // taken along with the rest. + for file in find_files(&composer.join("src")) { + if file.extension().and_then(|e| e.to_str()) != Some("php") + || file_name(&file) == "Compiler.php" + { + continue; + } + entries.push(read_entry(composer, &file, true)); + } + + // Add Composer resources. + for file in find_files(&composer.join("res")) { + entries.push(read_entry(composer, &file, false)); + } + + // Add vendor files. + let mut extra_files: Vec<&str> = VENDOR_EXTRA_FILES.to_vec(); + let mut unexpected_files = Vec::new(); + let vendor = composer.join("vendor"); + for file in find_files(&vendor) { + let relative = relative_path(&vendor, &file); + if is_excluded_vendor_path(&relative) { + continue; + } + if let Some(index) = extra_files.iter().position(|extra| *extra == relative) { + extra_files.remove(index); + } else if !is_license_or_php(&file_name(&file)) { + unexpected_files.push(relative.clone()); + } + let strip = is_php_source(&file_name(&file)); + entries.push(read_entry(composer, &file, strip)); + } + if !extra_files.is_empty() { + panic!( + "these files were expected but not added to the phar, they might be excluded or gone \ + from the source package: {extra_files:?}" + ); + } + if !unexpected_files.is_empty() { + panic!( + "these files were unexpectedly added to the phar, make sure they are excluded or \ + listed in VENDOR_EXTRA_FILES: {unexpected_files:?}" + ); + } + + // Add bin/composer. + let bin = std::fs::read(composer.join("bin/composer")).unwrap(); + let shebang = b"#!/usr/bin/env php"; + let content = match bin.strip_prefix(&shebang[..]) { + Some(rest) => { + let trimmed = rest + .iter() + .position(|byte| !byte.is_ascii_whitespace()) + .unwrap_or(rest.len()); + rest[trimmed..].to_vec() + } + None => bin, + }; + entries.push(Entry { + path: "bin/composer".to_string(), + content, + }); + + entries.push(read_entry(composer, &composer.join("LICENSE"), false)); + + entries +} + +/// Reads one file the way `Compiler::addFile` does, minus its whitespace stripping: the bundle +/// keeps the sources byte for byte, so a class the worker loads from here is the same file the +/// Rust side embeds elsewhere. +fn read_entry(composer: &Path, file: &Path, strip: bool) -> Entry { + let mut content = std::fs::read(file).unwrap(); + if !strip && file_name(file) == "LICENSE" { + content.insert(0, b'\n'); + content.push(b'\n'); + } + Entry { + path: relative_path(composer, file), + content, + } +} + +/// The files under `root`, sorted by path, with the entries Symfony's `Finder` skips by default +/// (dot files and VCS directories) left out. +fn find_files(root: &Path) -> Vec<PathBuf> { + let mut files = Vec::new(); + walk(root, &mut files); + files.sort(); + files +} + +fn walk(dir: &Path, files: &mut Vec<PathBuf>) { + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + let name = file_name(&path); + if name.starts_with('.') { + continue; + } + if entry.file_type().unwrap().is_dir() { + if VCS_DIRECTORIES.contains(&name.as_str()) { + continue; + } + walk(&path, files); + } else { + files.push(path); + } + } +} + +fn file_name(path: &Path) -> String { + path.file_name().unwrap().to_str().unwrap().to_string() +} + +fn relative_path(root: &Path, file: &Path) -> String { + file.strip_prefix(root) + .unwrap() + .to_str() + .unwrap() + .to_string() +} + +/// The `notPath`/`exclude` rules `Compiler::compile` applies to the vendor tree, against a path +/// relative to `vendor/`. +fn is_excluded_vendor_path(relative: &str) -> bool { + // Every rule wants a `/` in front of the file name, so a file sitting directly in `vendor/` + // (autoload.php) is never matched. + let Some((directories, name)) = relative.rsplit_once('/') else { + return false; + }; + if directories + .split('/') + .any(|segment| VENDOR_EXCLUDED_DIRECTORIES.contains(&segment)) + { + return true; + } + for substring in [ + "justinrainbow/json-schema/demo/", + "justinrainbow/json-schema/dist/", + "justinrainbow/json-schema/bin/", + "composer/pcre/extension.neon", + "composer/LICENSE", + ] { + if relative.contains(substring) { + return true; + } + } + if name.starts_with("UPGRADE") && (name.ends_with(".md") || name.ends_with(".txt")) { + return true; + } + if name.ends_with(".md") || name.ends_with(".mdown") { + let stem = name.trim_end_matches(".mdown").trim_end_matches(".md"); + if !stem.is_empty() && stem.bytes().all(|byte| byte.is_ascii_uppercase()) { + return true; + } + } + matches!( + name, + "composer.json" + | "composer.lock" + | ".gitignore" + | "appveyor.yml" + | "phpunit.xml.dist" + | "phpstan.neon.dist" + | "phpstan-config.neon" + | "phpstan-baseline.neon" + ) || matches!( + relative.rsplit_once("bin/").map(|(_, name)| name), + Some( + "jsonlint" + | "validate-json" + | "simple-phpunit" + | "phpstan" + | "phpstan.phar" + | "jsonlint.bat" + | "validate-json.bat" + | "simple-phpunit.bat" + | "phpstan.bat" + | "phpstan.phar.bat" + ) + ) +} + +/// `Compiler::compile`'s guard against silently packing something that is neither a license nor a +/// PHP source. +fn is_license_or_php(name: &str) -> bool { + name == "LICENSE" || name == "LICENSE.txt" || name.ends_with(".php") +} + +/// Whether `Compiler::compile` treats a vendor file as PHP source (`{\.php[\d.]*$}`). +fn is_php_source(name: &str) -> bool { + match name.rsplit_once(".php") { + Some((_, suffix)) => suffix + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'.'), + None => false, + } +} + +/// Identifies the bundle by its contents, naming both the sentinel entry and the directory the +/// Rust side extracts to when the worker cannot read the bundle in place. +fn bundle_id(entries: &[Entry]) -> String { + use sha2::Digest as _; + + let mut hasher = sha2::Sha256::new(); + for entry in entries { + hasher.update((entry.path.len() as u64).to_le_bytes()); + hasher.update(entry.path.as_bytes()); + hasher.update((entry.content.len() as u64).to_le_bytes()); + hasher.update(&entry.content); + } + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +const PHAR_FILE_COMPRESSED_GZ: u32 = 0x0000_1000; + +/// Lays out an unsigned native phar. Unsigned because a phar signature covers everything from the +/// start of the file, which for an embedded bundle means bytes the linker has not produced yet. +fn build_phar(entries: &[Entry]) -> Vec<u8> { + let mut manifest = Vec::new(); + let mut contents = Vec::new(); + let mut global_flags = 0u32; + + for entry in entries { + let mut encoder = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::best()); + encoder.write_all(&entry.content).unwrap(); + let deflated = encoder.finish().unwrap(); + let (data, flags) = if deflated.len() < entry.content.len() { + global_flags |= PHAR_FILE_COMPRESSED_GZ; + (deflated, 0o644 | PHAR_FILE_COMPRESSED_GZ) + } else { + (entry.content.clone(), 0o644) + }; + + manifest.extend_from_slice(&(entry.path.len() as u32).to_le_bytes()); + manifest.extend_from_slice(entry.path.as_bytes()); + manifest.extend_from_slice(&(entry.content.len() as u32).to_le_bytes()); + // A fixed timestamp, so that the same checkout always produces the same bundle. + manifest.extend_from_slice(&0u32.to_le_bytes()); + manifest.extend_from_slice(&(data.len() as u32).to_le_bytes()); + manifest.extend_from_slice(&crc32(&entry.content).to_le_bytes()); + manifest.extend_from_slice(&flags.to_le_bytes()); + manifest.extend_from_slice(&0u32.to_le_bytes()); + contents.extend_from_slice(&data); + } + + let mut header = Vec::new(); + header.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + // API version 1.1.0, the only field phar reads big-endian. + header.extend_from_slice(&[0x11, 0x00]); + header.extend_from_slice(&global_flags.to_le_bytes()); + // Neither an alias nor metadata: the alias is the one the worker passes to Phar::loadPhar. + header.extend_from_slice(&0u32.to_le_bytes()); + header.extend_from_slice(&0u32.to_le_bytes()); + + let mut bytes = b"<?php __HALT_COMPILER(); ?>\r\n".to_vec(); + bytes.extend_from_slice(&((header.len() + manifest.len()) as u32).to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.extend_from_slice(&manifest); + bytes.extend_from_slice(&contents); + bytes +} + +fn crc32(data: &[u8]) -> u32 { + let mut crc = flate2::Crc::new(); + crc.update(data); + crc.sum() +} diff --git a/crates/shirabe-php-rpc/php/worker.php b/crates/shirabe-php-rpc/php/worker.php index 5946a52f..d5fe67d4 100644 --- a/crates/shirabe-php-rpc/php/worker.php +++ b/crates/shirabe-php-rpc/php/worker.php @@ -716,6 +716,26 @@ ShirabeRpcRuntime::$dispatch = [ ShirabeRpcRuntime::ensureStubAutoloaderPriority(); return true; }, + // Tries to open the embedded runtime bundle and maps the phar to $alias. + // Returns whether the extraction succeeds or not. + '__shirabe_open_runtime_bundle' => static function ($args) { + [$executable, $alias, $sentinel, $bundleId] = $args; + if (!extension_loaded('phar') || !in_array('phar', stream_get_wrappers(), true)) { + return false; + } + try { + Phar::loadPhar($executable, $alias); + $read = @file_get_contents("phar://{$alias}/{$sentinel}"); + } catch (Throwable $e) { + $read = false; + } + // The phar bundle is unsigned, so the worker process is started with + // phar.require_hash=0. Verification result is saved and not verified + // again for the same phar file, so it is safe to restore the settings + // here. + ini_set('phar.require_hash', '1'); + return $read === $bundleId; + }, '__shirabe_enable_script_autoloader' => static function ($args) { ShirabeRpcRuntime::enableScriptAutoloader(); return true; diff --git a/crates/shirabe-php-rpc/src/composer_runtime.rs b/crates/shirabe-php-rpc/src/composer_runtime.rs new file mode 100644 index 00000000..5336a83f --- /dev/null +++ b/crates/shirabe-php-rpc/src/composer_runtime.rs @@ -0,0 +1,181 @@ +//! The Composer PHP runtime the worker loads. +//! +//! See `docs/dev/composer-runtime-bundle.md`. + +use crate::PluginValue; +use crate::call_function; + +const BUNDLE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/composer-runtime-bundle.phar")); + +/// Identifies the bundle by its contents; the sentinel entry holds the same string. +const BUNDLE_ID: &str = env!("SHIRABE_COMPOSER_RUNTIME_BUNDLE_ID"); + +/// The entry the worker reads back to tell a bundle it can use from one it cannot. +const SENTINEL_PATH: &str = "shirabe/bundle-id"; + +/// The alias `Phar::loadPhar` maps the bundle to in the worker. The bundle stores no alias of +/// its own, so this name is the only one its stream paths answer to. +const ALIAS: &str = "shirabe-composer-runtime.phar"; + +/// The path the Composer PHP runtime's files sit under in the worker, either inside this +/// executable or in the directory the bundle was extracted to. +pub fn base_path() -> anyhow::Result<String> { + static BASE: std::sync::OnceLock<Result<String, String>> = std::sync::OnceLock::new(); + BASE.get_or_init(|| resolve().map_err(|e| format!("{e:#}"))) + .clone() + .map_err(|e| anyhow::anyhow!(e)) +} + +fn resolve() -> anyhow::Result<String> { + if worker_opens_bundle()? { + return Ok(format!("phar://{ALIAS}")); + } + let directory = extract()?; + directory.into_os_string().into_string().map_err(|path| { + anyhow::anyhow!("the extracted Composer PHP runtime path {path:?} is not valid UTF-8") + }) +} + +/// Whether the worker can read the bundle straight out of this executable. It cannot when its +/// PHP has no phar extension, no zlib to inflate the entries, or a restriction on the phar stream +/// wrapper. +fn worker_opens_bundle() -> anyhow::Result<bool> { + let executable = std::env::current_exe()?; + let executable = executable.to_str().ok_or_else(|| { + anyhow::anyhow!("the path of this executable, {executable:?}, is not valid UTF-8") + })?; + let opened = call_function( + "__shirabe_open_runtime_bundle", + vec![ + PluginValue::string(executable), + PluginValue::string(ALIAS), + PluginValue::string(SENTINEL_PATH), + PluginValue::string(BUNDLE_ID), + ], + )? + .map_err(|throw| anyhow::anyhow!("{throw}"))?; + match opened { + PluginValue::Bool(opened) => Ok(opened), + other => Err(anyhow::anyhow!( + "opening the Composer PHP runtime bundle did not answer with a bool: {other:?}" + )), + } +} + +/// Unpacks the bundle into a content-addressed directory, so that a worker that cannot read the +/// bundle in place gets the same files from the filesystem. +fn extract() -> anyhow::Result<std::path::PathBuf> { + extract_into(&cache_directory()?.join("shirabe").join("runtime")) +} + +fn extract_into(root: &std::path::Path) -> anyhow::Result<std::path::PathBuf> { + let destination = root.join(BUNDLE_ID); + if destination.is_dir() { + return Ok(destination); + } + + std::fs::create_dir_all(root)?; + let staging = tempfile::tempdir_in(root)?; + let archive = staging.path().join("bundle.phar"); + std::fs::write(&archive, BUNDLE)?; + let unpacked = staging.path().join("unpacked"); + shirabe_php_shim::Phar::new(&archive)?.extract_to(&unpacked, None, true)?; + std::fs::remove_file(&archive)?; + + if let Err(error) = std::fs::rename(&unpacked, &destination) { + // Losing the race against another process that unpacked the same bundle is not a + // failure: the directory is named after the contents that went into it. + if !destination.is_dir() { + return Err(error.into()); + } + } + Ok(destination) +} + +fn cache_directory() -> anyhow::Result<std::path::PathBuf> { + if let Some(directory) = std::env::var_os("XDG_CACHE_HOME") + && !directory.is_empty() + { + return Ok(std::path::PathBuf::from(directory)); + } + let home = std::env::var_os("HOME") + .ok_or_else(|| anyhow::anyhow!("neither XDG_CACHE_HOME nor HOME is set"))?; + Ok(std::path::Path::new(&home).join(".cache")) +} + +#[cfg(test)] +mod tests { + use super::*; + use shirabe_symfony_process::PhpExecutableFinder; + + fn sentinel_of(directory: &std::path::Path) -> String { + std::fs::read_to_string(directory.join(SENTINEL_PATH)).expect("no sentinel in the bundle") + } + + /// PHP takes the first occurrence of the stub token in the file it opens, so no other copy of + /// it may precede the bundle in the executable. + #[test] + fn the_first_phar_in_this_executable_is_the_bundle() { + let executable = std::env::current_exe().expect("no path for this executable"); + let unpacked = tempfile::tempdir().unwrap(); + shirabe_php_shim::Phar::new(&executable) + .expect("this executable does not read back as a phar") + .extract_to(unpacked.path(), None, true) + .unwrap(); + + assert_eq!(sentinel_of(unpacked.path()), BUNDLE_ID); + } + + #[test] + fn extracting_the_bundle_names_the_directory_after_it() { + let root = tempfile::tempdir().unwrap(); + let directory = extract_into(root.path()).unwrap(); + + assert_eq!(directory, root.path().join(BUNDLE_ID)); + assert_eq!(sentinel_of(&directory), BUNDLE_ID); + assert!(directory.join("vendor/autoload.php").is_file()); + // A second call finds the unpacked bundle and leaves it alone. + assert_eq!(extract_into(root.path()).unwrap(), directory); + } + + #[test] + fn the_worker_reads_the_bundle_out_of_this_executable() { + if PhpExecutableFinder::new().find(false).is_none() { + // No PHP in this environment; the worker cannot start. + return; + } + + assert_eq!(base_path().unwrap(), format!("phar://{ALIAS}")); + } + + /// The other half of `base_path`: a worker whose PHP cannot open the bundle is handed the + /// unpacked tree, and has to reach the same classes through it. + #[test] + fn the_worker_loads_the_composer_runtime_from_an_unpacked_bundle() { + if PhpExecutableFinder::new().find(false).is_none() { + // No PHP in this environment; the worker cannot start. + return; + } + + let root = tempfile::tempdir().unwrap(); + let autoload = extract_into(root.path()) + .unwrap() + .join("vendor/autoload.php"); + call_function( + "__shirabe_require", + vec![PluginValue::string(autoload.to_str().unwrap())], + ) + .unwrap() + .unwrap(); + + // A class with no proxy stub, so that the answer is about the runtime and not about the + // stub autoloader. + let exists = call_function( + "class_exists", + vec![PluginValue::string(r"Composer\Util\Filesystem")], + ) + .unwrap() + .unwrap(); + assert_eq!(exists, PluginValue::Bool(true)); + } +} diff --git a/crates/shirabe-php-rpc/src/lib.rs b/crates/shirabe-php-rpc/src/lib.rs index a59ef648..e5234ecd 100644 --- a/crates/shirabe-php-rpc/src/lib.rs +++ b/crates/shirabe-php-rpc/src/lib.rs @@ -1,5 +1,6 @@ //! Rust-to-PHP RPC over a Unix domain socket. See `docs/dev/php-rpc.md`. +pub mod composer_runtime; pub mod frame; pub mod session; pub mod value; @@ -1093,7 +1094,10 @@ fn spawn_worker() -> anyhow::Result<Worker> { // supported) serialize_precision; pin the child to it in case a distro php.ini overrides // the default. .arg("-d") - .arg("serialize_precision=-1"); + .arg("serialize_precision=-1") + // The Composer runtime bundle has no phar signature. + .arg("-d") + .arg("phar.require_hash=0"); if xdebug::switches_xdebug_off() { // The environment variable takes precedence over every ini setting, so switching the // mode off takes both. See `docs/dev/xdebug.md`. diff --git a/crates/shirabe-php-shim/src/phar.rs b/crates/shirabe-php-shim/src/phar.rs index dccdfebf..ecd842f3 100644 --- a/crates/shirabe-php-shim/src/phar.rs +++ b/crates/shirabe-php-shim/src/phar.rs @@ -286,15 +286,35 @@ fn verify_phar_signature(path: &std::path::Path, bytes: &[u8]) -> anyhow::Result Ok(()) } +/// The special token to separate a phar stub and contents. +/// +/// The Shirabe executable embeds the Composer runtime bundle as a phar archive, so the token must +/// not appear in the binary. See `docs/dev/composer-runtime-bundle.md`. We use `black_box()` to +/// prevent the Rust compiler from inlining the function to a constant. +fn halt_compiler_token() -> [u8; 18] { + let mut token = *std::hint::black_box(b"__UNYG_PBZCVYRE();"); + for byte in &mut token { + if byte.is_ascii_uppercase() { + *byte = b'A' + (*byte - b'A' + 13) % 26; + } + } + token +} + fn parse_native_phar(path: &std::path::Path) -> anyhow::Result<Vec<PharEntry>> { let bytes = std::fs::read(path) .map_err(|e| corruption_error(path, &format!("unable to open phar: {}", e)))?; - let halt = b"__HALT_COMPILER();"; + let halt = halt_compiler_token(); let halt_pos = bytes .windows(halt.len()) .position(|window| window == halt) - .ok_or_else(|| corruption_error(path, "__HALT_COMPILER(); not found in stub"))?; + .ok_or_else(|| { + corruption_error( + path, + &format!("{} not found in stub", String::from_utf8_lossy(&halt)), + ) + })?; let mut offset = halt_pos + halt.len(); for close_tag in [&b" ?>"[..], &b"\n?>"[..]] { if bytes[offset..].starts_with(close_tag) { @@ -846,6 +866,17 @@ mod tests { use super::*; use crate::Catch as _; + #[test] + fn halt_compiler_token_decodes() { + assert_eq!( + halt_compiler_token(), + [ + b'_', b'_', b'H', b'A', b'L', b'T', b'_', b'C', b'O', b'M', b'P', b'I', b'L', b'E', + b'R', b'(', b')', b';', + ], + ); + } + fn write_file(dir: &std::path::Path, name: &str, content: &[u8]) -> std::path::PathBuf { let path = dir.join(name); std::fs::create_dir_all(path.parent().unwrap()).unwrap(); @@ -971,7 +1002,9 @@ mod tests { manifest.extend_from_slice(&0u32.to_le_bytes()); } - let mut bytes = b"<?php __HALT_COMPILER(); ?>\r\n".to_vec(); + let mut bytes = b"<?php ".to_vec(); + bytes.extend_from_slice(&halt_compiler_token()); + bytes.extend_from_slice(b" ?>\r\n"); bytes.extend_from_slice(&(manifest.len() as u32).to_le_bytes()); bytes.extend_from_slice(&manifest); bytes.extend_from_slice(&stored.0); diff --git a/crates/shirabe/src/event_dispatcher/event_dispatcher.rs b/crates/shirabe/src/event_dispatcher/event_dispatcher.rs index 368e2227..8f0b54be 100644 --- a/crates/shirabe/src/event_dispatcher/event_dispatcher.rs +++ b/crates/shirabe/src/event_dispatcher/event_dispatcher.rs @@ -1566,16 +1566,7 @@ try {{ /// Loads the Composer PHP runtime (symfony/console and friends) into the worker, needed /// before a `scripts` Command class can be autoloaded and hosted. pub(crate) fn ensure_composer_php_runtime() -> anyhow::Result<()> { - // TODO(plugin): the real PHP classes are taken from a Composer checkout for now; how - // they ship with a released Shirabe binary is part of the plugin distribution work. - let autoload = Self::composer_php_runtime_autoload().ok_or_else(|| -> anyhow::Error { - RuntimeException::new( - "unable to locate the Composer PHP runtime; set SHIRABE_COMPOSER_PHP_DIR \ - to a Composer checkout with its vendor directory installed" - .to_string(), - ) - .into() - })?; + let autoload = Self::composer_php_runtime_autoload()?; unwrap_php_result(call_function( "__shirabe_require", vec![PluginValue::string(autoload)], @@ -1590,22 +1581,27 @@ try {{ Self::ensure_composer_php_runtime() } - fn composer_php_runtime_autoload() -> Option<String> { + /// The `vendor/autoload.php` of the Composer PHP runtime: the checkout `SHIRABE_COMPOSER_PHP_DIR` + /// points at, or else the runtime bundle the executable carries. + fn composer_php_runtime_autoload() -> anyhow::Result<String> { if let Some(dir) = Platform::get_env("SHIRABE_COMPOSER_PHP_DIR") { let path = std::path::Path::new(&dir) .join("vendor") .join("autoload.php"); - if path.is_file() { - return path.to_str().map(|s| s.to_string()); + if !path.is_file() { + return Err(RuntimeException::new(format!( + "SHIRABE_COMPOSER_PHP_DIR points at {dir}, which has no \ + vendor/autoload.php; install the checkout's dependencies or unset it to use \ + the runtime the executable carries" + )) + .into()); } + return Ok(path.display().to_string()); } - // Development fallback: the Composer checkout sitting next to this workspace. - let dev = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../composer/vendor/autoload.php"); - if dev.is_file() { - return dev.canonicalize().ok()?.to_str().map(|s| s.to_string()); - } - None + Ok(format!( + "{}/vendor/autoload.php", + shirabe_php_rpc::composer_runtime::base_path()? + )) } /// Runs a boolean runtime query (`class_exists`, `is_a`, ...) inside the PHP worker, with diff --git a/crates/shirabe/tests/common/php_worker.rs b/crates/shirabe/tests/common/php_worker.rs index 4be064e4..af1b6b44 100644 --- a/crates/shirabe/tests/common/php_worker.rs +++ b/crates/shirabe/tests/common/php_worker.rs @@ -24,8 +24,8 @@ pub fn lock_php_worker() -> std::sync::MutexGuard<'static, ()> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// Requires the Composer PHP runtime (`composer/vendor/autoload.php`) into the worker, which is -/// what makes the real `Composer\` classes autoloadable there. +/// Requires the Composer PHP runtime's `vendor/autoload.php` into the worker, which is what makes +/// the real `Composer\` classes autoloadable there. pub fn load_composer_php_runtime() { shirabe::event_dispatcher::EventDispatcher::__ensure_composer_php_runtime().unwrap(); } diff --git a/crates/shirabe/tests/installed_versions_test.rs b/crates/shirabe/tests/installed_versions_test.rs index 9b81c3c8..b8c65485 100644 --- a/crates/shirabe/tests/installed_versions_test.rs +++ b/crates/shirabe/tests/installed_versions_test.rs @@ -512,12 +512,19 @@ fn test_worker_loads_the_installed_versions_file_shirabe_dumps() { let _worker = lock_php_worker(); load_composer_php_runtime(); + // Read in the worker rather than from Rust: the file it autoloads lives inside the runtime + // bundle, which only the PHP side has a stream wrapper for. let loaded = string_of(&php_eval( r"return (new \ReflectionClass(\Composer\InstalledVersions::class))->getFileName();", )); + let contents = string_of(&php_eval( + r"return file_get_contents( + (new \ReflectionClass(\Composer\InstalledVersions::class))->getFileName() + );", + )); assert_eq!( include_str!("../../../composer/src/Composer/InstalledVersions.php"), - std::fs::read_to_string(&loaded).unwrap(), + contents, "the worker autoloads {loaded}, which must match the file Shirabe dumps", ); } diff --git a/docs/dev/composer-runtime-bundle.md b/docs/dev/composer-runtime-bundle.md new file mode 100644 index 00000000..f152897f --- /dev/null +++ b/docs/dev/composer-runtime-bundle.md @@ -0,0 +1,40 @@ +# Composer runtime bundle + +Composer plugins and scripts expect the real PHP classes in `Composer\` itself +and packages that Composer depends on to be available. Shirabe embeds the +runtime PHP sources into the binary at compile time. + +## Building and embedding the bundle + +Shirabe archives all PHP sources and resources of Composer and its dependencies, +and embeds the archived phar file into the binary. Shirabe also calculates a +hash value from all included files and puts a file that contains the hash, +`shirabe/bundle-id`. It is used for verifying the bundle. See +`crates/shirabe-php-rpc/build.rs` for details. + +### Phar signature + +A phar signature is an optional signature to verify the archive's integrity, and +must be appended to the end of the file. Shirabe's runtime bundle has no +signature because the bundle is not always at the end of the executable file. +To disable phar verification, the PHP worker is started with `-d phar.require_hash=0`. +The worker restores it to `1` right after opening the bundle, so that the rest +of the process still verifies the phars it opens. A verification result is +cached per file, and the bundle is never re-verified. + +### `__HALT_COMPILER();` tokens + +A phar file consists of 3 or 4 sections: a stub, a manifest, the actual contents +and an optional signature. The stub and the manifest are separated by +`__HALT_COMPILER();` tokens, which means that Shirabe's executable binary +must not contain the tokens except for phar's one. + +## Accessing files at runtime + +Shirabe tries to open the embedded bundle, and sets the executable as the base +path of autoloading. If loading phar fails for some reason, e.g., no phar ext, +Shirabe unpacks the archive to `<Shirabe's cache dir>/runtime/<bundle id>` once, +and uses that directory for the base path instead. + +NOTE: the environment variable `SHIRABE_COMPOSER_PHP_DIR` can override the +location for development. diff --git a/docs/dev/php-rpc.md b/docs/dev/php-rpc.md index 8418db88..b1c992d7 100644 --- a/docs/dev/php-rpc.md +++ b/docs/dev/php-rpc.md @@ -7,12 +7,15 @@ runtime. The `shirabe-php-rpc` crate spawns the system PHP as a child process and talks to it over a Unix domain socket. There is exactly one child process per Shirabe process, shared by every caller. -## Locating PHP +## Locating and spawning PHP -The existing `PhpExecutableFinder` class resolves the PHP binary. The child is started with -`-d serialize_precision=-1` so the wire codec's float formatting is pinned to the default PHP -behavior, and with `-d xdebug.mode=off` unless `COMPOSER_ALLOW_XDEBUG` asks for Xdebug to stay -(see `xdebug.md`). +The existing `PhpExecutableFinder` class resolves the PHP binary. + +The child is started with the following arguments: + +* `-d serialize_precision=-1` for stable float formatting of `serialize()`/`unserialize()` +* `-d phar.require_hash=0` (see [docs/dev/composer-runtime-bundle.md](./composer-runtime-bundle.md)) +* `-d xdebug.mode=off` (see [docs/dev/xdebug.md](./xdebug.md)) ## Transport diff --git a/scripts/linters/lint b/scripts/linters/lint index 5f59945d..e90c3e0c 100755 --- a/scripts/linters/lint +++ b/scripts/linters/lint @@ -11,6 +11,7 @@ use Shirabe\Lint\Linters\NoBannedUse; use Shirabe\Lint\Linters\NoDecorativeSectionComment; use Shirabe\Lint\Linters\NoExceptionDowncast; use Shirabe\Lint\Linters\NoFormatTrailingComma; +use Shirabe\Lint\Linters\NoHaltCompilerLiteral; use Shirabe\Lint\Linters\NoModRs; use Shirabe\Lint\Linters\NoStdCollectionsMaps; use Shirabe\Lint\Linters\NoUseAsAlias; @@ -39,6 +40,9 @@ $runner = new Runner($rootDir, [ 'crates/shirabe/src/package/loader/root_package_loader.rs', 'crates/shirabe-spdx-licenses/src/spdx_licenses.rs', ]], + [new NoHaltCompilerLiteral(), [ + 'crates/shirabe-php-rpc/build.rs', + ]], [new NoModRs(), []], [new NoStdCollectionsMaps(), []], [new NoUseAsAlias(), []], diff --git a/scripts/linters/src/Linters/NoHaltCompilerLiteral.php b/scripts/linters/src/Linters/NoHaltCompilerLiteral.php new file mode 100644 index 00000000..30520ff7 --- /dev/null +++ b/scripts/linters/src/Linters/NoHaltCompilerLiteral.php @@ -0,0 +1,79 @@ +<?php + +declare(strict_types=1); + +namespace Shirabe\Lint\Linters; + +use Shirabe\Lint\Linter; +use Shirabe\Lint\Support\FileFinder; +use Shirabe\Lint\Support\Paths; + +final class NoHaltCompilerLiteral implements Linter +{ + // Assembled so that this file does not contain what it looks for. + private const TOKEN = '__' . 'HALT_COMPILER();'; + + public function name(): string + { + return 'no_halt_compiler_literal'; + } + + public function failureIntro(): string + { + return "Found a literal `" . self::TOKEN . "`.\n" + . "The executable carries the Composer runtime bundle as a phar that PHP finds by\n" + . "scanning for the first occurrence of that token, and every literal here ends up in\n" + . "the same binary, so an earlier one shadows the bundle. Build the token at run time\n" + . "instead — see " + . '`halt_compiler_token` in `crates/shirabe-php-shim/src/phar.rs`:'; + } + + public function check(string $rootDir, array $excludes): array + { + $errors = []; + + foreach ($this->embeddedFiles($rootDir) as $path) { + $relative = Paths::relativeTo($rootDir, $path); + if (in_array($relative, $excludes, true)) { + continue; + } + + foreach (file($path) as $idx => $raw) { + if (!str_contains($raw, self::TOKEN)) { + continue; + } + + $errors[] = "{$relative}:" . ($idx + 1) . ': ' . trim($raw); + } + } + + return $errors; + } + + /** + * The sources whose bytes reach the binary: Rust code, and the PHP files the Rust code + * embeds with `include_str!`. + * + * @return list<string> + */ + private function embeddedFiles(string $rootDir): array + { + $paths = FileFinder::rustFiles($rootDir); + + foreach (glob("{$rootDir}/crates/*/php", GLOB_ONLYDIR) ?: [] as $phpDir) { + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($phpDir, \FilesystemIterator::SKIP_DOTS), + ); + foreach ($iterator as $file) { + /** @var \SplFileInfo $file */ + if ($file->isFile()) { + $paths[] = $file->getPathname(); + } + } + } + + sort($paths); + + return $paths; + } +} |