diff options
Diffstat (limited to 'crates/shirabe/src/downloader')
| -rw-r--r-- | crates/shirabe/src/downloader/archive_downloader.rs | 175 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/downloader_interface.rs | 7 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/file_downloader.rs | 313 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/fossil_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/git_downloader.rs | 2 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/gzip_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/hg_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/path_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/perforce_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/phar_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/rar_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/svn_downloader.rs | 2 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/tar_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/xz_downloader.rs | 1 | ||||
| -rw-r--r-- | crates/shirabe/src/downloader/zip_downloader.rs | 166 |
15 files changed, 573 insertions, 101 deletions
diff --git a/crates/shirabe/src/downloader/archive_downloader.rs b/crates/shirabe/src/downloader/archive_downloader.rs index ac898d5..6c8a8f9 100644 --- a/crates/shirabe/src/downloader/archive_downloader.rs +++ b/crates/shirabe/src/downloader/archive_downloader.rs @@ -2,15 +2,17 @@ use anyhow::Result; use indexmap::IndexMap; -use shirabe_external_packages::symfony::component::finder::Finder; +use shirabe_external_packages::symfony::component::finder::{Finder, SplFileInfo}; use shirabe_php_shim::{ - DIRECTORY_SEPARATOR, RuntimeException, bin2hex, file_exists, is_dir, random_bytes, realpath, + DIRECTORY_SEPARATOR, PhpMixed, RuntimeException, basename, bin2hex, file_exists, is_dir, + random_bytes, realpath, }; use crate::dependency_resolver::operation::InstallOperation; use crate::downloader::DownloaderInterface; use crate::downloader::FileDownloader; use crate::package::PackageInterface; +use crate::util::Filesystem; use crate::util::Platform; pub trait ArchiveDownloader { @@ -122,22 +124,79 @@ pub trait ArchiveDownloader { .ensure_directory_exists(&temporary_dir); let file_name = self.inner().get_file_name(package, path); - let _ = file_name; + match self.extract(package, &file_name, &temporary_dir).await { + Err(e) => { + install_cleanup(self.inner_mut(), package, path, &temporary_dir)?; + Err(e) + } + Ok(_) => { + if file_exists(&file_name) { + self.inner().filesystem.borrow().unlink(&file_name)?; + } + + let mut rename_as_one = false; + if !file_exists(path) { + rename_as_one = true; + } else if self.inner().filesystem.borrow().is_dir_empty(path) { + let removed = self + .inner() + .filesystem + .borrow_mut() + .remove_directory_php(path); + match removed { + Ok(true) => { + rename_as_one = true; + } + Ok(false) => {} + Err(e) => { + // ignore error, and simply do not renameAsOne + if e.downcast_ref::<RuntimeException>().is_none() { + return Err(e); + } + } + } + } + + let content_dir = get_folder_content(&temporary_dir); + let single_dir_at_top_level = content_dir.len() == 1 + && content_dir + .first() + .map(|file| is_dir(&file.get_pathname())) + .unwrap_or(false); + + if rename_as_one { + // if the target $path is clear, we can rename the whole package in one go instead of looping over the contents + let extracted_dir = if single_dir_at_top_level { + content_dir.first().unwrap().get_pathname() + } else { + temporary_dir.clone() + }; + self.inner() + .filesystem + .borrow_mut() + .rename(&extracted_dir, path)?; + } else { + // only one dir in the archive, extract its contents out of it + let mut from = temporary_dir.clone(); + if single_dir_at_top_level { + from = content_dir.first().unwrap().get_pathname(); + } - // TODO(phase-c-promise): rewrite extract().then(onFulfilled/onRejected) + renameRecursively chain as an await sequence - let promise = self.extract(package, "", &temporary_dir)?; + rename_recursively(&self.inner().filesystem, package, &from, path)?; + } - // TODO(phase-b): the original PHP chains React promise `.then(onFulfilled, onRejected)` - // callbacks that capture `$this`, `$filesystem`, `$package`, `$path`, `$temporaryDir`, - // `$fileName`, and a recursive `$renameRecursively` closure. PromiseInterface::then in - // Rust expects `FnOnce(Option<PhpMixed>) -> Option<PhpMixed>` and the callbacks here - // need both `&mut self` access and to return another promise. This needs a structural - // rework (likely splitting the trait or adding a `then_boxed_result` adapter), plus a - // way to share `&mut self` with the closure (probably `Rc<RefCell<...>>`). - let _ = (&promise, &temporary_dir, package, path); - todo!( - "ArchiveDownloader::install: rewire .then(onFulfilled, onRejected) chain to match PromiseInterface signature" - ) + self.inner() + .filesystem + .borrow_mut() + .remove_directory_async(&temporary_dir) + .await?; + self.inner_mut() + .remove_cleanup_path(package, &temporary_dir); + self.inner_mut().remove_cleanup_path(package, path); + + Ok(None) + } + } } /// @inheritDoc @@ -145,3 +204,87 @@ pub trait ArchiveDownloader { ": Extracting archive" } } + +fn install_cleanup( + inner: &mut FileDownloader, + package: &dyn PackageInterface, + path: &str, + temporary_dir: &str, +) -> Result<()> { + // remove cache if the file was corrupted + inner.clear_last_cache_write(package); + + // clean up + inner + .filesystem + .borrow_mut() + .remove_directory(temporary_dir)?; + if is_dir(path) && realpath(path) != Some(Platform::get_cwd(false).unwrap_or_default()) { + inner.filesystem.borrow_mut().remove_directory(path)?; + } + inner.remove_cleanup_path(package, temporary_dir); + let realpath = realpath(path); + if let Some(realpath) = realpath { + inner.remove_cleanup_path(package, &realpath); + } + + Ok(()) +} + +/// Returns the folder content, excluding .DS_Store +fn get_folder_content(dir: &str) -> Vec<SplFileInfo> { + let mut finder = Finder::create(); + finder + .ignore_vcs(false) + .ignore_dot_files(false) + .not_name(".DS_Store") + .depth(0) + .r#in(dir); + + finder.iter().collect() +} + +/// Renames (and recursively merges if needed) a folder into another one +/// +/// For custom installers, where packages may share paths, and given Composer 2's parallelism, we need to make sure +/// that the source directory gets merged into the target one if the target exists. Otherwise rename() by default would +/// put the source into the target e.g. src/ => target/src/ (assuming target exists) instead of src/ => target/ +fn rename_recursively( + filesystem: &std::rc::Rc<std::cell::RefCell<Filesystem>>, + package: &dyn PackageInterface, + from: &str, + to: &str, +) -> Result<()> { + let content_dir = get_folder_content(from); + + // move files back out of the temp dir + for file in &content_dir { + let file = file.get_pathname(); + if is_dir(&format!("{}/{}", to, basename(&file))) { + if !is_dir(&file) { + return Err(RuntimeException { + message: format!( + "Installing {} would lead to overwriting the {}/{} directory with a file from the package, invalid operation.", + package, + to, + basename(&file) + ), + code: 0, + } + .into()); + } + rename_recursively( + filesystem, + package, + &file, + &format!("{}/{}", to, basename(&file)), + )?; + } else { + filesystem + .borrow_mut() + .rename(&file, &format!("{}/{}", to, basename(&file)))?; + } + } + + Ok(()) +} diff --git a/crates/shirabe/src/downloader/downloader_interface.rs b/crates/shirabe/src/downloader/downloader_interface.rs index 7b333e6..18026a4 100644 --- a/crates/shirabe/src/downloader/downloader_interface.rs +++ b/crates/shirabe/src/downloader/downloader_interface.rs @@ -3,6 +3,7 @@ use crate::package::PackageInterface; use shirabe_php_shim::PhpMixed; +#[async_trait::async_trait(?Send)] pub trait DownloaderInterface: std::fmt::Debug { fn get_installation_source(&self) -> String; @@ -21,7 +22,7 @@ pub trait DownloaderInterface: std::fmt::Debug { path: &str, prev_package: Option<&dyn PackageInterface>, ) -> anyhow::Result<Option<PhpMixed>> { - self.download(package, path, prev_package, true) + self.download(package, path, prev_package, true).await } async fn prepare( @@ -45,7 +46,7 @@ pub trait DownloaderInterface: std::fmt::Debug { package: &dyn PackageInterface, path: &str, ) -> anyhow::Result<Option<PhpMixed>> { - self.install(package, path, true) + self.install(package, path, true).await } async fn update( @@ -68,7 +69,7 @@ pub trait DownloaderInterface: std::fmt::Debug { package: &dyn PackageInterface, path: &str, ) -> anyhow::Result<Option<PhpMixed>> { - self.remove(package, path, true) + self.remove(package, path, true).await } async fn cleanup( diff --git a/crates/shirabe/src/downloader/file_downloader.rs b/crates/shirabe/src/downloader/file_downloader.rs index 452ce8c..dc8c5a5 100644 --- a/crates/shirabe/src/downloader/file_downloader.rs +++ b/crates/shirabe/src/downloader/file_downloader.rs @@ -6,7 +6,6 @@ use indexmap::IndexMap; use std::sync::{LazyLock, Mutex}; use crate::util::Silencer; -use shirabe_external_packages::react::promise::resolve as react_promise_resolve; use shirabe_php_shim::{ DIRECTORY_SEPARATOR, InvalidArgumentException, PATHINFO_BASENAME, PATHINFO_EXTENSION, PHP_URL_PATH, PhpMixed, RuntimeException, UnexpectedValueException, array_search, array_shift, @@ -49,7 +48,7 @@ pub static DOWNLOAD_METADATA: LazyLock<Mutex<IndexMap<String, PhpMixed>>> = /// @var array<string, array<string>> /// @private /// @internal -pub static RESPONSE_HEADERS: LazyLock<Mutex<IndexMap<String, IndexMap<String, Vec<String>>>>> = +pub static RESPONSE_HEADERS: LazyLock<Mutex<IndexMap<String, Vec<String>>>> = LazyLock::new(|| Mutex::new(IndexMap::new())); /// Base downloader for files @@ -70,7 +69,12 @@ pub struct FileDownloader { /// @var ProcessExecutor pub(crate) process: std::rc::Rc<std::cell::RefCell<ProcessExecutor>>, /// @var array<string, string> Map of package name to cache key - last_cache_writes: IndexMap<String, String>, + /// + /// Behind a Mutex so `download()` can record cache writes through `&self`. `download()` blocks + /// for a while and (once the workers are parallelized) runs from several threads; requiring + /// `&mut self` would force locking the whole FileDownloader for each download's duration. Only + /// this write needs guarding, so it is the only field isolated behind a lock. + last_cache_writes: Mutex<IndexMap<String, String>>, /// @var array<string, string[]> Map of package name to list of paths additional_cleanup_paths: IndexMap<String, Vec<String>>, } @@ -115,7 +119,7 @@ impl FileDownloader { cache, process, filesystem, - last_cache_writes: IndexMap::new(), + last_cache_writes: Mutex::new(IndexMap::new()), additional_cleanup_paths: IndexMap::new(), }; @@ -140,6 +144,7 @@ impl FileDownloader { } } +#[async_trait::async_trait(?Send)] impl DownloaderInterface for FileDownloader { /// @inheritDoc fn get_installation_source(&self) -> String { @@ -199,16 +204,223 @@ impl DownloaderInterface for FileDownloader { .borrow_mut() .ensure_directory_exists(&dir_of_file)?; - // TODO(phase-c-promise): rewrite the accept/reject/retry promise orchestration as an async loop. - // TODO(plugin): inline closures rely on captured $accept/$reject/$urls/$retries. In Rust - // we'd need a struct holding shared state — left as a phase-b refactor. - let _ = (output, &urls, &mut retries, cache_key_generator, &file_name); - let _ = PluginEvents::PRE_FILE_DOWNLOAD; - let _ = PluginEvents::POST_FILE_DOWNLOAD; + // The PHP $download/$accept/$reject closures form a retry loop driven by recursion; here it + // is expressed as a loop. $reject's "return $download()" maps to `continue`, "throw" to + // `return Err`, and the success path runs the verification block and returns the file name. + let _ = cache_key_generator; + loop { + // === $download() === + let url = urls[0].clone(); + // TODO(plugin): dispatch PreFileDownloadEvent and apply its custom cache key / processed url. + urls[0] = url.clone(); - todo!( - "phase-b: orchestrate download/accept/reject closures and call download() returning a PromiseInterface" - ) + let checksum = package.get_dist_sha1_checksum().map(|s| s.to_string()); + let cache_key = url.cache_key.clone(); + + // use from cache if it is present and has a valid checksum or we have no checksum to check against + let mut from_cache = false; + if let Some(cache) = self.cache.clone() { + let checksum_matches = match checksum.as_deref() { + None | Some("") => true, + Some(c) => Some(c) == cache.borrow_mut().sha1(&cache_key).as_deref(), + }; + if checksum_matches && cache.borrow_mut().copy_to(&cache_key, &file_name)? { + from_cache = true; + } + } + + if from_cache { + if output { + self.io.write_error3( + &format!( + " - Loading <info>{}</info> (<comment>{}</comment>) from cache", + package.get_name(), + package.get_full_pretty_version(true, 0) + ), + true, + io_interface::VERY_VERBOSE, + ); + } + // mark the file as having been written in cache even though it is only read from cache, so that if + // the cache is corrupt the archive will be deleted and the next attempt will re-download it + // see https://github.com/composer/composer/issues/10028 + if let Some(cache) = self.cache.as_ref() { + if !cache.borrow().is_read_only() { + self.last_cache_writes + .lock() + .unwrap() + .insert(package.get_name().to_string(), cache_key.clone()); + } + } + } else { + if output { + self.io.write_error(&format!( + " - Downloading <info>{}</info> (<comment>{}</comment>)", + package.get_name(), + package.get_full_pretty_version(true, 0) + )); + } + + let add_copy_result = self + .http_downloader + .borrow_mut() + .add_copy(&url.processed, &file_name, package.get_transport_options()) + .await; + match add_copy_result { + Ok(mut response) => { + // === $accept($response) === + let cache_key = urls[0].cache_key.clone(); + let file_size = match filesize(&file_name) { + Some(size) => PhpMixed::Int(size), + None => PhpMixed::String( + response + .get_header("Content-Length") + .unwrap_or_else(|| "?".to_string()), + ), + }; + DOWNLOAD_METADATA + .lock() + .unwrap() + .insert(package.get_name().to_string(), file_size); + + if Platform::get_env("GITHUB_ACTIONS").is_some() + && Platform::get_env("COMPOSER_TESTS_ARE_RUNNING").is_none() + { + RESPONSE_HEADERS.lock().unwrap().insert( + package.get_name().to_string(), + response.get_headers().clone(), + ); + } + + if let Some(cache) = self.cache.as_ref() { + if !cache.borrow().is_read_only() { + self.last_cache_writes + .lock() + .unwrap() + .insert(package.get_name().to_string(), cache_key.clone()); + cache.borrow_mut().copy_from(&cache_key, &file_name); + } + } + + response.collect(); + } + Err(e) => { + // === $reject($e) === + // clean up + if file_exists(&file_name) { + self.filesystem.borrow().unlink(&file_name)?; + } + self.clear_last_cache_write(package); + + if e.downcast_ref::<IrrecoverableDownloadException>().is_some() { + return Err(e); + } + + if e.downcast_ref::<MaxFileSizeExceededException>().is_some() { + return Err(e); + } + + if let Some(te) = e.downcast_ref::<TransportException>() { + // if we got an http response with a proper code, then requesting again will probably not help, abort + if 0 != te.get_code() + && !in_array( + PhpMixed::Int(te.get_code()), + &PhpMixed::List(vec![ + Box::new(PhpMixed::Int(500)), + Box::new(PhpMixed::Int(502)), + Box::new(PhpMixed::Int(503)), + Box::new(PhpMixed::Int(504)), + ]), + true, + ) + { + retries = 0; + } + + // special error code returned when network is being artificially disabled + if te.get_status_code() == Some(499) { + retries = 0; + urls.clear(); + } + } + + if retries > 0 { + usleep(500000); + retries -= 1; + + continue; + } + + if !urls.is_empty() { + urls.remove(0); + } + if urls.len() > 0 { + let code = e + .downcast_ref::<TransportException>() + .map_or(0, |te| te.get_code()); + if self.io.is_debug() { + self.io.write_error(&format!( + " Failed downloading {}: [{}] {}: {}", + package.get_name(), + get_class(&PhpMixed::Null), + code, + e + )); + self.io.write_error(&format!( + " Trying the next URL for {}", + package.get_name() + )); + } else { + self.io.write_error(&format!( + " Failed downloading {}, trying the next URL ({}: {})", + package.get_name(), + code, + e + )); + } + + retries = 3; + usleep(100000); + + continue; + } + + return Err(e); + } + } + } + + // === $result->then(verify) === + if !file_exists(&file_name) { + return Err(UnexpectedValueException { + message: format!( + "{} could not be saved to {}, make sure the directory is writable and you have internet connectivity", + url.base, file_name + ), + code: 0, + } + .into()); + } + + if let Some(checksum) = checksum.as_deref() { + if !checksum.is_empty() + && hash_file("sha1", &file_name).as_deref() != Some(checksum) + { + return Err(UnexpectedValueException { + message: format!( + "The checksum verification of the file failed (downloaded from {})", + url.base + ), + code: 0, + } + .into()); + } + } + + // TODO(plugin): dispatch PostFileDownloadEvent. + + return Ok(Some(PhpMixed::String(file_name))); + } } /// @inheritDoc @@ -393,70 +605,52 @@ impl ChangeReportInterface for FileDownloader { package: &dyn PackageInterface, path: &str, ) -> Result<Option<String>> { - // TODO(phase-c-promise): get_local_changes drives promises via http_downloader/process wait(); - // converting requires deciding whether ChangeReportInterface::get_local_changes becomes async. Left as-is. // TODO(phase-b): swap self.io to NullIO and restore — needs a take/swap helper let mut null_io = NullIO::new(); null_io.load_configuration(&mut *self.config.borrow_mut())?; - // TODO(phase-b): `e` is captured by both the inner closure (assignment in error handler) - // and the outer block (read after the closure). PHP closures capture by reference (`use (&$e)`); - // emulate via Rc<RefCell> or restructure when proper async/promise types land. - let e: std::cell::RefCell<Option<anyhow::Error>> = std::cell::RefCell::new(None); - let output_cell: std::cell::RefCell<String> = std::cell::RefCell::new(String::new()); let target_dir = Filesystem::trim_trailing_slash(path); - let result: Result<()> = (|| -> Result<()> { + // PHP attaches an onRejected handler to capture the error and drives the promise via + // httpDownloader->wait() / process->wait(); the single-threaded sync bridge block_on's the + // download/install futures, so a rejection surfaces directly as the Err captured below. + let result: Result<String> = (|| -> Result<String> { if is_dir(&format!("{}_compare", target_dir)) { self.filesystem .borrow_mut() .remove_directory(&format!("{}_compare", target_dir))?; } - let promise = - self.download(package, &format!("{}_compare", target_dir), None, false)?; - promise.then_with( - None, - Some(Box::new(|ex: PhpMixed| { - let _ = ex; - PhpMixed::Null - })), - ); - self.http_downloader.borrow_mut().wait()?; - if e.borrow().is_some() { - return Err(e.borrow_mut().take().unwrap()); - } - let promise = self.install(package, &format!("{}_compare", target_dir), false)?; - promise.then_with( - None, - Some(Box::new(|ex: PhpMixed| { - let _ = ex; - PhpMixed::Null - })), - ); - self.process.borrow_mut().wait()?; - if e.borrow().is_some() { - return Err(e.borrow_mut().take().unwrap()); - } + tokio::runtime::Runtime::new() + .unwrap() + .block_on(self.download( + package, + &format!("{}_compare", target_dir), + None, + false, + ))?; + tokio::runtime::Runtime::new() + .unwrap() + .block_on(self.install(package, &format!("{}_compare", target_dir), false))?; let mut comparer = Comparer::new(); comparer.set_source(format!("{}_compare", target_dir)); comparer.set_update(target_dir.clone()); comparer.do_compare(); - *output_cell.borrow_mut() = comparer.get_changed_as_string(true, false); + let output = comparer.get_changed_as_string(true, false); self.filesystem .borrow_mut() .remove_directory(&format!("{}_compare", target_dir))?; - Ok(()) + Ok(output) })(); - if let Err(err) = result { - *e.borrow_mut() = Some(err); - } - let e = e.into_inner(); - let output = output_cell.into_inner(); // TODO(phase-b): restore self.io = prev_io + let (e, output) = match result { + Ok(output) => (None, output), + Err(err) => (Some(err), String::new()), + }; + if let Some(err) = e { if self.io.is_debug() { return Err(err); @@ -499,15 +693,12 @@ impl FileDownloader { .to_string() } - pub(crate) fn clear_last_cache_write(&mut self, package: &dyn PackageInterface) { - if self.cache.is_some() && self.last_cache_writes.contains_key(package.get_name()) { - let key = self - .last_cache_writes - .get(package.get_name()) - .unwrap() - .clone(); + pub(crate) fn clear_last_cache_write(&self, package: &dyn PackageInterface) { + let mut last_cache_writes = self.last_cache_writes.lock().unwrap(); + if self.cache.is_some() && last_cache_writes.contains_key(package.get_name()) { + let key = last_cache_writes.get(package.get_name()).unwrap().clone(); self.cache.as_ref().unwrap().borrow_mut().remove(&key); - self.last_cache_writes.shift_remove(package.get_name()); + last_cache_writes.shift_remove(package.get_name()); } } diff --git a/crates/shirabe/src/downloader/fossil_downloader.rs b/crates/shirabe/src/downloader/fossil_downloader.rs index 25f3a31..1e164b8 100644 --- a/crates/shirabe/src/downloader/fossil_downloader.rs +++ b/crates/shirabe/src/downloader/fossil_downloader.rs @@ -250,6 +250,7 @@ impl FossilDownloader { // TODO(phase-b): wire up VcsDownloader trait properly. FossilDownloader extends VcsDownloader // which implements DownloaderInterface in PHP. Delegating each trait method to todo!() until the // inner VcsDownloaderBase exposes the matching impl surface. +#[async_trait::async_trait(?Send)] impl DownloaderInterface for FossilDownloader { fn get_installation_source(&self) -> String { todo!() diff --git a/crates/shirabe/src/downloader/git_downloader.rs b/crates/shirabe/src/downloader/git_downloader.rs index e43d31b..a5e3638 100644 --- a/crates/shirabe/src/downloader/git_downloader.rs +++ b/crates/shirabe/src/downloader/git_downloader.rs @@ -4,7 +4,6 @@ use crate::io::io_interface; use anyhow::Result; use indexmap::IndexMap; use shirabe_external_packages::composer::pcre::{CaptureKey, Preg}; -use shirabe_external_packages::react::promise; use shirabe_php_shim::{ PhpMixed, RuntimeException, array_map, basename, dirname, implode, in_array, is_dir, preg_quote, realpath, rtrim, sprintf, strlen, strpos, substr, trim, version_compare, @@ -1359,6 +1358,7 @@ impl DvcsDownloaderInterface for GitDownloader { // TODO(phase-b): GitDownloader extends VcsDownloader which implements DownloaderInterface. // Delegating each trait method to todo!() until the inner VcsDownloaderBase exposes the // matching impl surface. +#[async_trait::async_trait(?Send)] impl crate::downloader::DownloaderInterface for GitDownloader { fn get_installation_source(&self) -> String { todo!() diff --git a/crates/shirabe/src/downloader/gzip_downloader.rs b/crates/shirabe/src/downloader/gzip_downloader.rs index d852a00..fe44fed 100644 --- a/crates/shirabe/src/downloader/gzip_downloader.rs +++ b/crates/shirabe/src/downloader/gzip_downloader.rs @@ -129,6 +129,7 @@ impl GzipDownloader { } } +#[async_trait::async_trait(?Send)] impl crate::downloader::DownloaderInterface for GzipDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() diff --git a/crates/shirabe/src/downloader/hg_downloader.rs b/crates/shirabe/src/downloader/hg_downloader.rs index 56f3b6f..b12e348 100644 --- a/crates/shirabe/src/downloader/hg_downloader.rs +++ b/crates/shirabe/src/downloader/hg_downloader.rs @@ -220,6 +220,7 @@ impl HgDownloader { // TODO(phase-b): wire up VcsDownloader trait properly. HgDownloader extends VcsDownloader which // implements DownloaderInterface in PHP. Delegating each trait method to todo!() until the inner // VcsDownloaderBase exposes the matching impl surface. +#[async_trait::async_trait(?Send)] impl DownloaderInterface for HgDownloader { fn get_installation_source(&self) -> String { todo!() diff --git a/crates/shirabe/src/downloader/path_downloader.rs b/crates/shirabe/src/downloader/path_downloader.rs index f677a50..46090ac 100644 --- a/crates/shirabe/src/downloader/path_downloader.rs +++ b/crates/shirabe/src/downloader/path_downloader.rs @@ -537,6 +537,7 @@ impl VcsCapableDownloaderInterface for PathDownloader { // overrides download/install/remove with &mut self signatures that diverge from the trait. The // trait methods here delegate to the inner FileDownloader; the bespoke overrides on the struct // itself are not yet routed through the trait. +#[async_trait::async_trait(?Send)] impl DownloaderInterface for PathDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() diff --git a/crates/shirabe/src/downloader/perforce_downloader.rs b/crates/shirabe/src/downloader/perforce_downloader.rs index fdb0f0d..b3f7bc9 100644 --- a/crates/shirabe/src/downloader/perforce_downloader.rs +++ b/crates/shirabe/src/downloader/perforce_downloader.rs @@ -159,6 +159,7 @@ impl PerforceDownloader { // TODO(phase-b): wire up VcsDownloader trait properly. PerforceDownloader extends VcsDownloader // which implements DownloaderInterface in PHP. Delegating each trait method to todo!() until the // inner VcsDownloaderBase exposes the matching impl surface. +#[async_trait::async_trait(?Send)] impl DownloaderInterface for PerforceDownloader { fn get_installation_source(&self) -> String { todo!() diff --git a/crates/shirabe/src/downloader/phar_downloader.rs b/crates/shirabe/src/downloader/phar_downloader.rs index 9cfb18f..a598437 100644 --- a/crates/shirabe/src/downloader/phar_downloader.rs +++ b/crates/shirabe/src/downloader/phar_downloader.rs @@ -63,6 +63,7 @@ impl PharDownloader { } } +#[async_trait::async_trait(?Send)] impl DownloaderInterface for PharDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() diff --git a/crates/shirabe/src/downloader/rar_downloader.rs b/crates/shirabe/src/downloader/rar_downloader.rs index c6031d9..f482582 100644 --- a/crates/shirabe/src/downloader/rar_downloader.rs +++ b/crates/shirabe/src/downloader/rar_downloader.rs @@ -143,6 +143,7 @@ impl RarDownloader { } } +#[async_trait::async_trait(?Send)] impl crate::downloader::DownloaderInterface for RarDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() diff --git a/crates/shirabe/src/downloader/svn_downloader.rs b/crates/shirabe/src/downloader/svn_downloader.rs index f3d0fef..162e91c 100644 --- a/crates/shirabe/src/downloader/svn_downloader.rs +++ b/crates/shirabe/src/downloader/svn_downloader.rs @@ -3,7 +3,6 @@ use crate::io::io_interface; use indexmap::IndexMap; use shirabe_external_packages::composer::pcre::{CaptureKey, Preg}; -use shirabe_external_packages::react::promise; use shirabe_php_shim::{PhpMixed, RuntimeException, is_dir, version_compare}; use crate::config::Config; @@ -432,6 +431,7 @@ impl SvnDownloader { // TODO(phase-b): wire up VcsDownloader trait properly. SvnDownloader extends VcsDownloader which // implements DownloaderInterface in PHP. Delegating each trait method to todo!() until the inner // VcsDownloaderBase exposes the matching impl surface. +#[async_trait::async_trait(?Send)] impl DownloaderInterface for SvnDownloader { fn get_installation_source(&self) -> String { todo!() diff --git a/crates/shirabe/src/downloader/tar_downloader.rs b/crates/shirabe/src/downloader/tar_downloader.rs index c720ccf..09b6f5a 100644 --- a/crates/shirabe/src/downloader/tar_downloader.rs +++ b/crates/shirabe/src/downloader/tar_downloader.rs @@ -58,6 +58,7 @@ impl TarDownloader { } } +#[async_trait::async_trait(?Send)] impl DownloaderInterface for TarDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() diff --git a/crates/shirabe/src/downloader/xz_downloader.rs b/crates/shirabe/src/downloader/xz_downloader.rs index d15287d..ca867df 100644 --- a/crates/shirabe/src/downloader/xz_downloader.rs +++ b/crates/shirabe/src/downloader/xz_downloader.rs @@ -77,6 +77,7 @@ impl XzDownloader { } } +#[async_trait::async_trait(?Send)] impl crate::downloader::DownloaderInterface for XzDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() diff --git a/crates/shirabe/src/downloader/zip_downloader.rs b/crates/shirabe/src/downloader/zip_downloader.rs index 88b3d3a..eb057e0 100644 --- a/crates/shirabe/src/downloader/zip_downloader.rs +++ b/crates/shirabe/src/downloader/zip_downloader.rs @@ -14,7 +14,8 @@ use shirabe_external_packages::symfony::component::process::Process; use shirabe_php_shim::{ DIRECTORY_SEPARATOR, ErrorException, PhpMixed, RuntimeException, UnexpectedValueException, ZipArchive, bin2hex, class_exists, file_exists, file_get_contents, filesize, function_exists, - hash_file, is_file, json_encode, random_int, version_compare, + hash_file, is_file, json_encode, random_int, str_contains, str_replace, strlen, substr, + version_compare, }; use std::sync::Mutex; @@ -270,27 +271,153 @@ impl ZipDownloader { } } - // TODO(phase-c-promise): execute_async + .then fallback closure captures &mut self/io; - // recursive promise flattening, not a mechanical await chain. - // TODO(phase-b): full try_fallback closure deferred — PHP captures `$io`, `$self` - // and several locals by reference, conflicting with Rust's borrow checker because - // `extract_with_zip_archive` later needs `&mut self`. Restructure once the - // promise/closure plumbing supports that shape. - let _ = ( - is_last_chance, - file, - path, - executable, - package, - &command, - &self.inner.io, - ); - match self.inner.process.borrow_mut().execute_async(&command, ()) { - Ok(_promise) => todo!("phase-b: chain promise.then with fallback closure"), - Err(_e) => todo!("phase-b: pipe execute_async error into try_fallback"), + let process_result = self + .inner + .process + .borrow_mut() + .execute_async(&command, ()) + .await; + match process_result { + Ok(process) => { + if !process.is_successful() { + if self.cleanup_executed.contains_key(package.get_name()) { + return Err(RuntimeException { + message: format!( + "Failed to extract {} as the installation was aborted by another package operation.", + package.get_name() + ), + code: 0, + } + .into()); + } + + let output = process.get_error_output(); + let output = + str_replace(&format!(", {}.zip or {}.ZIP", file, file), "", &output); + + return self + .try_fallback( + RuntimeException { + message: format!( + "Failed to extract {}: ({}) {}\n\n{}", + package.get_name(), + process + .get_exit_code() + .map(|c| c.to_string()) + .unwrap_or_default(), + command.join(" "), + output + ), + code: 0, + } + .into(), + is_last_chance, + file, + path, + package, + &executable, + ) + .await; + } + + Ok(None) + } + Err(e) => { + self.try_fallback(e, is_last_chance, file, path, package, &executable) + .await + } } } + async fn try_fallback( + &mut self, + process_error: anyhow::Error, + is_last_chance: bool, + file: &str, + path: &str, + package: &dyn PackageInterface, + executable: &str, + ) -> Result<Option<PhpMixed>> { + if is_last_chance { + return Err(process_error); + } + + if str_contains(&process_error.to_string(), "zip bomb") { + return Err(process_error); + } + + if !is_file(file) { + self.inner + .io + .write_error(&format!(" <warning>{}</warning>", process_error)); + self.inner.io.write_error(" <warning>This most likely is due to a custom installer plugin not handling the returned Promise from the downloader</warning>"); + self.inner.io.write_error(" <warning>See https://github.com/composer/installers/commit/5006d0c28730ade233a8f42ec31ac68fb1c5c9bb for an example fix</warning>"); + } else { + self.inner + .io + .write_error(&format!(" <warning>{}</warning>", process_error)); + self.inner.io.write_error(" The archive may contain identical file names with different capitalization (which fails on case insensitive filesystems)"); + self.inner.io.write_error(&format!( + " Unzip with {} command failed, falling back to ZipArchive class", + executable + )); + + // additional debug data to try to figure out GH actions issues https://github.com/composer/composer/issues/11148 + if Platform::get_env("GITHUB_ACTIONS").is_some() + && Platform::get_env("COMPOSER_TESTS_ARE_RUNNING").is_none() + { + self.inner.io.write_error(" <warning>Additional debug info, please report to https://github.com/composer/composer/issues/11148 if you see this:</warning>"); + self.inner.io.write_error(&format!( + "File size: {}", + filesize(file).map(|s| s.to_string()).unwrap_or_default() + )); + self.inner.io.write_error(&format!( + "File SHA1: {}", + hash_file("sha1", file).unwrap_or_default() + )); + self.inner.io.write_error(&format!( + "First 100 bytes (hex): {}", + bin2hex( + substr(&file_get_contents(file).unwrap_or_default(), 0, Some(100)) + .as_bytes() + ) + )); + self.inner.io.write_error(&format!( + "Last 100 bytes (hex): {}", + bin2hex( + substr(&file_get_contents(file).unwrap_or_default(), -100, None).as_bytes() + ) + )); + if strlen(package.get_dist_url().unwrap_or("")) > 0 { + self.inner.io.write_error(&format!( + "Origin URL: {}", + self.inner + .process_url(package, package.get_dist_url().unwrap_or(""))? + )); + let headers = { + let response_headers = crate::downloader::file_downloader::RESPONSE_HEADERS + .lock() + .unwrap(); + match response_headers.get(package.get_name()) { + Some(list) => PhpMixed::List( + list.iter() + .map(|s| Box::new(PhpMixed::String(s.clone()))) + .collect(), + ), + None => PhpMixed::List(vec![]), + } + }; + self.inner.io.write_error(&format!( + "Response Headers: {}", + json_encode(&headers).unwrap_or_default() + )); + } + } + } + + self.extract_with_zip_archive(package, file, path).await + } + async fn extract_with_zip_archive( &mut self, package: &dyn PackageInterface, @@ -425,6 +552,7 @@ impl ZipDownloader { // TODO(phase-b): ZipDownloader::download is overridden with extra setup (UNZIP_COMMANDS init, // etc.). The trait method here delegates straight to the inner FileDownloader; the bespoke // override on the struct itself takes &mut self and is not yet routed through the trait. +#[async_trait::async_trait(?Send)] impl crate::downloader::DownloaderInterface for ZipDownloader { fn get_installation_source(&self) -> String { self.inner.get_installation_source() |