//! ref: composer/src/Composer/Repository/ComposerRepository.php use indexmap::IndexMap; use shirabe_external_packages::composer::metadata_minifier::MetadataMinifier; use shirabe_external_packages::composer::pcre::{CaptureKey, Preg}; use shirabe_php_shim::{ Countable, InvalidArgumentException, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE, LogicException, PHP_EOL, PhpMixed, RuntimeException, UnexpectedValueException, extension_loaded, hash, http_build_query, in_array, json_decode, parse_url_all, realpath, strtolower, strtr, urlencode, var_export, }; use shirabe_semver::compiling_matcher::CompilingMatcher; use shirabe_semver::constraint::AnyConstraint; use shirabe_semver::constraint::MatchAllConstraint; use shirabe_semver::constraint::SimpleConstraint; use crate::advisory::PartialSecurityAdvisory; use crate::cache::Cache; use crate::config::Config; use crate::downloader::TransportException; use crate::event_dispatcher::EventDispatcher; use crate::io::IOInterface; use crate::io::IOInterfaceImmutable; use crate::json::JsonFile; use crate::package::BasePackageHandle; use crate::package::PackageInterface; use crate::package::PackageInterfaceHandle; use crate::package::base_package::{self, BasePackage}; use crate::package::loader::ArrayLoader; use crate::package::version::StabilityFilter; use crate::package::version::VersionParser; use crate::plugin::PluginEvents; use crate::plugin::PostFileDownloadEvent; use crate::plugin::PreFileDownloadEvent; use crate::repository::ArrayRepository; use crate::repository::ConfigurableRepositoryInterface; use crate::repository::LoadPackagesResult; use crate::repository::PlatformRepository; use crate::repository::RepositoryInterface; use crate::repository::RepositoryInterfaceHandle; use crate::repository::RepositoryInterfaceWeakHandle; use crate::repository::RepositorySecurityException; use crate::repository::{PartialOrSecurityAdvisory, SecurityAdvisoryResult}; use crate::repository::{SEARCH_FULLTEXT, SEARCH_VENDOR}; use crate::util::HttpDownloader; use crate::util::Url; use crate::util::http::Response; use crate::util::r#loop::Loop; #[derive(Debug)] pub enum RootData { Data(IndexMap), True, } #[derive(Debug)] pub struct SecurityAdvisoryConfig { pub metadata: bool, pub api_url: Option, } #[derive(Debug)] pub struct SourceMirror { pub url: String, pub preferred: bool, } #[derive(Debug)] pub struct DistMirror { pub url: String, pub preferred: bool, } #[derive(Debug)] pub struct ProviderListingEntry { pub sha256: String, } #[derive(Debug)] pub struct ComposerRepository { inner: ArrayRepository, /// Weak reference to the outermost repository handle wrapping this `ComposerRepository`, /// injected via `set_self_handle`. Used to wire package -> repository back-references. self_weak: std::cell::RefCell>, /// @phpstan-var array{url: string, options?: mixed[], type?: 'composer', allow_ssl_downgrade?: bool} repo_config: IndexMap, options: IndexMap, /// non-empty-string url: String, /// non-empty-string base_url: String, io: std::rc::Rc>, http_downloader: std::rc::Rc>, r#loop: std::rc::Rc>, pub(crate) cache: Cache, pub(crate) notify_url: Option, pub(crate) search_url: Option, pub(crate) providers_api_url: Option, pub(crate) has_providers: bool, pub(crate) providers_url: Option, pub(crate) list_url: Option, pub(crate) has_available_package_list: bool, pub(crate) available_packages: Option>, pub(crate) available_package_patterns: Option>, pub(crate) lazy_providers_url: Option, pub(crate) provider_listing: Option>, pub(crate) loader: ArrayLoader, allow_ssl_downgrade: bool, event_dispatcher: Option>>, source_mirrors: Option>>, dist_mirrors: Option>, degraded_mode: bool, root_data: Option, has_partial_packages: bool, partial_packages_by_name: Option>>>, displayed_warning_about_non_matching_package_index: bool, security_advisory_config: Option, /// list of package names which are fresh and can be loaded from the cache directly in case loadPackage is called several times /// useful for v2 metadata repositories with lazy providers freshMetadataUrls: IndexMap, /// list of package names which returned a 404 and should not be re-fetched in case loadPackage is called several times /// useful for v2 metadata repositories with lazy providers packagesNotFoundCache: IndexMap, version_parser: VersionParser, } #[derive(Debug)] pub enum FindPackageReturn { Package(BasePackageHandle), Packages(Vec), None, } #[derive(Debug)] pub struct LoadAsyncPackagesResult { pub names_found: IndexMap, pub packages: IndexMap, } impl ConfigurableRepositoryInterface for ComposerRepository { fn get_repo_config(&self) -> IndexMap { self.repo_config.clone() } } impl ComposerRepository { pub fn new( mut repo_config: IndexMap, io: std::rc::Rc>, config: &Config, http_downloader: std::rc::Rc>, event_dispatcher: Option>>, ) -> anyhow::Result { // parent::__construct(); let inner = ArrayRepository::new(Vec::new()); let url_str = repo_config .get("url") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); if !Preg::is_match(r"{^[\w.]+\??://}", &url_str)? { if let Some(local_file_path) = realpath(&url_str) { // it is a local path, add file scheme repo_config.insert( "url".to_string(), PhpMixed::String(format!("file://{}", local_file_path)), ); } else { // otherwise, assume http as the default protocol repo_config.insert( "url".to_string(), PhpMixed::String(format!("http://{}", url_str)), ); } } let url_after = repo_config .get("url") .and_then(|v| v.as_string()) .unwrap_or("") .trim_end_matches('/') .to_string(); repo_config.insert("url".to_string(), PhpMixed::String(url_after.clone())); if url_after.is_empty() { return Err(InvalidArgumentException { message: "The repository url must not be an empty string".to_string(), code: 0, } .into()); } if url_after.starts_with("https?") { let scheme = if extension_loaded("openssl") { "https" } else { "http" }; let rest = &url_after[6..]; repo_config.insert( "url".to_string(), PhpMixed::String(format!("{}{}", scheme, rest)), ); } let current_url = repo_config .get("url") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let url_bits = parse_url_all(&strtr(¤t_url, "\\", "/")); let url_bits_arr = url_bits.as_array(); let scheme_present = url_bits_arr .and_then(|a| a.get("scheme")) .and_then(|v| v.as_string()) .map_or(false, |s| !s.is_empty()); if url_bits_arr.is_none() || !scheme_present { return Err(UnexpectedValueException { message: format!("Invalid url given for Composer repository: {}", current_url), code: 0, } .into()); } if !repo_config.contains_key("options") { repo_config.insert("options".to_string(), PhpMixed::Array(IndexMap::new())); } let mut allow_ssl_downgrade = false; if let Some(v) = repo_config.get("allow_ssl_downgrade") { if v.as_bool() == Some(true) { allow_ssl_downgrade = true; } } let options = repo_config .get("options") .and_then(|v| v.as_array()) .cloned() .unwrap_or_default() .into_iter() .map(|(k, v)| (k, *v)) .collect::>(); let mut url = repo_config .get("url") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); // force url for packagist.org to repo.packagist.org let mut match_packagist: IndexMap = IndexMap::new(); if Preg::is_match3( r"{^(?Phttps?)://packagist\.org/?$}i", &url, Some(&mut match_packagist), )? { let proto = match_packagist .get(&CaptureKey::ByName("proto".to_string())) .cloned() .unwrap_or_default(); url = format!("{}://repo.packagist.org", proto); } let base_url_trimmed = Preg::replace(r"{(?:/[^/\\]+\.json)?(?:[?#].*)?$}", "", &url)?; let base_url = base_url_trimmed.trim_end_matches('/').to_string(); assert!(!base_url.is_empty()); let cache_dir = format!( "{}/{}", config.get("cache-repo-dir").as_string().unwrap_or(""), Preg::replace(r"{[^a-z0-9.]}i", "-", &Url::sanitize(url.clone()))?, ); let cache = Cache::new(io.clone(), &cache_dir, Some("a-z0-9.$~"), None, false); let version_parser = VersionParser::new(); let loader = ArrayLoader::new(Some(version_parser.clone()), true); let r#loop = std::rc::Rc::new(std::cell::RefCell::new(Loop::new( http_downloader.clone(), None, ))); let mut this = Self { inner: inner?, self_weak: std::cell::RefCell::new(None), repo_config, options, url, base_url, io, http_downloader, r#loop, cache, notify_url: None, search_url: None, providers_api_url: None, has_providers: false, providers_url: None, list_url: None, has_available_package_list: false, available_packages: None, available_package_patterns: None, lazy_providers_url: None, provider_listing: None, loader, allow_ssl_downgrade, event_dispatcher, source_mirrors: None, dist_mirrors: None, degraded_mode: false, root_data: None, has_partial_packages: false, partial_packages_by_name: None, displayed_warning_about_non_matching_package_index: false, security_advisory_config: None, freshMetadataUrls: IndexMap::new(), packagesNotFoundCache: IndexMap::new(), version_parser, }; this.cache .set_read_only(config.get("cache-read-only").as_bool().unwrap_or(false)); Ok(this) } pub fn get_repo_name(&self) -> String { format!("composer repo ({})", Url::sanitize(self.url.clone())) } pub fn get_repo_config_pub(&self) -> IndexMap { self.repo_config.clone() } /// @inheritDoc pub fn find_package( &mut self, name: String, constraint: PhpMixed, ) -> anyhow::Result> { // this call initializes loadRootServerFile which is needed for the rest below to work let has_providers = self.has_providers()?; let name = strtolower(&name); let constraint: AnyConstraint = match constraint { PhpMixed::String(s) => self.version_parser.parse_constraints(&s)?.clone(), _ => { // already a ConstraintInterface object passed as opaque PhpMixed self.version_parser.parse_constraints("")?.clone() } }; if self.lazy_providers_url.is_some() { if self.has_partial_packages()? && self .partial_packages_by_name .as_ref() .map_or(false, |m| m.contains_key(&name)) { let packages = self.what_provides(&name, None, None, IndexMap::new())?; let packages_vec: Vec = packages.into_values().collect(); return Ok( match self.filter_packages(packages_vec, Some(&constraint), true) { FindPackageReturn::Package(p) => Some(p), _ => None, }, ); } if self.has_available_package_list && !self.lazy_providers_repo_contains(&name)? { return Ok(None); } let mut map: IndexMap> = IndexMap::new(); map.insert(name.clone(), Some(constraint)); let packages = self.load_async_packages(map, None, None, IndexMap::new())?; if !packages.packages.is_empty() { return Ok(packages.packages.into_iter().next().map(|(_, v)| v)); } return Ok(None); } if has_providers { for provider_name in self.get_provider_names()? { if name == provider_name { let packages = self.what_provides(&provider_name, None, None, IndexMap::new())?; let packages_vec: Vec = packages.into_values().collect(); return Ok( match self.filter_packages(packages_vec, Some(&constraint), true) { FindPackageReturn::Package(p) => Some(p), _ => None, }, ); } } return Ok(None); } Ok(self.inner.find_package( &name, crate::repository::FindPackageConstraint::Constraint(constraint), )) } /// @inheritDoc pub fn find_packages( &mut self, name: String, constraint: Option, ) -> anyhow::Result> { // this call initializes loadRootServerFile which is needed for the rest below to work let has_providers = self.has_providers()?; let name = strtolower(&name); let constraint: Option = match constraint { None => None, Some(PhpMixed::String(s)) => Some(self.version_parser.parse_constraints(&s)?.clone()), Some(_) => None, }; if self.lazy_providers_url.is_some() { if self.has_partial_packages()? && self .partial_packages_by_name .as_ref() .map_or(false, |m| m.contains_key(&name)) { let packages = self.what_provides(&name, None, None, IndexMap::new())?; let packages_vec: Vec = packages.into_values().collect(); return Ok( match self.filter_packages(packages_vec, constraint.as_ref(), false) { FindPackageReturn::Packages(v) => v, _ => vec![], }, ); } if self.has_available_package_list && !self.lazy_providers_repo_contains(&name)? { return Ok(vec![]); } let mut map: IndexMap> = IndexMap::new(); map.insert(name.clone(), constraint); let result = self.load_async_packages(map, None, None, IndexMap::new())?; return Ok(result.packages.into_values().collect()); } if has_providers { for provider_name in self.get_provider_names()? { if name == provider_name { let packages = self.what_provides(&provider_name, None, None, IndexMap::new())?; let packages_vec: Vec = packages.into_values().collect(); return Ok( match self.filter_packages(packages_vec, constraint.as_ref(), false) { FindPackageReturn::Packages(v) => v, _ => vec![], }, ); } } return Ok(vec![]); } Ok(self.inner.find_packages( &name, constraint.map(crate::repository::FindPackageConstraint::Constraint), )) } fn filter_packages( &self, packages: Vec, constraint: Option<&AnyConstraint>, return_first_match: bool, ) -> FindPackageReturn { if constraint.is_none() { if return_first_match { return match packages.into_iter().next() { Some(p) => FindPackageReturn::Package(p), None => FindPackageReturn::None, }; } return FindPackageReturn::Packages(packages); } let constraint = constraint.unwrap(); let mut filtered_packages: Vec = Vec::new(); for package in packages.into_iter() { let pkg_constraint = SimpleConstraint::new("==".to_string(), package.get_version().to_string(), None); if constraint.matches(&pkg_constraint.into()) { if return_first_match { return FindPackageReturn::Package(package); } filtered_packages.push(package); } } if return_first_match { return FindPackageReturn::None; } FindPackageReturn::Packages(filtered_packages) } pub fn get_packages(&mut self) -> anyhow::Result> { let has_providers = self.has_providers()?; if self.lazy_providers_url.is_some() { if let Some(ref available_packages) = self.available_packages.clone() { if self.available_package_patterns.is_none() { let mut package_map: IndexMap> = IndexMap::new(); for name in available_packages.values() { package_map .insert(name.clone(), Some(MatchAllConstraint::new(None).into())); } let result = self.load_async_packages(package_map, None, None, IndexMap::new())?; return Ok(result.packages.into_values().collect()); } } if self.has_partial_packages()? { if self.partial_packages_by_name.is_none() { return Err(LogicException { message: "hasPartialPackages failed to initialize $this->partialPackagesByName" .to_string(), code: 0, } .into()); } let partial = self.partial_packages_by_name.clone().unwrap(); let flat: Vec> = partial.into_values().flatten().collect(); return self .create_packages_flat(flat, Some("packages.json inline packages".to_string())); } return Err(LogicException { message: "Composer repositories that have lazy providers and no available-packages list can not load the complete list of packages, use getPackageNames instead.".to_string(), code: 0, }.into()); } if has_providers { return Err(LogicException { message: "Composer repositories that have providers can not load the complete list of packages, use getPackageNames instead.".to_string(), code: 0, }.into()); } Ok(self.inner.get_packages()) } /// @param packageFilter Package pattern filter which can include "*" as a wildcard pub fn get_package_names( &mut self, package_filter: Option<&str>, ) -> anyhow::Result> { let has_providers = self.has_providers()?; let package_filter_regex: Option = match package_filter { Some(p) if !p.is_empty() => Some(base_package::package_name_to_regexp(p)), _ => None, }; let filter_results = |results: Vec| -> anyhow::Result> { match &package_filter_regex { Some(regex) => { let results_refs: Vec<&str> = results.iter().map(|s| s.as_str()).collect(); Ok(Preg::grep(regex, &results_refs)?) } None => Ok(results), } }; if self.lazy_providers_url.is_some() { if let Some(ref available_packages) = self.available_packages { let keys: Vec = available_packages.keys().cloned().collect(); return filter_results(keys); } if self.list_url.is_some() { // no need to call $filterResults here as the $packageFilter is applied in the function itself return self.load_package_list(package_filter); } if self.has_partial_packages()? && self.partial_packages_by_name.is_some() { let keys: Vec = self .partial_packages_by_name .as_ref() .unwrap() .keys() .cloned() .collect(); return filter_results(keys); } return Ok(vec![]); } if has_providers { return filter_results(self.get_provider_names()?); } let mut names: Vec = Vec::new(); for package in self.get_packages()? { names.push(package.get_pretty_name().to_string()); } filter_results(names) } fn get_vendor_names(&mut self) -> anyhow::Result> { let cache_key = "vendor-list.txt"; let cache_age = self.cache.get_age(cache_key); if let Some(age) = cache_age { if age < 600 { if let Some(cached_data) = self.cache.read(cache_key) { let cached_data: Vec = cached_data.split('\n').map(|s| s.to_string()).collect(); return Ok(cached_data); } } } let names = self.get_package_names(None)?; let mut uniques: IndexMap = IndexMap::new(); for name in &names { let vendor = name.splitn(2, '/').next().unwrap_or("").to_string(); uniques.insert(vendor, true); } let vendors: Vec = uniques.keys().cloned().collect(); if !self.cache.is_read_only() { self.cache.write(cache_key, &vendors.join("\n")); } Ok(vendors) } fn load_package_list(&mut self, package_filter: Option<&str>) -> anyhow::Result> { if self.list_url.is_none() { return Err(LogicException { message: "Make sure to call loadRootServerFile before loadPackageList".to_string(), code: 0, } .into()); } let mut url = self.list_url.clone().unwrap(); if let Some(filter) = package_filter { if !filter.is_empty() { url.push_str(&format!("?filter={}", urlencode(filter))); let result = self .http_downloader .borrow_mut() .get(&url, self.options.clone())? .decode_json()?; let package_names: Vec = result .as_array() .and_then(|a| a.get("packageNames")) .and_then(|v| v.as_list()) .map(|l| { l.iter() .filter_map(|v| v.as_string().map(|s| s.to_string())) .collect() }) .unwrap_or_default(); return Ok(package_names); } } let cache_key = "package-list.txt"; let cache_age = self.cache.get_age(cache_key); if let Some(age) = cache_age { if age < 600 { if let Some(cached_data) = self.cache.read(cache_key) { let cached_data: Vec = cached_data.split('\n').map(|s| s.to_string()).collect(); return Ok(cached_data); } } } let result = self .http_downloader .borrow_mut() .get(&url, self.options.clone())? .decode_json()?; let package_names: Vec = result .as_array() .and_then(|a| a.get("packageNames")) .and_then(|v| v.as_list()) .map(|l| { l.iter() .filter_map(|v| v.as_string().map(|s| s.to_string())) .collect() }) .unwrap_or_default(); if !self.cache.is_read_only() { self.cache.write(cache_key, &package_names.join("\n")); } Ok(package_names) } pub fn load_packages( &mut self, mut package_name_map: IndexMap>, acceptable_stabilities: IndexMap, stability_flags: IndexMap, already_loaded: IndexMap>, ) -> anyhow::Result { // this call initializes loadRootServerFile which is needed for the rest below to work let has_providers = self.has_providers()?; if !has_providers && !self.has_partial_packages()? && self.lazy_providers_url.is_none() { let inner_result = self.inner.load_packages( package_name_map, acceptable_stabilities, stability_flags, already_loaded, ); return Ok(inner_result); } let mut packages: IndexMap = IndexMap::new(); let mut names_found: IndexMap = IndexMap::new(); if has_providers || self.has_partial_packages()? { let names: Vec = package_name_map.keys().cloned().collect(); for name in names { let mut matches: IndexMap = IndexMap::new(); // if a repo has no providers but only partial packages and the partial packages are missing // then we don't want to call whatProvides as it would try to load from the providers and fail if !has_providers && !self .partial_packages_by_name .as_ref() .map_or(false, |m| m.contains_key(&name)) { continue; } let candidates = self.what_provides( &name, Some(&acceptable_stabilities), Some(&stability_flags), already_loaded.clone(), )?; let constraint = package_name_map .get(&name) .and_then(|c| c.as_ref().map(|c| c.clone())); for (_uid, candidate) in candidates.iter() { if candidate.get_name() != name { return Err(LogicException { message: "whatProvides should never return a package with a different name than the requested one".to_string(), code: 0, }.into()); } names_found.insert(name.clone(), true); let matches_constraint = match &constraint { None => true, Some(c) => { let pkg_c = SimpleConstraint::new( "==".to_string(), candidate.get_version().to_string(), None, ); c.matches(&pkg_c.into()) } }; if matches_constraint { let hash_c = candidate.ptr_id().to_string(); matches.insert(hash_c, candidate.clone()); if let Some(alias) = candidate.as_alias() { let aliased = alias.get_alias_of(); let aliased_hash = aliased.ptr_id().to_string(); if !matches.contains_key(&aliased_hash) { matches.insert(aliased_hash, aliased.into()); } } } } // add aliases of matched packages even if they did not match the constraint for (_uid, candidate) in candidates.iter() { if let Some(alias) = candidate.as_alias() { let aliased = alias.get_alias_of(); let aliased_hash = aliased.ptr_id().to_string(); if matches.contains_key(&aliased_hash) { let hash_c = candidate.ptr_id().to_string(); matches.insert(hash_c, candidate.clone()); } } } for (k, v) in matches.into_iter() { packages.insert(k, v); } package_name_map.shift_remove(&name); } } if self.lazy_providers_url.is_some() && !package_name_map.is_empty() { if self.has_available_package_list { let names: Vec = package_name_map.keys().cloned().collect(); for name in names { if !self.lazy_providers_repo_contains(&strtolower(&name))? { package_name_map.shift_remove(&name); } } } let result = self.load_async_packages( package_name_map, Some(&acceptable_stabilities), Some(&stability_flags), already_loaded, )?; for (k, v) in result.packages.into_iter() { packages.insert(k, v); } for (k, v) in result.names_found.into_iter() { names_found.insert(k, v); } } Ok(LoadPackagesResult { names_found: names_found.keys().cloned().collect(), packages, }) } /// @inheritDoc pub fn search( &mut self, query: String, mode: i64, r#type: Option, ) -> anyhow::Result>> { self.load_root_server_file(Some(600))?; if let Some(search_url) = self.search_url.clone() { if mode == SEARCH_FULLTEXT { let url = search_url .replace("%query%", &urlencode(&query)) .replace("%type%", r#type.as_deref().unwrap_or("")); let search = self .http_downloader .borrow_mut() .get(&url, self.options.clone())? .decode_json()?; let results_arr = search .as_array() .and_then(|a| a.get("results")) .and_then(|v| v.as_list()) .cloned() .unwrap_or_default(); if results_arr.is_empty() { return Ok(vec![]); } let mut results: Vec> = Vec::new(); for result in results_arr.iter() { let arr = match result.as_array() { Some(a) => a, None => continue, }; // do not show virtual packages in results as they are not directly useful from a composer perspective if let Some(v) = arr.get("virtual") { // PHP's `empty()` is false when the value is truthy let is_empty = match &**v { PhpMixed::Null => true, PhpMixed::Bool(false) => true, PhpMixed::Int(0) => true, PhpMixed::Float(f) if *f == 0.0 => true, PhpMixed::String(s) if s.is_empty() || s == "0" => true, PhpMixed::List(l) if l.is_empty() => true, PhpMixed::Array(a) if a.is_empty() => true, _ => false, }; if !is_empty { continue; } } results.push( arr.iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(), ); } return Ok(results); } } if mode == SEARCH_VENDOR { let mut results: Vec> = Vec::new(); let parts = Preg::split(r"{\s+}", &query)?; let regex = format!("{{(?:{})}}i", parts.join("|")); let vendor_names = self.get_vendor_names()?; let vendor_names_refs: Vec<&str> = vendor_names.iter().map(|s| s.as_str()).collect(); for name in Preg::grep(®ex, &vendor_names_refs)? { let mut entry = IndexMap::new(); entry.insert("name".to_string(), PhpMixed::String(name)); entry.insert("description".to_string(), PhpMixed::String(String::new())); results.push(entry); } return Ok(results); } if self.has_providers()? || self.lazy_providers_url.is_some() { // optimize search for "^foo/bar" where at least "^foo/" is present by loading this directly from the listUrl if present let mut match_groups: IndexMap = IndexMap::new(); if Preg::is_match_strict_groups3( r"{^\^(?P(?P[a-z0-9_.-]+)/[a-z0-9_.-]*)\*?$}i", &query, Some(&mut match_groups), )? && self.list_url.is_some() { let q = match_groups .get(&CaptureKey::ByName("query".to_string())) .cloned() .unwrap_or_default(); let vendor = match_groups .get(&CaptureKey::ByName("vendor".to_string())) .cloned() .unwrap_or_default(); let url = format!( "{}?vendor={}&filter={}", self.list_url.as_ref().unwrap(), urlencode(&vendor), urlencode(&format!("{}*", q)), ); let result = self .http_downloader .borrow_mut() .get(&url, self.options.clone())? .decode_json()?; let mut results: Vec> = Vec::new(); if let Some(list) = result .as_array() .and_then(|a| a.get("packageNames")) .and_then(|v| v.as_list()) { for name_mixed in list.iter() { if let Some(name) = name_mixed.as_string() { let mut entry = IndexMap::new(); entry.insert("name".to_string(), PhpMixed::String(name.to_string())); entry .insert("description".to_string(), PhpMixed::String(String::new())); results.push(entry); } } } return Ok(results); } let mut results: Vec> = Vec::new(); let parts = Preg::split(r"{\s+}", &query)?; let regex = format!("{{(?:{})}}i", parts.join("|")); let package_names = self.get_package_names(None)?; let package_names_refs: Vec<&str> = package_names.iter().map(|s| s.as_str()).collect(); for name in Preg::grep(®ex, &package_names_refs)? { let mut entry = IndexMap::new(); entry.insert("name".to_string(), PhpMixed::String(name)); entry.insert("description".to_string(), PhpMixed::String(String::new())); results.push(entry); } return Ok(results); } // TODO(phase-b): inner.search returns Vec; convert to PHP-shaped map let inner_results = self.inner.search(query, mode, None); let converted: Vec> = inner_results .into_iter() .map(|sr| { let mut m: IndexMap = IndexMap::new(); m.insert("name".to_string(), PhpMixed::String(sr.name)); if let Some(d) = sr.description { m.insert("description".to_string(), PhpMixed::String(d)); } if let Some(u) = sr.url { m.insert("url".to_string(), PhpMixed::String(u)); } m }) .collect(); Ok(converted) } pub fn has_security_advisories(&mut self) -> anyhow::Result { self.load_root_server_file(Some(600))?; Ok(self .security_advisory_config .as_ref() .map_or(false, |c| c.metadata || c.api_url.is_some())) } /// @inheritDoc pub fn get_security_advisories( &mut self, mut package_constraint_map: IndexMap, allow_partial_advisories: bool, ) -> anyhow::Result { self.load_root_server_file(Some(600))?; if self.security_advisory_config.is_none() { return Ok(SecurityAdvisoryResult { names_found: vec![], advisories: IndexMap::new(), }); } let mut advisories: IndexMap> = IndexMap::new(); let mut names_found: IndexMap = IndexMap::new(); let api_url = self .security_advisory_config .as_ref() .and_then(|c| c.api_url.clone()); // respect available-package-patterns / available-packages directives from the repo if self.has_available_package_list { let names: Vec = package_constraint_map.keys().cloned().collect(); for name in names { if !self.lazy_providers_repo_contains(&strtolower(&name))? { package_constraint_map.shift_remove(&name); } } } let parser = VersionParser::new(); let semver_parser = shirabe_semver::version_parser::VersionParser; let repo_name = self.get_repo_name(); let create = |data: &IndexMap, name: &str, package_constraint_map: &IndexMap| -> anyhow::Result> { let advisory = PartialSecurityAdvisory::create(name, data, &semver_parser)?; let is_full = matches!(advisory, PartialOrSecurityAdvisory::Full(_)); if !allow_partial_advisories && !is_full { let data_mixed = PhpMixed::Array( data.iter() .map(|(k, v)| (k.clone(), Box::new(v.clone()))) .collect(), ); return Err(RuntimeException { message: format!( "Advisory for {} could not be loaded as a full advisory from {}{}{}", name, repo_name, PHP_EOL, var_export(&data_mixed, true), ), code: 0, } .into()); } let affected_versions: &AnyConstraint = match &advisory { PartialOrSecurityAdvisory::Partial(p) => &p.affected_versions, PartialOrSecurityAdvisory::Full(p) => p.affected_versions(), }; let constraint = package_constraint_map.get(name); if let Some(c) = constraint { if !affected_versions.matches(c) { return Ok(None); } } else { return Ok(None); } Ok(Some(advisory)) }; if self .security_advisory_config .as_ref() .map_or(false, |c| c.metadata) && (allow_partial_advisories || api_url.is_none()) { let names: Vec = package_constraint_map.keys().cloned().collect(); for name in names { let name = strtolower(&name); // skip platform packages, root package and composer-plugin-api if PlatformRepository::is_platform_package(&name) || name == "__root__" { continue; } let spec = tokio::runtime::Runtime::new() .unwrap() .block_on(self.start_cached_async_download(&name, Some(&name)))?; // [$response] = $spec; let response = spec .as_list() .and_then(|l| l.first()) .map(|b| (**b).clone()) .unwrap_or(PhpMixed::Null); let response_arr = match response.as_array() { Some(a) => a.clone(), None => continue, }; let sec_advs_arr = match response_arr .get("security-advisories") .and_then(|v| v.as_array()) { Some(a) => a.clone(), None => continue, }; names_found.insert(name.clone(), true); if !sec_advs_arr.is_empty() { let mut entries: Vec = Vec::new(); for (_k, data_mixed) in sec_advs_arr.iter() { if let Some(data) = data_mixed.as_array() { let data_map: IndexMap = data .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); if let Some(adv) = create(&data_map, &name, &package_constraint_map)? { entries.push(adv); } } } advisories.insert(name.clone(), entries); } package_constraint_map.shift_remove(&name); } } if let Some(api_url) = api_url { if !package_constraint_map.is_empty() { let mut options = self.options.clone(); let http_entry = options .entry("http".to_string()) .or_insert(PhpMixed::Array(IndexMap::new())); if let PhpMixed::Array(http_map) = http_entry { http_map.insert( "method".to_string(), Box::new(PhpMixed::String("POST".to_string())), ); if let Some(header_box) = http_map.get("header") { // cast to array let arr = match &**header_box { PhpMixed::List(l) => l.clone(), other => vec![Box::new(other.clone())], }; http_map.insert("header".to_string(), Box::new(PhpMixed::List(arr))); } let mut headers = match http_map.get("header") { Some(b) => match &**b { PhpMixed::List(l) => l.clone(), _ => vec![], }, None => vec![], }; headers.push(Box::new(PhpMixed::String( "Content-type: application/x-www-form-urlencoded".to_string(), ))); http_map.insert("header".to_string(), Box::new(PhpMixed::List(headers))); http_map.insert("timeout".to_string(), Box::new(PhpMixed::Int(10))); let packages_list: Vec<(String, String)> = package_constraint_map .keys() .map(|k| ("packages".to_string(), k.clone())) .collect(); let body = http_build_query( &packages_list .iter() .map(|(k, v)| (k.as_str(), v.as_str())) .collect::>(), "&", "=", ); http_map.insert("content".to_string(), Box::new(PhpMixed::String(body))); } let response = self.http_downloader.borrow_mut().get(&api_url, options)?; let mut warned = false; let decoded = response.decode_json()?; let advisories_response = decoded .as_array() .and_then(|a| a.get("advisories")) .and_then(|v| v.as_array()) .cloned() .unwrap_or_default(); for (name, list_box) in advisories_response.iter() { if !package_constraint_map.contains_key(name) { if !warned { self.io.write_error(&format!( "{} returned names which were not requested in response to the security-advisories API. {} was not requested but is present in the response. Requested names were: {}", self.get_repo_name(), name, package_constraint_map.keys().cloned().collect::>().join(", "), )); warned = true; } continue; } let list = match list_box.as_list() { Some(l) => l.clone(), None => continue, }; if !list.is_empty() { let mut entries: Vec = Vec::new(); for data_mixed in list.iter() { if let Some(data) = data_mixed.as_array() { let data_map: IndexMap = data .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); if let Some(adv) = create(&data_map, name, &package_constraint_map)? { entries.push(adv); } } } advisories.insert(name.clone(), entries); } names_found.insert(name.clone(), true); } } } let advisories_filtered: IndexMap> = advisories .into_iter() .filter(|(_, adv)| !adv.is_empty()) .collect(); Ok(SecurityAdvisoryResult { names_found: names_found.keys().cloned().collect(), advisories: advisories_filtered, }) } pub fn get_providers( &mut self, package_name: &str, ) -> anyhow::Result>> { self.load_root_server_file(None)?; let mut result: IndexMap> = IndexMap::new(); if let Some(providers_api_url) = self.providers_api_url.clone() { let api_result = match self.http_downloader.borrow_mut().get( &providers_api_url.replace("%package%", package_name), self.options.clone(), ) { Ok(resp) => resp.decode_json()?, Err(e) => { if let Some(te) = e.downcast_ref::() { if te.get_status_code() == Some(404) { return Ok(result); } } return Err(e); } }; if let Some(providers) = api_result .as_array() .and_then(|a| a.get("providers")) .and_then(|v| v.as_list()) { for provider_mixed in providers.iter() { if let Some(provider) = provider_mixed.as_array() { if let Some(name) = provider.get("name").and_then(|v| v.as_string()) { let entry: IndexMap = provider .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); result.insert(name.to_string(), entry); } } } } return Ok(result); } if self.has_partial_packages()? { if self.partial_packages_by_name.is_none() { return Err(LogicException { message: "hasPartialPackages failed to initialize $this->partialPackagesByName" .to_string(), code: 0, } .into()); } for (_name, versions) in self.partial_packages_by_name.as_ref().unwrap().iter() { for candidate in versions.iter() { let candidate_name = candidate .get("name") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); if result.contains_key(&candidate_name) || !candidate .get("provide") .and_then(|v| v.as_array()) .map_or(false, |a| a.contains_key(package_name)) { continue; } let mut entry: IndexMap = IndexMap::new(); entry.insert("name".to_string(), PhpMixed::String(candidate_name.clone())); entry.insert( "description".to_string(), candidate .get("description") .cloned() .unwrap_or(PhpMixed::String(String::new())), ); entry.insert( "type".to_string(), candidate .get("type") .cloned() .unwrap_or(PhpMixed::String(String::new())), ); result.insert(candidate_name, entry); } } } if Countable::count(&self.inner) > 0 { for (k, v) in self.inner.get_providers(package_name.to_string()) { // TODO(phase-b): ProviderInfo -> IndexMap conversion needed let mut entry: IndexMap = IndexMap::new(); entry.insert("name".to_string(), PhpMixed::String(v.name)); if let Some(d) = v.description { entry.insert("description".to_string(), PhpMixed::String(d)); } entry.insert("type".to_string(), PhpMixed::String(v.r#type)); result.insert(k, entry); } } Ok(result) } fn get_provider_names(&mut self) -> anyhow::Result> { self.load_root_server_file(None)?; if self.provider_listing.is_none() { let data = self.load_root_server_file(None)?; if let RootData::Data(arr) = &data { let arr_clone = arr.clone(); self.load_provider_listings(&arr_clone)?; } } if self.lazy_providers_url.is_some() { // Can not determine list of provided packages for lazy repositories return Ok(vec![]); } if self.providers_url.is_some() && self.provider_listing.is_some() { return Ok(self .provider_listing .as_ref() .unwrap() .keys() .cloned() .collect()); } Ok(vec![]) } fn configure_package_transport_options(&self, package: PackageInterfaceHandle) { for url in package.get_dist_urls() { if url.starts_with(&self.base_url) { package.set_transport_options(self.options.clone()); return; } } } fn has_providers(&mut self) -> anyhow::Result { self.load_root_server_file(None)?; Ok(self.has_providers) } /// @param name package name fn what_provides( &mut self, name: &str, acceptable_stabilities: Option<&IndexMap>, stability_flags: Option<&IndexMap>, already_loaded: IndexMap>, ) -> anyhow::Result> { let mut packages_source: Option = None; let packages: IndexMap; let loading_partial_package: bool; if !self.has_partial_packages()? || !self .partial_packages_by_name .as_ref() .map_or(false, |m| m.contains_key(name)) { // skip platform packages, root package and composer-plugin-api if PlatformRepository::is_platform_package(name) || name == "__root__" { return Ok(IndexMap::new()); } if self.provider_listing.is_none() { let data = self.load_root_server_file(None)?; if let RootData::Data(arr) = &data { let arr_clone = arr.clone(); self.load_provider_listings(&arr_clone)?; } } let mut use_last_modified_check = false; let hash_opt: Option; let url: String; let cache_key: String; if self.lazy_providers_url.is_some() && !self .provider_listing .as_ref() .map_or(false, |m| m.contains_key(name)) { hash_opt = None; url = self .lazy_providers_url .as_ref() .unwrap() .replace("%package%", name); cache_key = format!("provider-{}.json", strtr(name, "/", "$")); use_last_modified_check = true; } else if let Some(providers_url) = self.providers_url.clone() { // package does not exist in this repo if !self .provider_listing .as_ref() .map_or(false, |m| m.contains_key(name)) { return Ok(IndexMap::new()); } let listing = self.provider_listing.as_ref().unwrap(); let entry = listing.get(name).unwrap(); hash_opt = Some(entry.sha256.clone()); url = providers_url .replace("%package%", name) .replace("%hash%", &entry.sha256); cache_key = format!("provider-{}.json", strtr(name, "/", "$")); } else { return Ok(IndexMap::new()); } let mut packages_opt: Option> = None; if !use_last_modified_check && hash_opt.is_some() && self.cache.sha256(&cache_key).as_deref() == hash_opt.as_deref() { if let Some(raw) = self.cache.read(&cache_key) { let decoded = json_decode(&raw, true)?; if let Some(arr) = decoded.as_array() { let map: IndexMap = arr .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); packages_opt = Some(map); packages_source = Some(format!( "cached file ({} originating from {})", cache_key, Url::sanitize(url.clone()) )); } } } else if use_last_modified_check { if let Some(contents_raw) = self.cache.read(&cache_key) { let contents = json_decode(&contents_raw, true)?; let contents_arr = contents.as_array().cloned(); // we already loaded some packages from this file, so assume it is fresh and avoid fetching it again if already_loaded.contains_key(name) { if let Some(arr) = &contents_arr { let map: IndexMap = arr .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); packages_opt = Some(map); packages_source = Some(format!( "cached file ({} originating from {})", cache_key, Url::sanitize(url.clone()) )); } } else if let Some(arr) = &contents_arr { if let Some(last_modified) = arr.get("last-modified").and_then(|v| v.as_string()) { let response = self.fetch_file_if_last_modified(&url, &cache_key, last_modified)?; match response { FetchFileIfLastModifiedResult::NotModified => { let map: IndexMap = arr .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); packages_opt = Some(map); packages_source = Some(format!( "cached file ({} originating from {})", cache_key, Url::sanitize(url.clone()) )); } FetchFileIfLastModifiedResult::Data(data) => { packages_opt = Some(data); packages_source = Some(format!( "downloaded file ({})", Url::sanitize(url.clone()) )); } } } } } } if packages_opt.is_none() { match self.fetch_file( &url, Some(&cache_key), hash_opt.as_deref(), use_last_modified_check, ) { Ok(p) => { packages_opt = Some(p); packages_source = Some(format!("downloaded file ({})", Url::sanitize(url.clone()))); } Err(e) => { // 404s are acceptable for lazy provider repos if let Some(te) = e.downcast_ref::() { let status_code = te.get_status_code(); if self.lazy_providers_url.is_some() && in_array( match status_code { Some(c) => PhpMixed::Int(c), None => PhpMixed::Null, }, &PhpMixed::List(vec![ Box::new(PhpMixed::Int(404)), Box::new(PhpMixed::Int(499)), ]), true, ) { let mut p: IndexMap = IndexMap::new(); p.insert("packages".to_string(), PhpMixed::Array(IndexMap::new())); packages_opt = Some(p); packages_source = Some(format!( "not-found file ({})", Url::sanitize(url.clone()) )); if status_code == Some(499) { self.io.error( &format!("{}", te.get_message()), &[], ); } } else { return Err(e); } } else { return Err(e); } } } } packages = packages_opt.unwrap(); loading_partial_package = false; } else { let mut versions_map: IndexMap = IndexMap::new(); let mut packages_inner: IndexMap = IndexMap::new(); let entries = self .partial_packages_by_name .as_ref() .unwrap() .get(name) .cloned() .unwrap_or_default(); let entries_mixed: Vec> = entries .into_iter() .map(|m| { Box::new(PhpMixed::Array( m.into_iter().map(|(k, v)| (k, Box::new(v))).collect(), )) }) .collect(); versions_map.insert("versions".to_string(), PhpMixed::List(entries_mixed)); packages_inner.insert( "packages".to_string(), PhpMixed::Array( versions_map .into_iter() .map(|(k, v)| (k, Box::new(v))) .collect(), ), ); packages = packages_inner; packages_source = Some(format!( "root file ({})", Url::sanitize(self.get_packages_json_url()) )); loading_partial_package = true; } let mut result: IndexMap = IndexMap::new(); let mut versions_to_load: IndexMap> = IndexMap::new(); let packages_inner = packages .get("packages") .and_then(|v| v.as_array()) .cloned() .unwrap_or_default(); for (_pkg_key, versions_mixed) in packages_inner.iter() { // $versions can be either array or list let iter_versions: Vec = match &**versions_mixed { PhpMixed::Array(a) => a.values().map(|v| (**v).clone()).collect(), PhpMixed::List(l) => l.iter().map(|v| (**v).clone()).collect(), _ => continue, }; for version_mixed in iter_versions.iter() { let version_arr = match version_mixed.as_array() { Some(a) => a, None => continue, }; let mut version: IndexMap = version_arr .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); let normalized_name = strtolower( version .get("name") .and_then(|v| v.as_string()) .unwrap_or(""), ); // only load the actual named package, not other packages that might find themselves in the same file if normalized_name != name { continue; } if !loading_partial_package && self.has_partial_packages()? && self .partial_packages_by_name .as_ref() .map_or(false, |m| m.contains_key(&normalized_name)) { continue; } let uid_key = match version.get("uid") { Some(PhpMixed::Int(i)) => i.to_string(), Some(PhpMixed::String(s)) => s.clone(), Some(other) => format!("{:?}", other), None => continue, }; if !versions_to_load.contains_key(&uid_key) { let has_version_normalized = version.contains_key("version_normalized"); if !has_version_normalized { let v = version .get("version") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let normalized = self.version_parser.normalize(&v, None)?; version.insert( "version_normalized".to_string(), PhpMixed::String(normalized), ); } else if version .get("version_normalized") .and_then(|v| v.as_string()) .map_or(false, |s| s == VersionParser::DEFAULT_BRANCH_ALIAS) { // handling of existing repos which need to remain composer v1 compatible, in case the version_normalized contained VersionParser::DEFAULT_BRANCH_ALIAS, we renormalize it let v = version .get("version") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let normalized = self.version_parser.normalize(&v, None)?; version.insert( "version_normalized".to_string(), PhpMixed::String(normalized), ); } let version_normalized = version .get("version_normalized") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); // avoid loading packages which have already been loaded if already_loaded .get(name) .map_or(false, |m| m.contains_key(&version_normalized)) { continue; } if self.is_version_acceptable( None, &normalized_name, &version, acceptable_stabilities, stability_flags, )? { versions_to_load.insert(uid_key, version); } } } } // load acceptable packages in the providers let versions_to_load_vec: Vec> = versions_to_load.values().cloned().collect(); let loaded_packages = self.create_packages_flat(versions_to_load_vec, packages_source)?; let uids: Vec = versions_to_load.keys().cloned().collect(); let self_handle = self.self_handle(); for (index, package) in loaded_packages.into_iter().enumerate() { if let Some(h) = self_handle.as_ref() { package.set_repository(h.clone())?; } let uid = &uids[index]; if let Some(alias) = package.as_alias() { let aliased = alias.get_alias_of(); if let Some(h) = self_handle.as_ref() { let aliased_pkg: PackageInterfaceHandle = aliased.clone().into(); aliased_pkg.set_repository(h.clone())?; } result.insert(uid.clone(), aliased.into()); result.insert(format!("{}-alias", uid), package); } else { result.insert(uid.clone(), package); } } Ok(result) } /// @inheritDoc pub fn initialize(&mut self) -> anyhow::Result<()> { self.inner.initialize(); let repo_data = self.load_data_from_server()?; let source = format!( "root file ({})", Url::sanitize(self.get_packages_json_url()) ); for package in self.create_packages_flat(repo_data, Some(source))? { self.add_package(package); } Ok(()) } /// Adds a new package to the repository pub fn add_package(&mut self, package: BasePackageHandle) { self.configure_package_transport_options(package.clone()); self.inner.add_package(package.into()); } /// Forwards the outermost handle's weak to the inner `ArrayRepository` so that packages added /// through `add_package` get the back-reference to this `ComposerRepository` wrapper. pub fn set_self_handle(&self, weak: RepositoryInterfaceWeakHandle) { *self.self_weak.borrow_mut() = Some(weak.clone()); self.inner.set_self_handle(weak); } fn self_handle(&self) -> Option { self.self_weak .borrow() .as_ref() .and_then(std::rc::Weak::upgrade) .map(RepositoryInterfaceHandle::from_rc) } /// @param packageNames array of package name => ConstraintInterface|null - if a constraint is provided, only packages matching it will be loaded fn load_async_packages( &mut self, mut package_names: IndexMap>, acceptable_stabilities: Option<&IndexMap>, stability_flags: Option<&IndexMap>, already_loaded: IndexMap>, ) -> anyhow::Result { self.load_root_server_file(None)?; let mut packages: IndexMap = IndexMap::new(); let mut names_found: IndexMap = IndexMap::new(); if self.lazy_providers_url.is_none() { return Err(LogicException { message: "loadAsyncPackages only supports v2 protocol composer repos with a metadata-url" .to_string(), code: 0, } .into()); } // load ~dev versions of the packages as well if needed let names_snapshot: Vec = package_names.keys().cloned().collect(); for name in names_snapshot { let constraint = package_names .get(&name) .and_then(|c| c.as_ref().map(|c| c.clone())); if acceptable_stabilities.is_none() || stability_flags.is_none() || StabilityFilter::is_package_acceptable( acceptable_stabilities.unwrap(), stability_flags.unwrap(), &[name.clone()], "dev", ) { package_names.insert(format!("{}~dev", name), constraint); } // if only dev stability is requested, we skip loading the non dev file if acceptable_stabilities.map_or(false, |m| m.contains_key("dev") && m.len() == 1) && stability_flags.map_or(false, |m| m.is_empty()) { package_names.shift_remove(&name); } } let names_iter: Vec<(String, Option)> = package_names .iter() .map(|(k, v)| { let cloned: Option = v.clone(); (k.clone(), cloned) }) .collect(); for (name, constraint) in names_iter { let name = strtolower(&name); let real_name = Preg::replace(r"{~dev$}", "", &name)?; // skip platform packages, root package and composer-plugin-api if PlatformRepository::is_platform_package(&real_name) || real_name == "__root__" { continue; } let version_parser = self.version_parser.clone(); let spec = tokio::runtime::Runtime::new() .unwrap() .block_on(self.start_cached_async_download(&name, Some(&real_name)))?; // [$response, $packagesSource] = $spec; let spec_list = spec.as_list().cloned().unwrap_or_default(); let response = spec_list .first() .map(|b| (**b).clone()) .unwrap_or(PhpMixed::Null); let packages_source_val = spec_list .get(1) .map(|b| (**b).clone()) .unwrap_or(PhpMixed::Null); let packages_source: Option = packages_source_val.as_string().map(|s| s.to_string()); if response.is_null() { continue; } let response_arr = match response.as_array() { Some(a) => a.clone(), None => continue, }; let inner_packages = response_arr.get("packages"); let versions_mixed = match inner_packages .and_then(|v| v.as_array()) .and_then(|a| a.get(&real_name)) .cloned() { Some(b) => *b, None => continue, }; let mut versions: Vec> = match &versions_mixed { PhpMixed::List(l) => l .iter() .filter_map(|v| { v.as_array().map(|a| { a.iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect::>() }) }) .collect(), PhpMixed::Array(a) => a .values() .filter_map(|v| { v.as_array().map(|a| { a.iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect::>() }) }) .collect(), _ => continue, }; let minified = response_arr .get("minified") .and_then(|v| v.as_string()) .map_or(false, |s| s == "composer/2.0"); if minified { // TODO(phase-b): MetadataMinifier::expand expects/returns IndexMap but versions is Vec versions = todo!("MetadataMinifier::expand signature mismatch with Vec"); } names_found.insert(real_name.clone(), true); let mut versions_to_load: Vec> = Vec::new(); for version in versions.into_iter() { let mut version = version; let has_vn = version.contains_key("version_normalized"); if !has_vn { let v = version .get("version") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let normalized = version_parser.normalize(&v, None)?; version.insert( "version_normalized".to_string(), PhpMixed::String(normalized), ); } else if version .get("version_normalized") .and_then(|v| v.as_string()) .map_or(false, |s| s == VersionParser::DEFAULT_BRANCH_ALIAS) { // handling of existing repos which need to remain composer v1 compatible, in case the version_normalized contained VersionParser::DEFAULT_BRANCH_ALIAS, we renormalize it let v = version .get("version") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let normalized = version_parser.normalize(&v, None)?; version.insert( "version_normalized".to_string(), PhpMixed::String(normalized), ); } let version_normalized = version .get("version_normalized") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); // avoid loading packages which have already been loaded if already_loaded .get(&real_name) .map_or(false, |m| m.contains_key(&version_normalized)) { continue; } let acceptable = ComposerRepository::is_version_acceptable_static( constraint.as_ref(), &real_name, &version, acceptable_stabilities, stability_flags, )?; if acceptable { versions_to_load.push(version); } } let loaded_packages: Vec = ComposerRepository::create_packages_static(versions_to_load, packages_source)?; let self_handle = self.self_handle(); for package in loaded_packages.into_iter() { if let Some(h) = self_handle.as_ref() { package.set_repository(h.clone())?; } let hash_c = package.ptr_id().to_string(); if let Some(alias) = package.as_alias() { let aliased = alias.get_alias_of(); if let Some(h) = self_handle.as_ref() { let aliased_pkg: PackageInterfaceHandle = aliased.clone().into(); aliased_pkg.set_repository(h.clone())?; } let aliased_hash = aliased.ptr_id().to_string(); if !packages.contains_key(&aliased_hash) { packages.insert(aliased_hash, aliased.into()); } } packages.insert(hash_c, package); } } Ok(LoadAsyncPackagesResult { names_found, packages, }) } async fn start_cached_async_download( &mut self, file_name: &str, package_name: Option<&str>, ) -> anyhow::Result { if self.lazy_providers_url.is_none() { return Err(LogicException { message: "startCachedAsyncDownload only supports v2 protocol composer repos with a metadata-url".to_string(), code: 0, }.into()); } let name = strtolower(file_name); let package_name = package_name .map(|s| s.to_string()) .unwrap_or_else(|| name.clone()); let url = self .lazy_providers_url .as_ref() .unwrap() .replace("%package%", &name); let cache_key = format!("provider-{}.json", strtr(&name, "/", "~")); let mut last_modified: Option = None; let contents_opt: Option>; if let Some(raw) = self.cache.read(&cache_key) { let decoded = json_decode(&raw, true)?; if let Some(arr) = decoded.as_array() { let map: IndexMap = arr .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); last_modified = map .get("last-modified") .and_then(|v| v.as_string()) .map(|s| s.to_string()); contents_opt = Some(map); } else { contents_opt = None; } } else { contents_opt = None; } // PHP: return $this->asyncFetchFile(...)->then(function ($response) use (...): array { ... }); let response = self .async_fetch_file(&url, &cache_key, last_modified.as_deref()) .await?; let mut packages_source = format!("downloaded file ({})", Url::sanitize(url.clone())); let response_data = if response.as_bool() == Some(true) { packages_source = format!( "cached file ({} originating from {})", cache_key, Url::sanitize(url.clone()) ); contents_opt .map(|m| PhpMixed::Array(m.into_iter().map(|(k, v)| (k, Box::new(v))).collect())) .unwrap_or(PhpMixed::Null) } else { response }; let response_arr = response_data.as_array(); let has_pkg = response_arr .and_then(|a| a.get("packages")) .and_then(|v| v.as_array()) .map_or(false, |a| a.contains_key(&package_name)); let has_advisories = response_arr.map_or(false, |a| a.contains_key("security-advisories")); if !has_pkg && !has_advisories { return Ok(PhpMixed::List(vec![ Box::new(PhpMixed::Null), Box::new(PhpMixed::String(packages_source)), ])); } Ok(PhpMixed::List(vec![ Box::new(response_data), Box::new(PhpMixed::String(packages_source)), ])) } /// @param name package name (must be lowercased already) fn is_version_acceptable( &self, constraint: Option<&AnyConstraint>, name: &str, version_data: &IndexMap, acceptable_stabilities: Option<&IndexMap>, stability_flags: Option<&IndexMap>, ) -> anyhow::Result { Self::is_version_acceptable_with_loader( &self.loader, constraint, name, version_data, acceptable_stabilities, stability_flags, ) } fn is_version_acceptable_static( constraint: Option<&AnyConstraint>, name: &str, version_data: &IndexMap, acceptable_stabilities: Option<&IndexMap>, stability_flags: Option<&IndexMap>, ) -> anyhow::Result { Self::is_version_acceptable_with_loader( &ArrayLoader::new(Some(VersionParser::new()), true), constraint, name, version_data, acceptable_stabilities, stability_flags, ) } fn is_version_acceptable_with_loader( loader: &ArrayLoader, constraint: Option<&AnyConstraint>, name: &str, version_data: &IndexMap, acceptable_stabilities: Option<&IndexMap>, stability_flags: Option<&IndexMap>, ) -> anyhow::Result { let mut versions: Vec = vec![ version_data .get("version_normalized") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(), ]; if let Some(alias) = loader.get_branch_alias(version_data)? { versions.push(alias); } for version in versions.iter() { if acceptable_stabilities.is_some() && stability_flags.is_some() && !StabilityFilter::is_package_acceptable( acceptable_stabilities.unwrap(), stability_flags.unwrap(), &[name.to_string()], &VersionParser::parse_stability(version), ) { continue; } if let Some(c) = constraint { if !CompilingMatcher::r#match(c, SimpleConstraint::OP_EQ, version.clone()) { continue; } } return Ok(true); } Ok(false) } pub fn get_packages_json_url(&self) -> String { let json_url_parts = parse_url_all(&strtr(&self.url, "\\", "/")); let has_json = json_url_parts .as_array() .and_then(|a| a.get("path")) .and_then(|v| v.as_string()) .map_or(false, |p| p.contains(".json")); if has_json { return self.url.clone(); } format!("{}/packages.json", self.url) } fn load_root_server_file(&mut self, root_max_age: Option) -> anyhow::Result { if let Some(rd) = &self.root_data { return Ok(clone_root_data(rd)); } if !extension_loaded("openssl") && self.url.starts_with("https") { return Err(RuntimeException { message: format!( "You must enable the openssl extension in your php.ini to load information from {}", self.url ), code: 0, }.into()); } let mut data: Option> = None; if let Some(cached_raw) = self.cache.read("packages.json") { let cached_decoded = json_decode(&cached_raw, true)?; if let Some(arr) = cached_decoded.as_array() { let cached_data: IndexMap = arr .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); let age = self.cache.get_age("packages.json"); if root_max_age.is_some() && age.is_some() && age.unwrap() <= root_max_age.unwrap() { data = Some(cached_data); } else if let Some(last_modified) = cached_data .get("last-modified") .and_then(|v| v.as_string()) .map(|s| s.to_string()) { let response = self.fetch_file_if_last_modified( &self.get_packages_json_url(), "packages.json", &last_modified, )?; data = Some(match response { FetchFileIfLastModifiedResult::NotModified => cached_data, FetchFileIfLastModifiedResult::Data(d) => d, }); } } } if data.is_none() { data = Some(self.fetch_file( &self.get_packages_json_url(), Some("packages.json"), None, true, )?); } let mut data = data.unwrap(); if let Some(notify_batch) = data .get("notify-batch") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.notify_url = Some(self.canonicalize_url(¬ify_batch)?); } else if let Some(notify) = data .get("notify") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.notify_url = Some(self.canonicalize_url(¬ify)?); } if let Some(search) = data .get("search") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.search_url = Some(self.canonicalize_url(&search)?); } if let Some(mirrors) = data.get("mirrors").and_then(|v| v.as_list()).cloned() { for mirror_mixed in mirrors.iter() { let mirror = match mirror_mixed.as_array() { Some(a) => a, None => continue, }; if let Some(git_url) = mirror .get("git-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { let preferred = mirror .get("preferred") .and_then(|v| v.as_bool()) .unwrap_or(false); self.source_mirrors .get_or_insert_with(IndexMap::new) .entry("git".to_string()) .or_insert_with(Vec::new) .push(SourceMirror { url: git_url, preferred, }); } if let Some(hg_url) = mirror .get("hg-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { let preferred = mirror .get("preferred") .and_then(|v| v.as_bool()) .unwrap_or(false); self.source_mirrors .get_or_insert_with(IndexMap::new) .entry("hg".to_string()) .or_insert_with(Vec::new) .push(SourceMirror { url: hg_url, preferred, }); } if let Some(dist_url) = mirror .get("dist-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { let preferred = mirror .get("preferred") .and_then(|v| v.as_bool()) .unwrap_or(false); let url = self.canonicalize_url(&dist_url)?; self.dist_mirrors .get_or_insert_with(Vec::new) .push(DistMirror { url, preferred }); } } } if let Some(providers_lazy_url) = data .get("providers-lazy-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.lazy_providers_url = Some(self.canonicalize_url(&providers_lazy_url)?); self.has_providers = true; self.has_partial_packages = data .get("packages") .map(|v| match v { PhpMixed::Array(a) => !a.is_empty(), PhpMixed::List(l) => !l.is_empty(), _ => false, }) .unwrap_or(false); } // metadata-url indicates V2 repo protocol so it takes over from all the V1 types // V2 only has lazyProviders and possibly partial packages, but no ability to process anything else, // V2 also supports async loading if let Some(metadata_url) = data .get("metadata-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.lazy_providers_url = Some(self.canonicalize_url(&metadata_url)?); self.providers_url = None; self.has_providers = false; self.has_partial_packages = data .get("packages") .map(|v| match v { PhpMixed::Array(a) => !a.is_empty(), PhpMixed::List(l) => !l.is_empty(), _ => false, }) .unwrap_or(false); self.allow_ssl_downgrade = false; // provides a list of package names that are available in this repo // this disables lazy-provider behavior in the sense that if a list is available we assume it is finite and won't search for other packages in that repo // while if no list is there lazyProvidersUrl is used when looking for any package name to see if the repo knows it if let Some(available) = data .get("available-packages") .and_then(|v| v.as_list()) .cloned() { if !available.is_empty() { let avail_packages: Vec = available .iter() .filter_map(|v| v.as_string().map(|s| strtolower(s))) .collect(); let mut combined: IndexMap = IndexMap::new(); for k in avail_packages.iter() { combined.insert(k.clone(), k.clone()); } self.available_packages = Some(combined); self.has_available_package_list = true; } } // Provides a list of package name patterns (using * wildcards to match any substring, e.g. "vendor/*") that are available in this repo // Disables lazy-provider behavior as with available-packages, but may allow much more compact expression of packages covered by this repository. // Over-specifying covered packages is safe, but may result in increased traffic to your repository. if let Some(patterns) = data .get("available-package-patterns") .and_then(|v| v.as_list()) .cloned() { if !patterns.is_empty() { let mapped: Vec = patterns .iter() .filter_map(|v| v.as_string()) .map(|p| base_package::package_name_to_regexp(p)) .collect(); self.available_package_patterns = Some(mapped); self.has_available_package_list = true; } } // Remove legacy keys as most repos need to be compatible with Composer v1 // as well but we are not interested in the old format anymore at this point data.shift_remove("providers-url"); data.shift_remove("providers"); data.shift_remove("providers-includes"); if let Some(sec) = data .get("security-advisories") .and_then(|v| v.as_array()) .cloned() { let metadata = sec .get("metadata") .and_then(|v| v.as_bool()) .unwrap_or(false); let api_url_str = sec .get("api-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()); let api_url = if let Some(s) = api_url_str { Some(self.canonicalize_url(s)?) } else { None }; self.security_advisory_config = Some(SecurityAdvisoryConfig { metadata, api_url: api_url.clone(), }); if api_url.is_none() && !self.has_available_package_list { return Err(UnexpectedValueException { message: format!( "Invalid security advisory configuration on {}: If the repository does not provide a security-advisories.api-url then available-packages or available-package-patterns are required to be provided for performance reason.", self.get_repo_name() ), code: 0, }.into()); } } } if self.allow_ssl_downgrade { self.url = self.url.replace("https://", "http://"); self.base_url = self.base_url.replace("https://", "http://"); } if let Some(providers_url) = data .get("providers-url") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.providers_url = Some(self.canonicalize_url(&providers_url)?); self.has_providers = true; } if let Some(list) = data .get("list") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.list_url = Some(self.canonicalize_url(&list)?); } let providers_non_empty = data .get("providers") .map(|v| match v { PhpMixed::Array(a) => !a.is_empty(), PhpMixed::List(l) => !l.is_empty(), _ => false, }) .unwrap_or(false); let providers_includes_non_empty = data .get("providers-includes") .map(|v| match v { PhpMixed::Array(a) => !a.is_empty(), PhpMixed::List(l) => !l.is_empty(), _ => false, }) .unwrap_or(false); if providers_non_empty || providers_includes_non_empty { self.has_providers = true; } if let Some(providers_api) = data .get("providers-api") .and_then(|v| v.as_string()) .filter(|s| !s.is_empty()) .map(|s| s.to_string()) { self.providers_api_url = Some(self.canonicalize_url(&providers_api)?); } self.root_data = Some(RootData::Data(data.clone())); Ok(RootData::Data(data)) } fn canonicalize_url(&self, url: &str) -> anyhow::Result { if url.is_empty() { return Err(InvalidArgumentException { message: "Expected a string with a value and not an empty string".to_string(), code: 0, } .into()); } if url.starts_with('/') { let mut matches: IndexMap = IndexMap::new(); if Preg::is_match3(r"{^[^:]++://[^/]*+}", &self.url, Some(&mut matches))? { return Ok(format!( "{}{}", matches .get(&CaptureKey::ByIndex(0)) .cloned() .unwrap_or_default(), url )); } return Ok(self.url.clone()); } Ok(url.to_string()) } fn load_data_from_server(&mut self) -> anyhow::Result>> { let data = self.load_root_server_file(None)?; let data = match data { RootData::True => { return Err(LogicException { message: "loadRootServerFile should not return true during initialization" .to_string(), code: 0, } .into()); } RootData::Data(d) => d, }; self.load_includes(&data) } fn has_partial_packages(&mut self) -> anyhow::Result { if self.has_partial_packages && self.partial_packages_by_name.is_none() { self.initialize_partial_packages()?; } Ok(self.has_partial_packages) } fn load_provider_listings(&mut self, data: &IndexMap) -> anyhow::Result<()> { if let Some(providers) = data.get("providers").and_then(|v| v.as_array()) { if self.provider_listing.is_none() { self.provider_listing = Some(IndexMap::new()); } let listing = self.provider_listing.as_mut().unwrap(); for (k, v) in providers.iter() { if let Some(arr) = v.as_array() { if let Some(sha256) = arr.get("sha256").and_then(|v| v.as_string()) { listing.insert( k.clone(), ProviderListingEntry { sha256: sha256.to_string(), }, ); } } } } if self.providers_url.is_some() { if let Some(includes) = data .get("provider-includes") .and_then(|v| v.as_array()) .cloned() { for (include, metadata_mixed) in includes.iter() { let metadata = match metadata_mixed.as_array() { Some(a) => a, None => continue, }; let sha256 = metadata .get("sha256") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let url = format!("{}/{}", self.base_url, include.replace("%hash%", &sha256)); let cache_key = include.replace("%hash%", "").replace("$", ""); let included_data: IndexMap = if self.cache.sha256(&cache_key).as_deref() == Some(sha256.as_str()) { let raw = self.cache.read(&cache_key).unwrap_or_default(); let decoded = json_decode(&raw, true)?; decoded .as_array() .map(|a| { a.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect() }) .unwrap_or_default() } else { self.fetch_file(&url, Some(&cache_key), Some(&sha256), false)? }; self.load_provider_listings(&included_data)?; } } } Ok(()) } fn load_includes( &mut self, data: &IndexMap, ) -> anyhow::Result>> { let mut packages: Vec> = Vec::new(); // legacy repo handling if !data.contains_key("packages") && !data.contains_key("includes") { for (_k, pkg_mixed) in data.iter() { let pkg = match pkg_mixed.as_array() { Some(a) => a, None => continue, }; if let Some(versions) = pkg.get("versions").and_then(|v| v.as_array()) { for (_, metadata) in versions.iter() { if let Some(m) = metadata.as_array() { packages .push(m.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect()); } } } } return Ok(packages); } if let Some(pkgs) = data.get("packages").and_then(|v| v.as_array()).cloned() { for (package, versions_mixed) in pkgs.iter() { let package_name = strtolower(package); let versions = match versions_mixed.as_array() { Some(a) => a.clone(), None => continue, }; for (_version, metadata_mixed) in versions.iter() { let metadata = match metadata_mixed.as_array() { Some(a) => a.clone(), None => continue, }; let metadata_map: IndexMap = metadata .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); packages.push(metadata_map.clone()); let meta_name = metadata_map .get("name") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); if !self.displayed_warning_about_non_matching_package_index && package_name != strtolower(&meta_name) { self.displayed_warning_about_non_matching_package_index = true; self.io.write_error(&format!( "Warning: the packages key '{}' doesn't match the name defined in the package metadata '{}' in repository {}", package, meta_name, self.base_url )); } } } } if let Some(includes) = data.get("includes").and_then(|v| v.as_array()).cloned() { for (include, metadata_mixed) in includes.iter() { let metadata = match metadata_mixed.as_array() { Some(a) => a, None => continue, }; let sha1 = metadata .get("sha1") .and_then(|v| v.as_string()) .map(|s| s.to_string()); let included_data: IndexMap = if let Some(ref sha1) = sha1 { if self.cache.sha1(include).as_deref() == Some(sha1.as_str()) { let raw = self.cache.read(include).unwrap_or_default(); let decoded = json_decode(&raw, true)?; decoded .as_array() .map(|a| a.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect()) .unwrap_or_default() } else { self.fetch_file(include, None, None, false)? } } else { self.fetch_file(include, None, None, false)? }; let included_packages = self.load_includes(&included_data)?; for p in included_packages.into_iter() { packages.push(p); } } } Ok(packages) } fn create_packages_flat( &mut self, packages: Vec>, source: Option, ) -> anyhow::Result> { if packages.is_empty() { return Ok(vec![]); } let mut packages = packages; let result = (|| -> anyhow::Result> { for data in packages.iter_mut() { if !data.contains_key("notification-url") { data.insert( "notification-url".to_string(), match &self.notify_url { Some(s) => PhpMixed::String(s.clone()), None => PhpMixed::Null, }, ); } } let package_instances = self.loader.load_packages(packages.clone())?; let mut results: Vec = Vec::new(); for package in package_instances.into_iter() { if let Some(src_type) = package.get_source_type() { if let Some(mirrors) = self.source_mirrors.as_ref().and_then(|m| m.get(&src_type)) { let converted: Vec> = mirrors .iter() .map(|m| { let mut im: IndexMap = IndexMap::new(); im.insert("url".to_string(), PhpMixed::String(m.url.clone())); im.insert("preferred".to_string(), PhpMixed::Bool(m.preferred)); im }) .collect(); package.set_source_mirrors(Some(converted)); } } if let Some(dist_mirrors) = self.dist_mirrors.as_ref() { let converted: Vec> = dist_mirrors .iter() .map(|m| { let mut im: IndexMap = IndexMap::new(); im.insert("url".to_string(), PhpMixed::String(m.url.clone())); im.insert("preferred".to_string(), PhpMixed::Bool(m.preferred)); im }) .collect(); package.set_dist_mirrors(Some(converted)); } self.configure_package_transport_options(package.clone()); results.push(package.into()); } Ok(results) })(); result.map_err(|e| { RuntimeException { message: format!( "Could not load packages in {}{}: [{}] {}", self.get_repo_name(), source .as_ref() .map(|s| format!(" from {}", s)) .unwrap_or_default(), "Exception", e.to_string() ), code: 0, } .into() }) } fn create_packages_static( packages: Vec>, _source: Option, ) -> anyhow::Result> { if packages.is_empty() { return Ok(vec![]); } let loader = ArrayLoader::new(Some(VersionParser::new()), true); Ok(loader .load_packages(packages)? .into_iter() .map(|p| p.into()) .collect()) } fn fetch_file( &mut self, filename: &str, cache_key: Option<&str>, sha256: Option<&str>, store_last_modified_time: bool, ) -> anyhow::Result> { if filename.is_empty() { return Err(InvalidArgumentException { message: "$filename should not be an empty string".to_string(), code: 0, } .into()); } let (mut filename, cache_key_owned): (String, Option) = match cache_key { None => { let ck = filename.to_string(); let fn_full = format!("{}/{}", self.base_url, filename); (fn_full, Some(ck)) } Some(ck) => (filename.to_string(), Some(ck.to_string())), }; // url-encode $ signs in URLs as bad proxies choke on them if let Some(pos) = filename.find('$') { if pos > 0 && Preg::is_match(r"{^https?://}i", &filename)? { filename = format!("{}%24{}", &filename[..pos], &filename[pos + 1..]); } } let mut retries: i64 = 3; let mut data: Option> = None; while { let cont = retries > 0; retries -= 1; cont } { let attempt: anyhow::Result<()> = (|| -> anyhow::Result<()> { let mut options = self.options.clone(); if let Some(dispatcher) = self.event_dispatcher.as_ref() { let mut dispatcher = dispatcher.borrow_mut(); let mut pre_file_download_event = PreFileDownloadEvent::new( PluginEvents::PRE_FILE_DOWNLOAD.to_string(), self.http_downloader.clone(), filename.clone(), "metadata".to_string(), { let mut m: IndexMap = IndexMap::new(); // TODO(plugin): pass repository self-reference m.insert("repository".to_string(), PhpMixed::Null); m.into() }, ); pre_file_download_event.set_transport_options( self.options .clone() .into_iter() .map(|(k, v)| (k, Box::new(v))) .collect(), ); // TODO(phase-b): dispatcher.dispatch expects Option, not concrete event types; // need a way to pass PreFileDownloadEvent through EventDispatcher's API. let _ = &mut pre_file_download_event; dispatcher.dispatch(Some(pre_file_download_event.get_name()), None)?; filename = pre_file_download_event.get_processed_url().to_string(); options = pre_file_download_event .get_transport_options() .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); } let mut response = self .http_downloader .borrow_mut() .get(&filename, options.clone())?; let mut json = response.get_body().unwrap_or("").to_string(); if let Some(sha256_val) = sha256 { if sha256_val != hash("sha256", &json) { // undo downgrade before trying again if http seems to be hijacked or modifying content somehow if self.allow_ssl_downgrade { self.url = self.url.replace("http://", "https://"); self.base_url = self.base_url.replace("http://", "https://"); filename = filename.replace("http://", "https://"); } if retries > 0 { std::thread::sleep(std::time::Duration::from_micros(100000)); return Err(RetryMarker.into()); } // TODO use scarier wording once we know for sure it doesn't do false positives anymore return Err(RepositorySecurityException(shirabe_php_shim::Exception { message: format!( "The contents of {} do not match its signature. This could indicate a man-in-the-middle attack or e.g. antivirus software corrupting files. Try running composer again and report this if you think it is a mistake.", filename ), code: 0, }).into()); } } if let Some(dispatcher) = self.event_dispatcher.as_ref() { let mut dispatcher = dispatcher.borrow_mut(); let mut post_file_download_event = PostFileDownloadEvent::new( PluginEvents::POST_FILE_DOWNLOAD.to_string(), None, sha256.map(|s| s.to_string()), filename.clone(), "metadata".to_string(), { let mut m: IndexMap = IndexMap::new(); // TODO(plugin): pass response and repository self-reference m.insert("response".to_string(), PhpMixed::Null); m.insert("repository".to_string(), PhpMixed::Null); m.into() }, ); // TODO(phase-b): dispatcher.dispatch expects Option, not concrete event types let _ = &mut post_file_download_event; dispatcher.dispatch(Some(post_file_download_event.get_name()), None)?; } let decoded = response.decode_json()?; let mut data_local: IndexMap = decoded .as_array() .map(|a| a.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect()) .unwrap_or_default(); HttpDownloader::output_warnings(self.io.clone(), &self.url, &data_local); if let Some(ck) = cache_key_owned.as_ref() { if !ck.is_empty() && !self.cache.is_read_only() { if store_last_modified_time { if let Some(last_modified_date) = response.get_header("last-modified") { data_local.insert( "last-modified".to_string(), PhpMixed::String(last_modified_date), ); let as_mixed = PhpMixed::Array( data_local .iter() .map(|(k, v)| (k.clone(), Box::new(v.clone()))) .collect(), ); json = JsonFile::encode(&as_mixed, 0); } } self.cache.write(ck, &json); } } response.collect(); data = Some(data_local); Ok(()) })(); match attempt { Ok(()) => break, Err(e) => { if e.downcast_ref::().is_some() { continue; } if e.downcast_ref::().is_some() { return Err(e); } if let Some(te) = e.downcast_ref::() { if te.get_status_code() == Some(404) { return Err(e); } } if e.downcast_ref::().is_some() { return Err(e); } if let Some(ck) = cache_key_owned.as_ref() { if !ck.is_empty() { if let Some(contents) = self.cache.read(ck) { if !self.degraded_mode { self.io.write_error(&format!( "{} could not be fully loaded ({}), package information was loaded from the local cache and may be out of date", self.url, e.to_string() )); } self.degraded_mode = true; let parsed = JsonFile::parse_json( Some(&contents), Some(&format!("{}{}", self.cache.get_root(), ck)), )?; let map: IndexMap = parsed .as_array() .map(|a| { a.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect() }) .unwrap_or_default(); data = Some(map); break; } } } return Err(e); } } } match data { Some(d) => Ok(d), None => Err(LogicException { message: "ComposerRepository: Undefined $data. Please report at https://github.com/composer/composer/issues/new.".to_string(), code: 0, }.into()), } } fn fetch_file_if_last_modified( &mut self, filename: &str, cache_key: &str, last_modified_time: &str, ) -> anyhow::Result { if filename.is_empty() { return Err(InvalidArgumentException { message: "$filename should not be an empty string".to_string(), code: 0, } .into()); } let mut filename = filename.to_string(); let result: anyhow::Result = (|| { let mut options = self.options.clone(); if let Some(dispatcher) = self.event_dispatcher.as_ref() { let mut dispatcher = dispatcher.borrow_mut(); let mut pre_file_download_event = PreFileDownloadEvent::new( PluginEvents::PRE_FILE_DOWNLOAD.to_string(), self.http_downloader.clone(), filename.clone(), "metadata".to_string(), { let mut m: IndexMap = IndexMap::new(); m.insert("repository".to_string(), PhpMixed::Null); m.into() }, ); pre_file_download_event.set_transport_options( self.options .clone() .into_iter() .map(|(k, v)| (k, Box::new(v))) .collect(), ); // TODO(phase-b): dispatcher.dispatch expects Option, not concrete event types let _ = &mut pre_file_download_event; dispatcher.dispatch(Some(pre_file_download_event.get_name()), None)?; filename = pre_file_download_event.get_processed_url().to_string(); options = pre_file_download_event .get_transport_options() .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); } // cast http.header to array, then append let http_entry = options .entry("http".to_string()) .or_insert(PhpMixed::Array(IndexMap::new())); if let PhpMixed::Array(http_map) = http_entry { if let Some(existing) = http_map.get("header") { let arr = match &**existing { PhpMixed::List(l) => l.clone(), other => vec![Box::new(other.clone())], }; http_map.insert("header".to_string(), Box::new(PhpMixed::List(arr))); } let mut headers = match http_map.get("header") { Some(b) => match &**b { PhpMixed::List(l) => l.clone(), _ => vec![], }, None => vec![], }; headers.push(Box::new(PhpMixed::String(format!( "If-Modified-Since: {}", last_modified_time )))); http_map.insert("header".to_string(), Box::new(PhpMixed::List(headers))); } let mut response = self .http_downloader .borrow_mut() .get(&filename, options.clone())?; let mut json = response.get_body().unwrap_or("").to_string(); if json.is_empty() && response.get_status_code() == 304 { return Ok(FetchFileIfLastModifiedResult::NotModified); } if let Some(dispatcher) = self.event_dispatcher.as_ref() { let mut dispatcher = dispatcher.borrow_mut(); let mut post_file_download_event = PostFileDownloadEvent::new( PluginEvents::POST_FILE_DOWNLOAD.to_string(), None, None, filename.clone(), "metadata".to_string(), { let mut m: IndexMap = IndexMap::new(); m.insert("response".to_string(), PhpMixed::Null); m.insert("repository".to_string(), PhpMixed::Null); m.into() }, ); // TODO(phase-b): dispatcher.dispatch expects Option, not concrete event types let _ = &mut post_file_download_event; dispatcher.dispatch(Some(post_file_download_event.get_name()), None)?; } let decoded = response.decode_json()?; let mut data: IndexMap = decoded .as_array() .map(|a| a.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect()) .unwrap_or_default(); HttpDownloader::output_warnings(self.io.clone(), &self.url, &data); let last_modified_date = response.get_header("last-modified"); response.collect(); if let Some(ref lmd) = last_modified_date { data.insert("last-modified".to_string(), PhpMixed::String(lmd.clone())); let as_mixed = PhpMixed::Array( data.iter() .map(|(k, v)| (k.clone(), Box::new(v.clone()))) .collect(), ); json = JsonFile::encode(&as_mixed, 0); } if !self.cache.is_read_only() { self.cache.write(cache_key, &json); } Ok(FetchFileIfLastModifiedResult::Data(data)) })(); match result { Ok(v) => Ok(v), Err(e) => { if e.downcast_ref::().is_some() { return Err(e); } if let Some(te) = e.downcast_ref::() { if te.get_status_code() == Some(404) { return Err(e); } } if !self.degraded_mode { self.io.write_error(&format!( "{} could not be fully loaded ({}), package information was loaded from the local cache and may be out of date", self.url, e.to_string() )); } self.degraded_mode = true; Ok(FetchFileIfLastModifiedResult::NotModified) } } } async fn async_fetch_file( &mut self, filename: &str, cache_key: &str, last_modified_time: Option<&str>, ) -> anyhow::Result { if filename.is_empty() { return Err(InvalidArgumentException { message: "$filename should not be an empty string".to_string(), code: 0, } .into()); } if self.packagesNotFoundCache.contains_key(filename) { let mut empty: IndexMap = IndexMap::new(); empty.insert("packages".to_string(), PhpMixed::Array(IndexMap::new())); return Ok(PhpMixed::Array( empty.into_iter().map(|(k, v)| (k, Box::new(v))).collect(), )); } if self.freshMetadataUrls.contains_key(filename) && last_modified_time.is_some() { // make it look like we got a 304 response return Ok(PhpMixed::Bool(true)); } let mut filename = filename.to_string(); let mut options = self.options.clone(); if let Some(dispatcher) = self.event_dispatcher.as_ref() { let mut dispatcher = dispatcher.borrow_mut(); let mut pre_file_download_event = PreFileDownloadEvent::new( PluginEvents::PRE_FILE_DOWNLOAD.to_string(), self.http_downloader.clone(), filename.clone(), "metadata".to_string(), { let mut m: IndexMap = IndexMap::new(); m.insert("repository".to_string(), PhpMixed::Null); m.into() }, ); pre_file_download_event.set_transport_options( self.options .clone() .into_iter() .map(|(k, v)| (k, Box::new(v))) .collect(), ); // TODO(phase-b): dispatcher.dispatch expects Option, not concrete event types let _ = &mut pre_file_download_event; dispatcher.dispatch(Some(pre_file_download_event.get_name()), None)?; filename = pre_file_download_event.get_processed_url().to_string(); options = pre_file_download_event .get_transport_options() .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); } if let Some(last_modified_time) = last_modified_time { let http_entry = options .entry("http".to_string()) .or_insert(PhpMixed::Array(IndexMap::new())); if let PhpMixed::Array(http_map) = http_entry { if let Some(existing) = http_map.get("header") { let arr = match &**existing { PhpMixed::List(l) => l.clone(), other => vec![Box::new(other.clone())], }; http_map.insert("header".to_string(), Box::new(PhpMixed::List(arr))); } let mut headers = match http_map.get("header") { Some(b) => match &**b { PhpMixed::List(l) => l.clone(), _ => vec![], }, None => vec![], }; headers.push(Box::new(PhpMixed::String(format!( "If-Modified-Since: {}", last_modified_time )))); http_map.insert("header".to_string(), Box::new(PhpMixed::List(headers))); } } let response_result = self .http_downloader .borrow_mut() .add(&filename, options) .await; match response_result { Ok(response) => self.async_fetch_file_accept(response, &filename, cache_key), Err(e) => self.async_fetch_file_reject(e, &filename, cache_key, last_modified_time), } } /// The onFulfilled handler of `asyncFetchFile`: turns the HTTP response into decoded metadata, /// caches it, and records the URL as fresh. fn async_fetch_file_accept( &mut self, mut response: Response, filename: &str, cache_key: &str, ) -> anyhow::Result { // package not found is acceptable for a v2 protocol repository if response.get_status_code() == 404 { self.packagesNotFoundCache .insert(filename.to_string(), true); let mut empty: IndexMap = IndexMap::new(); empty.insert("packages".to_string(), PhpMixed::Array(IndexMap::new())); return Ok(PhpMixed::Array( empty.into_iter().map(|(k, v)| (k, Box::new(v))).collect(), )); } let mut json = response.get_body().unwrap_or("").to_string(); if json.is_empty() && response.get_status_code() == 304 { self.freshMetadataUrls.insert(filename.to_string(), true); return Ok(PhpMixed::Bool(true)); } // TODO(plugin): dispatch PostFileDownloadEvent let decoded = response.decode_json()?; let mut data: IndexMap = decoded .as_array() .map(|a| a.iter().map(|(k, v)| (k.clone(), (**v).clone())).collect()) .unwrap_or_default(); HttpDownloader::output_warnings(self.io.clone(), &self.url, &data); let last_modified_date = response.get_header("last-modified"); response.collect(); if let Some(lmd) = last_modified_date { data.insert("last-modified".to_string(), PhpMixed::String(lmd)); let as_mixed = PhpMixed::Array( data.iter() .map(|(k, v)| (k.clone(), Box::new(v.clone()))) .collect(), ); json = JsonFile::encode(&as_mixed, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } if !self.cache.is_read_only() { self.cache.write(cache_key, &json); } self.freshMetadataUrls.insert(filename.to_string(), true); Ok(PhpMixed::Array( data.into_iter().map(|(k, v)| (k, Box::new(v))).collect(), )) } /// The onRejected handler of `asyncFetchFile`: marks the package as not found / the repo as /// degraded, and fakes a 304/404 response from cache where appropriate. fn async_fetch_file_reject( &mut self, e: anyhow::Error, filename: &str, cache_key: &str, last_modified_time: Option<&str>, ) -> anyhow::Result { if let Some(te) = e.downcast_ref::() { if te.get_status_code() == Some(404) { self.packagesNotFoundCache .insert(filename.to_string(), true); return Ok(PhpMixed::Bool(false)); } } if !self.degraded_mode { self.io.write_error(&format!( "{} could not be fully loaded ({}), package information was loaded from the local cache and may be out of date", self.url, e )); } self.degraded_mode = true; // if the file is in the cache, we fake a 304 Not Modified to allow the process to continue if last_modified_time.is_some() { let resp = Response::new_fake(&self.url, 304, IndexMap::new(), String::new()); return self.async_fetch_file_accept(resp, filename, cache_key); } // special error code returned when network is being artificially disabled if let Some(te) = e.downcast_ref::() { if te.get_status_code() == Some(499) { let resp = Response::new_fake(&self.url, 404, IndexMap::new(), String::new()); return self.async_fetch_file_accept(resp, filename, cache_key); } } Err(e) } /// This initializes the packages key of a partial packages.json that contain some packages inlined + a providers-lazy-url /// /// This should only be called once fn initialize_partial_packages(&mut self) -> anyhow::Result<()> { let root_data = self.load_root_server_file(None)?; let root_data = match root_data { RootData::True => return Ok(()), RootData::Data(d) => d, }; self.partial_packages_by_name = Some(IndexMap::new()); if let Some(packages) = root_data .get("packages") .and_then(|v| v.as_array()) .cloned() { for (package, versions_mixed) in packages.iter() { let versions = match versions_mixed.as_array() { Some(a) => a.clone(), None => continue, }; for (_v_key, version_mixed) in versions.iter() { let version = match version_mixed.as_array() { Some(a) => a, None => continue, }; let name_str = version .get("name") .and_then(|v| v.as_string()) .unwrap_or("") .to_string(); let version_package_name = strtolower(&name_str); let version_map: IndexMap = version .iter() .map(|(k, v)| (k.clone(), (**v).clone())) .collect(); self.partial_packages_by_name .as_mut() .unwrap() .entry(version_package_name.clone()) .or_insert_with(Vec::new) .push(version_map); if !self.displayed_warning_about_non_matching_package_index && version_package_name != strtolower(package) { self.io.write_error(&format!( "Warning: the packages key '{}' doesn't match the name defined in the package metadata '{}' in repository {}", package, name_str, self.base_url )); self.displayed_warning_about_non_matching_package_index = true; } } } } // wipe rootData as it is fully consumed at this point and this saves some memory self.root_data = Some(RootData::True); Ok(()) } /// Checks if the package name is present in this lazy providers repo /// /// @return true if the package name is present in availablePackages or matched by availablePackagePatterns pub(crate) fn lazy_providers_repo_contains(&self, name: &str) -> anyhow::Result { if !self.has_available_package_list { return Err(LogicException { message: "lazyProvidersRepoContains should not be called unless hasAvailablePackageList is true".to_string(), code: 0, }.into()); } if let Some(ref available) = self.available_packages { if available.contains_key(name) { return Ok(true); } } if let Some(ref patterns) = self.available_package_patterns { for provider_regex in patterns.iter() { if Preg::is_match(provider_regex, name)? { return Ok(true); } } } Ok(false) } } #[derive(Debug)] enum FetchFileIfLastModifiedResult { NotModified, Data(IndexMap), } #[derive(Debug)] struct RetryMarker; impl std::fmt::Display for RetryMarker { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "RetryMarker") } } impl std::error::Error for RetryMarker {} fn clone_root_data(rd: &RootData) -> RootData { match rd { RootData::True => RootData::True, RootData::Data(d) => RootData::Data(d.clone()), } }