From 76ff09ba0e0ca599d380f24526bcf30c1075f37a Mon Sep 17 00:00:00 2001 From: KtorZ Date: Fri, 8 Sep 2023 14:55:08 +0200 Subject: [PATCH] Ensure that version resolution works offline And so, even for unpinned package. In this case, we can't do a HEAD request. So we fallback by looking at what's available in the cache and using the most recently downloaded version from the cache. This is only a best effort as the most recently downloaded one may not be the actual latest. But common, this is a case where (a) someone didn't pin any version, (b) is trying to build on in an offline setup. We could possibly make that edge-case better but, let's see if anyone ever complains about it first. --- crates/aiken-project/src/deps.rs | 2 +- crates/aiken-project/src/deps/downloader.rs | 1 + crates/aiken-project/src/error.rs | 15 +++ crates/aiken-project/src/paths.rs | 139 +++++++++++++++----- 4 files changed, 122 insertions(+), 35 deletions(-) diff --git a/crates/aiken-project/src/deps.rs b/crates/aiken-project/src/deps.rs index 385b950f..049228b5 100644 --- a/crates/aiken-project/src/deps.rs +++ b/crates/aiken-project/src/deps.rs @@ -10,7 +10,7 @@ use crate::{ error::Error, package_name::PackageName, paths, - telemetry::{Event, EventListener}, + telemetry::{DownloadSource, Event, EventListener}, }; use self::{ diff --git a/crates/aiken-project/src/deps/downloader.rs b/crates/aiken-project/src/deps/downloader.rs index 8ff8b8cf..09b29a38 100644 --- a/crates/aiken-project/src/deps/downloader.rs +++ b/crates/aiken-project/src/deps/downloader.rs @@ -11,6 +11,7 @@ use crate::{ error::Error, package_name::PackageName, paths::{self, CacheKey}, + telemetry::EventListener, }; use super::manifest::Package; diff --git a/crates/aiken-project/src/error.rs b/crates/aiken-project/src/error.rs index 9071830d..f657c83b 100644 --- a/crates/aiken-project/src/error.rs +++ b/crates/aiken-project/src/error.rs @@ -107,6 +107,13 @@ pub enum Error { )] UnknownPackageVersion { package: Package }, + #[error( + "I need to resolve a package {}/{}, but couldn't find it.", + package.name.owner, + package.name.repo, + )] + UnableToResolvePackage { package: Package }, + #[error("I couldn't parse the provided stake address.")] MalformedStakeAddress { error: Option, @@ -188,6 +195,7 @@ impl GetSource for Error { Error::ZipExtract(_) => None, Error::JoinError(_) => None, Error::UnknownPackageVersion { .. } => None, + Error::UnableToResolvePackage { .. } => None, Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, @@ -213,6 +221,7 @@ impl GetSource for Error { Error::ZipExtract(_) => None, Error::JoinError(_) => None, Error::UnknownPackageVersion { .. } => None, + Error::UnableToResolvePackage { .. } => None, Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, @@ -247,6 +256,7 @@ impl Diagnostic for Error { Error::ZipExtract(_) => None, Error::JoinError(_) => None, Error::UnknownPackageVersion { .. } => Some(Box::new("aiken::packages::resolve")), + Error::UnableToResolvePackage { .. } => Some(Box::new("aiken::package::download")), Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, @@ -306,6 +316,7 @@ impl Diagnostic for Error { Error::ZipExtract(_) => None, Error::JoinError(_) => None, Error::UnknownPackageVersion{..} => Some(Box::new("Perhaps, double-check the package repository and version?")), + Error::UnableToResolvePackage{..} => Some(Box::new("The network is unavailable and the package isn't in the local cache either. Try connecting to the Internet so I can look it up?")), Error::Json(error) => Some(Box::new(format!("{error}"))), Error::MalformedStakeAddress { error } => Some(Box::new(format!("A stake address must be provided either as a base16-encoded string, or as a bech32-encoded string with the 'stake' or 'stake_test' prefix.{hint}", hint = match error { Some(error) => format!("\n\nHere's the error I encountered: {error}"), @@ -366,6 +377,7 @@ impl Diagnostic for Error { Error::ZipExtract(_) => None, Error::JoinError(_) => None, Error::UnknownPackageVersion { .. } => None, + Error::UnableToResolvePackage { .. } => None, Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, @@ -391,6 +403,7 @@ impl Diagnostic for Error { Error::ZipExtract(_) => None, Error::JoinError(_) => None, Error::UnknownPackageVersion { .. } => None, + Error::UnableToResolvePackage { .. } => None, Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, @@ -416,6 +429,7 @@ impl Diagnostic for Error { Error::ZipExtract { .. } => None, Error::JoinError { .. } => None, Error::UnknownPackageVersion { .. } => None, + Error::UnableToResolvePackage { .. } => None, Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, @@ -441,6 +455,7 @@ impl Diagnostic for Error { Error::ZipExtract { .. } => None, Error::JoinError { .. } => None, Error::UnknownPackageVersion { .. } => None, + Error::UnableToResolvePackage { .. } => None, Error::Json { .. } => None, Error::MalformedStakeAddress { .. } => None, Error::NoValidatorNotFound { .. } => None, diff --git a/crates/aiken-project/src/paths.rs b/crates/aiken-project/src/paths.rs index 675f4e81..61784a03 100644 --- a/crates/aiken-project/src/paths.rs +++ b/crates/aiken-project/src/paths.rs @@ -1,7 +1,12 @@ use crate::deps::manifest::Package; -use crate::{error::Error, package_name::PackageName}; +use crate::{ + error::Error, + package_name::PackageName, + telemetry::{Event, EventListener}, +}; +use regex::Regex; use reqwest::Client; -use std::path::PathBuf; +use std::{fs, path::PathBuf}; pub fn project_config() -> PathBuf { PathBuf::from("aiken.toml") @@ -28,7 +33,7 @@ pub fn build_deps_package(package_name: &PackageName) -> PathBuf { } pub fn package_cache_zipball(cache_key: &CacheKey) -> PathBuf { - packages_cache().join(cache_key.get_key()) + packages_cache().join(format!("{}.zip", cache_key.get_key())) } pub fn packages_cache() -> PathBuf { @@ -47,37 +52,36 @@ pub struct CacheKey { } impl CacheKey { - pub async fn new(http: &Client, package: &Package) -> Result { - let version = match hex::decode(&package.version) { - Ok(..) => Ok(package.version.to_string()), - Err(..) => { - let url = format!( - "https://api.github.com/repos/{}/{}/zipball/{}", - package.name.owner, package.name.repo, package.version - ); - let response = http - .head(url) - .header("User-Agent", "aiken-lang") - .send() - .await?; - let etag = response - .headers() - .get("etag") - .ok_or(Error::UnknownPackageVersion { - package: package.clone(), - })? - .to_str() - .unwrap() - .replace('"', ""); - Ok(format!("main@{etag}")) - } - }; - version.map(|version| CacheKey { - key: format!( - "{}-{}-{}.zip", - package.name.owner, package.name.repo, version - ), - }) + pub async fn new( + http: &Client, + event_listener: &T, + package: &Package, + ) -> Result + where + T: EventListener, + { + Ok(CacheKey::from_package( + package, + if is_git_sha_or_tag(&package.version) { + Ok(package.version.to_string()) + } else { + match new_cache_key_from_network(http, package).await { + Err(_) => { + event_listener.handle_event(Event::PackageResolveFallback { + name: format!("{}", package.name), + }); + new_cache_key_from_cache(package) + } + Ok(cache_key) => Ok(cache_key), + } + }?, + )) + } + + fn from_package(package: &Package, version: String) -> CacheKey { + CacheKey { + key: format!("{}-{}-{}", package.name.owner, package.name.repo, version), + } } pub fn get_key(&self) -> &str { @@ -85,6 +89,73 @@ impl CacheKey { } } +async fn new_cache_key_from_network(http: &Client, package: &Package) -> Result { + let url = format!( + "https://api.github.com/repos/{}/{}/zipball/{}", + package.name.owner, package.name.repo, package.version + ); + let response = http + .head(url) + .header("User-Agent", "aiken-lang") + .send() + .await?; + let etag = response + .headers() + .get("etag") + .ok_or(Error::UnknownPackageVersion { + package: package.clone(), + })? + .to_str() + .unwrap() + .replace('"', ""); + Ok(format!( + "{version}@{etag}", + version = package.version.replace('/', "_") + )) +} + +fn new_cache_key_from_cache(target: &Package) -> Result { + let packages = fs::read_dir(packages_cache())?; + + let prefix = CacheKey::from_package(target, target.version.replace('/', "_")) + .get_key() + .to_string(); + let mut most_recently_modified_date = None; + let mut most_recently_modified = None; + + for pkg in packages { + let entry = pkg.unwrap(); + + let filename = entry + .file_name() + .into_string() + .expect("cache filename are valid utf8 strings"); + + if filename.starts_with(&prefix) { + let last_modified = entry.metadata()?.modified()?; + if Some(last_modified) > most_recently_modified_date { + most_recently_modified_date = Some(last_modified); + most_recently_modified = Some(filename); + } + } + } + + match most_recently_modified { + None => Err(Error::UnableToResolvePackage { + package: target.clone(), + }), + Some(pkg) => Ok(format!( + "{version}{etag}", + version = target.version, + etag = pkg + .strip_prefix(&prefix) + .expect("cache filename starts with a valid version prefix") + .strip_suffix(".zip") + .expect("cache files are all zip archives") + )), + } +} + // Best-effort to assert whether a version refers is a git sha digest or a tag. When it is, we // avoid re-downloading it if it's already fetched. But when it isn't, and thus refer to a branch, // we always re-download it. Note however that the download might be short-circuited by the