Invalidate cache using etag for deps by branch

Aiken's build system uses an internal global cache system to avoid
  downloading the same packages over and over across projects. However,
  prior to this commit, the cache key would be based of the dependency
  version which can be either:

  - A commit hash
  - A branch or tag name

  However, in the latter case, it means that the very first time we end
  up fetching a dependency will lock its version forever (or until the
  cache is cleared). This was inconvenient.

  This commit changes that so that we use not only a branch name as
  cache key, but additionally, the etag returned by the GitHub API
  server. The etag is part of the HTTP headers, so it can be fetched
  quickly using a simple HEAD request. It changes whenever the content
  behind the endpoint changes -- which happens to be exactly what we
  want. With this, we can quickly check whether an upstream package has
  been updated and download the latest version should users have
  specified a branch name as a version number.

  For example, my current cache now looks as follow:

  ```
   /Users/ktorz/Library/Caches/aiken/packages/
   ├── aiken-lang-stdlib-1cedbe85b7c7e9c4036d63d45cad4ced27b0d50b.zip
   ├── aiken-lang-stdlib-6b482fa00ec37fe936c93155e8c670f32288a686.zip
   ├── aiken-lang-stdlib-7ca9e659688ea88e1cfdc439b6c20c4c7fae9985.zip
   └── aiken-lang-stdlib-main@04eb45df3c77f6611bbdff842a0e311be2c56390f0fa01f020d69c93ff567fe5.zip
  ```

crates/aiken-project/src/deps/downloader.rs

@@ -3,7 +3,11 @@ use std::{io::Cursor, path::Path};
 use futures::future;
 use reqwest::Client;
 
-use crate::{config::PackageName, error::Error, paths};
+use crate::{
+    config::PackageName,
+    error::Error,
+    paths::{self, CacheKey},
+};
 
 use super::manifest::Package;
 
@@ -41,15 +45,20 @@ impl<'a> Downloader<'a> {
         &self,
         package: &Package,
     ) -> Result<bool, Error> {
-        self.ensure_package_downloaded(package).await?;
-        self.extract_package_from_cache(&package.name, &package.version)
+        let cache_key = paths::CacheKey::new(&self.http, package).await?;
+        self.ensure_package_downloaded(package, &cache_key).await?;
+        self.extract_package_from_cache(&package.name, &cache_key)
             .await
     }
 
-    pub async fn ensure_package_downloaded(&self, package: &Package) -> Result<bool, Error> {
+    pub async fn ensure_package_downloaded(
+        &self,
+        package: &Package,
+        cache_key: &CacheKey,
+    ) -> Result<bool, Error> {
         let packages_cache_path = paths::packages_cache();
-        let zipball_path =
-            paths::package_cache_zipball(&package.name, &package.version.to_string());
+
+        let zipball_path = paths::package_cache_zipball(cache_key);
 
         if !packages_cache_path.exists() {
             tokio::fs::create_dir_all(packages_cache_path).await?;
@@ -83,7 +92,7 @@ impl<'a> Downloader<'a> {
     pub async fn extract_package_from_cache(
         &self,
         name: &PackageName,
-        version: &str,
+        cache_key: &CacheKey,
     ) -> Result<bool, Error> {
         let destination = self.root_path.join(paths::build_deps_package(name));
 
@@ -94,9 +103,7 @@ impl<'a> Downloader<'a> {
 
         tokio::fs::create_dir_all(&destination).await?;
 
-        let zipball_path = self
-            .root_path
-            .join(paths::package_cache_zipball(name, version));
+        let zipball_path = self.root_path.join(paths::package_cache_zipball(cache_key));
 
         let zipball = tokio::fs::read(zipball_path).await?;
 

crates/aiken-project/src/deps/manifest.rs

@@ -87,7 +87,7 @@ impl Manifest {
     }
 }
 
-#[derive(Deserialize, Serialize)]
+#[derive(Deserialize, Serialize, Clone)]
 pub struct Package {
     pub name: PackageName,
     pub version: String,

crates/aiken-project/src/error.rs

@@ -1,4 +1,4 @@
-use crate::{pretty, script::EvalHint};
+use crate::{deps::manifest::Package, pretty, script::EvalHint};
 use aiken_lang::{
     ast::{BinOp, Span},
     parser::error::ParseError,
@@ -106,6 +106,14 @@ pub enum Error {
         src: String,
         evaluation_hint: Option<EvalHint>,
     },
+
+    #[error(
+        "I was unable to resolve '{}' for {}/{}",
+        package.version,
+        package.name.owner,
+        package.name.repo
+    )]
+    UnknownPackageVersion { package: Package },
 }
 
 impl Error {
@@ -187,6 +195,7 @@ impl Error {
             Error::Http(_) => None,
             Error::ZipExtract(_) => None,
             Error::JoinError(_) => None,
+            Error::UnknownPackageVersion { .. } => None,
         }
     }
 
@@ -208,6 +217,7 @@ impl Error {
             Error::Http(_) => None,
             Error::ZipExtract(_) => None,
             Error::JoinError(_) => None,
+            Error::UnknownPackageVersion { .. } => None,
         }
     }
 }
@@ -257,6 +267,7 @@ impl Diagnostic for Error {
             Error::Http(_) => Some(Box::new("aiken::deps")),
             Error::ZipExtract(_) => None,
             Error::JoinError(_) => None,
+            Error::UnknownPackageVersion { .. } => Some(Box::new("aiken::deps")),
         }
     }
 
@@ -312,6 +323,7 @@ impl Diagnostic for Error {
             Error::Http(_) => None,
             Error::ZipExtract(_) => None,
             Error::JoinError(_) => None,
+            Error::UnknownPackageVersion{..} => None,
         }
     }
 
@@ -345,6 +357,7 @@ impl Diagnostic for Error {
             Error::Http(_) => None,
             Error::ZipExtract(_) => None,
             Error::JoinError(_) => None,
+            Error::UnknownPackageVersion { .. } => None,
         }
     }
 
@@ -366,6 +379,7 @@ impl Diagnostic for Error {
             Error::Http(_) => None,
             Error::ZipExtract(_) => None,
             Error::JoinError(_) => None,
+            Error::UnknownPackageVersion { .. } => None,
         }
     }
 
@@ -387,6 +401,7 @@ impl Diagnostic for Error {
             Error::Http { .. } => None,
             Error::ZipExtract { .. } => None,
             Error::JoinError { .. } => None,
+            Error::UnknownPackageVersion { .. } => None,
         }
     }
 }

crates/aiken-project/src/paths.rs

@@ -1,7 +1,8 @@
+use crate::deps::manifest::Package;
+use crate::{config::PackageName, error::Error};
+use reqwest::Client;
 use std::path::PathBuf;
 
-use crate::config::PackageName;
-
 pub fn manifest() -> PathBuf {
     PathBuf::from("aiken.lock")
 }
@@ -22,11 +23,8 @@ pub fn build_deps_package(package_name: &PackageName) -> PathBuf {
     packages().join(format!("{}-{}", package_name.owner, package_name.repo))
 }
 
-pub fn package_cache_zipball(package_name: &PackageName, version: &str) -> PathBuf {
-    packages_cache().join(format!(
-        "{}-{}-{}.zip",
-        package_name.owner, package_name.repo, version
-    ))
+pub fn package_cache_zipball(cache_key: &CacheKey) -> PathBuf {
+    packages_cache().join(cache_key.get_key())
 }
 
 pub fn packages_cache() -> PathBuf {
@@ -38,3 +36,46 @@ pub fn default_aiken_cache() -> PathBuf {
         .expect("Failed to determine user cache directory")
         .join("aiken")
 }
+
+pub struct CacheKey {
+    key: String,
+}
+
+impl CacheKey {
+    pub async fn new(http: &Client, package: &Package) -> Result<CacheKey, Error> {
+        let version = match hex::decode(&package.version) {
+            Ok(..) => Ok(package.version.to_string()),
+            Err(..) => {
+                let url = format!(
+                    "https://api.github.com/repos/{}/{}/zipball/{}",
+                    package.name.owner, package.name.repo, package.version
+                );
+                let response = http
+                    .head(url)
+                    .header("User-Agent", "aiken-lang")
+                    .send()
+                    .await?;
+                let etag = response
+                    .headers()
+                    .get("etag")
+                    .ok_or(Error::UnknownPackageVersion {
+                        package: package.clone(),
+                    })?
+                    .to_str()
+                    .unwrap()
+                    .replace('"', "");
+                Ok(format!("main@{etag}"))
+            }
+        };
+        version.map(|version| CacheKey {
+            key: format!(
+                "{}-{}-{}.zip",
+                package.name.owner, package.name.repo, version
+            ),
+        })
+    }
+
+    pub fn get_key(&self) -> &str {
+        self.key.as_ref()
+    }
+}