Throttle calls to package registry for version resolution

The 'HEAD' call that is done to resolve package revisions from
  unpinned versions is already quite cheap, but it would still be better
  to avoid overloading Github with such calls; especially for users of a
  language-server that would compile on-the-fly very often. Upstream
  packages don't change often so there's no need to constantly check the
  etag.

  So we now keep a local version of etags that we fetched, as well as a
  timestamp from the last time we fetched them so that we only re-fetch
  them if more than an hour has elapsed. This should be fairly resilient
  while still massively improving the UX for people showing up after a
  day and trying to use latest 'main' features.

  This means that we now effectively have two caching levels:

  - In the manifest, we store previously fetched etags.
  - In the filesystem, we have a cache of already downloaded zip archives.

  The first cache is basically invalidated every hour, while the second
  cache is only invalidated when a etag changes. For pinned versions,
  nothing is invalidated as they are considered immutable.

crates/aiken-project/src/deps/downloader.rs

@@ -8,6 +8,7 @@ use reqwest::Client;
 use zip::result::ZipError;
 
 use crate::{
+    deps::manifest::Manifest,
     error::Error,
     package_name::PackageName,
     paths::{self, CacheKey},
@@ -34,28 +35,29 @@ impl<'a> Downloader<'a> {
         event_listener: &T,
         packages: I,
         project_name: &PackageName,
+        manifest: &mut Manifest,
     ) -> Result<Vec<(PackageName, bool)>, Error>
     where
         T: EventListener,
         I: Iterator<Item = &'a Package>,
     {
-        future::try_join_all(
-            packages
-                .filter(|package| project_name != &package.name)
-                .map(|package| self.ensure_package_in_build_directory(event_listener, package)),
-        )
-        .await
+        let mut tasks = vec![];
+
+        for package in packages.filter(|package| project_name != &package.name) {
+            let cache_key =
+                paths::CacheKey::new(&self.http, event_listener, package, manifest).await?;
+            let task = self.ensure_package_in_build_directory(package, cache_key);
+            tasks.push(task);
+        }
+
+        future::try_join_all(tasks).await
     }
 
-    pub async fn ensure_package_in_build_directory<T>(
+    pub async fn ensure_package_in_build_directory(
         &self,
-        event_listener: &T,
         package: &Package,
-    ) -> Result<(PackageName, bool), Error>
-    where
-        T: EventListener,
-    {
-        let cache_key = paths::CacheKey::new(&self.http, event_listener, package).await?;
+        cache_key: CacheKey,
+    ) -> Result<(PackageName, bool), Error> {
         let downloaded = self
             .ensure_package_downloaded(package, &cache_key)
             .await

crates/aiken-project/src/deps/manifest.rs

@@ -1,8 +1,12 @@
-use std::{fs, path::Path};
-
 use aiken_lang::ast::Span;
 use miette::NamedSource;
 use serde::{Deserialize, Serialize};
+use std::{
+    collections::BTreeMap,
+    fs,
+    path::Path,
+    time::{Duration, SystemTime},
+};
 
 use crate::{
     config::{Config, Dependency, Platform},
@@ -16,6 +20,8 @@ use crate::{
 pub struct Manifest {
     pub requirements: Vec<Dependency>,
     pub packages: Vec<Package>,
+    #[serde(default)]
+    pub etags: BTreeMap<String, (SystemTime, String)>,
 }
 
 impl Manifest {
@@ -76,6 +82,34 @@ impl Manifest {
 
         Ok(())
     }
+
+    pub fn lookup_etag(&self, package: &Package) -> Option<String> {
+        match self.etags.get(&etag_key(package)) {
+            None => None,
+            Some((last_fetched, etag)) => {
+                let elapsed = SystemTime::now().duration_since(*last_fetched).unwrap();
+                // Discard any etag older than an hour. So that we throttle call to the package
+                // registry but we ensure a relatively good synchonization of local packages.
+                if elapsed > Duration::from_secs(3600) {
+                    None
+                } else {
+                    Some(etag.clone())
+                }
+            }
+        }
+    }
+
+    pub fn insert_etag(&mut self, package: &Package, etag: String) {
+        self.etags
+            .insert(etag_key(package), (SystemTime::now(), etag));
+    }
+}
+
+fn etag_key(package: &Package) -> String {
+    format!(
+        "{}/{}@{}",
+        package.name.owner, package.name.repo, package.version
+    )
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug)]
@@ -104,6 +138,7 @@ where
             })
             .collect(),
         requirements: config.dependencies.clone(),
+        etags: BTreeMap::new(),
     };
 
     Ok(manifest)