feat: tons of boilerplate for fetching packages

crates/cli/src/lib.rs

@@ -153,6 +153,27 @@ impl telemetry::EventListener for Terminal {
                     );
                 }
             }
+            telemetry::Event::DownloadingPackage { name: String } => {
+                println!(
+                    "{} {}",
+                    "...Downloading {}".bold().purple(),
+                    name.bright_blue()
+                )
+            }
+            telemetry::Event::PackagesDownloaded { start, count } => {
+                let elapsed = format!("{:.2}s", start.elapsed().as_millis() as f32 / 1000.);
+
+                let msg = match count {
+                    1 => format!("1 package in {}", elapsed),
+                    _ => format!("{} packages in {}", count, elapsed),
+                };
+
+                println!(
+                    "{} {}",
+                    "...Downloaded {}".bold().purple(),
+                    msg.bright_blue()
+                )
+            }
         }
     }
 }

crates/project/Cargo.toml

@@ -11,7 +11,9 @@ authors = ["Lucas Rosa <x@rvcas.dev>", "Kasey White <kwhitemsg@gmail.com>"]
 [dependencies]
 aiken-lang = { path = "../lang", version = "0.0.26" }
 askama = "0.10.5"
+dirs = "4.0.0"
 fslock = "0.2.1"
+futures = "0.3.25"
 hex = "0.4.3"
 ignore = "0.4.18"
 itertools = "0.10.1"
@@ -21,9 +23,11 @@ pallas-traverse = "0.14.0"
 petgraph = "0.6.2"
 pulldown-cmark = { version = "0.8.0", default-features = false }
 regex = "1.6.0"
+reqwest = "0.11.13"
 serde = { version = "1.0.144", features = ["derive"] }
 serde_json = { version = "1.0.85", features = ["preserve_order"] }
 thiserror = "1.0.37"
+tokio = { version = "1.23.0", features = ["full"] }
 toml = "0.5.9"
 uplc = { path = '../uplc', version = "0.0.25" }
 walkdir = "2.3.2"

crates/project/src/config.rs

@@ -1,14 +1,15 @@
 use std::{fmt::Display, fs, io, path::PathBuf};
 
-use serde::Deserialize;
+use serde::{de::Visitor, Deserialize, Serialize};
 
 #[derive(Deserialize)]
 pub struct Config {
-    pub name: String,
+    pub name: PackageName,
     pub version: String,
     #[serde(default)]
     pub description: String,
     pub repository: Option<Repository>,
+    pub dependencies: Vec<Dependency>,
 }
 
 #[derive(Deserialize)]
@@ -18,13 +19,82 @@ pub struct Repository {
     pub platform: Platform,
 }
 
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
+#[serde(rename_all = "lowercase")]
 pub enum Platform {
     Github,
     Gitlab,
     Bitbucket,
 }
 
+#[derive(Deserialize, Serialize)]
+pub struct Dependency {
+    pub name: PackageName,
+    pub version: String,
+    pub source: Platform,
+}
+
+#[derive(PartialEq, Eq, Hash, Clone)]
+pub struct PackageName {
+    pub owner: String,
+    pub repo: String,
+}
+
+impl Display for PackageName {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}/{}", self.owner, self.repo)
+    }
+}
+
+impl Serialize for PackageName {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for PackageName {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        struct PackageNameVisitor;
+
+        impl<'de> Visitor<'de> for PackageNameVisitor {
+            type Value = PackageName;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                formatter
+                    .write_str("a string representing an owner and repo, ex: aiken-lang/stdlib")
+            }
+
+            fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                let mut name = v.split('/');
+
+                let owner = name.next().ok_or_else(|| {
+                    serde::de::Error::invalid_value(serde::de::Unexpected::Str(v), &self)
+                })?;
+
+                let repo = name.next().ok_or_else(|| {
+                    serde::de::Error::invalid_value(serde::de::Unexpected::Str(v), &self)
+                })?;
+
+                Ok(PackageName {
+                    owner: owner.to_string(),
+                    repo: repo.to_string(),
+                })
+            }
+        }
+
+        deserializer.deserialize_str(PackageNameVisitor)
+    }
+}
+
 impl Display for Platform {
     fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::result::Result<(), ::std::fmt::Error> {
         match *self {

crates/project/src/deps.rs

@@ -1,13 +1,23 @@
-use std::path::Path;
+use std::{collections::HashSet, fs, path::Path};
+
+use aiken_lang::ast::Span;
+use miette::NamedSource;
+use serde::{Deserialize, Serialize};
+use tokio::time::Instant;
 
 use crate::{
+    config::{Config, Dependency, PackageName},
     error::Error,
     paths,
     telemetry::{Event, EventListener},
 };
 
-use self::manifest::Manifest;
+use self::{
+    downloader::Downloader,
+    manifest::{Manifest, Package},
+};
 
+pub mod downloader;
 pub mod manifest;
 
 pub enum UseManifest {
@@ -15,11 +25,113 @@ pub enum UseManifest {
     No,
 }
 
+#[derive(Deserialize, Serialize)]
+pub struct LocalPackages {
+    packages: Vec<Dependency>,
+}
+
+impl LocalPackages {
+    pub fn load(root_path: &Path) -> Result<Self, Error> {
+        let path = root_path.join(paths::packages_toml());
+
+        if !path.exists() {
+            return Ok(Self {
+                packages: Vec::new(),
+            });
+        }
+
+        let src = fs::read_to_string(&path)?;
+
+        let result: Self = toml::from_str(&src).map_err(|e| Error::TomlLoading {
+            path: path.clone(),
+            src: src.clone(),
+            named: NamedSource::new(path.display().to_string(), src),
+            // this isn't actually a legit way to get the span
+            location: e.line_col().map(|(line, col)| Span {
+                start: line,
+                end: col,
+            }),
+        })?;
+
+        Ok(result)
+    }
+
+    pub fn save(&self, root_path: &Path) -> Result<(), Error> {
+        let path = root_path.join(paths::packages_toml());
+
+        let toml = toml::to_string(&self).expect("packages.toml serialization");
+
+        fs::write(&path, &toml)?;
+
+        Ok(())
+    }
+
+    fn remove_extra_packages(&self, manifest: &Manifest, root_path: &Path) -> Result<(), Error> {
+        for (package, _version) in self.extra_local_packages(manifest) {
+            let path = root_path.join(paths::build_deps_package(&package));
+
+            if path.exists() {
+                fs::remove_dir_all(&path)?;
+            }
+        }
+
+        Ok(())
+    }
+    pub fn extra_local_packages(&self, manifest: &Manifest) -> Vec<(PackageName, String)> {
+        let manifest_packages: HashSet<_> = manifest
+            .packages
+            .iter()
+            .map(|p| (&p.name, &p.version))
+            .collect();
+
+        self.packages
+            .iter()
+            .filter(|dep| !manifest_packages.contains(&(&dep.name, &dep.version)))
+            .map(|dep| (dep.name.clone(), dep.version.clone()))
+            .collect()
+    }
+
+    pub fn missing_local_packages<'a>(
+        &self,
+        manifest: &'a Manifest,
+        root: &PackageName,
+    ) -> Vec<&'a Package> {
+        manifest
+            .packages
+            .iter()
+            .filter(|p| {
+                &p.name != root
+                    && !matches!(
+                        self.packages.iter().find(|p2| p2.name == p.name),
+                        Some(Dependency { version, .. }) if &p.version == version,
+                    )
+            })
+            .collect()
+    }
+}
+
+impl From<&Manifest> for LocalPackages {
+    fn from(value: &Manifest) -> Self {
+        Self {
+            packages: value
+                .packages
+                .iter()
+                .map(|p| Dependency {
+                    name: p.name.clone(),
+                    version: p.version.clone(),
+                    source: p.source.into(),
+                })
+                .collect(),
+        }
+    }
+}
+
 pub fn download<T>(
     event_listener: &T,
-    new_package: Option<(Vec<String>, bool)>,
+    new_package: Option<Vec<String>>,
     use_manifest: UseManifest,
     root_path: &Path,
+    config: &Config,
 ) -> Result<Manifest, Error>
 where
     T: EventListener,
@@ -37,5 +149,77 @@ where
         build_lock.lock_with_pid().expect("Build locking")
     }
 
-    todo!()
+    let project_name = config.name.clone();
+
+    if let Some(packages) = new_package {
+        for _package in packages {
+            // config.dependencies.push(Dependency {
+            //     name: (),
+            //     version: (),
+            //     source: package.sour,
+            // })
+        }
+    }
+
+    let runtime = tokio::runtime::Runtime::new().expect("Unable to start Tokio");
+
+    let (manifest, changed) = Manifest::load(event_listener)?;
+
+    let local = LocalPackages::load(root_path)?;
+
+    local.remove_extra_packages(&manifest, root_path)?;
+
+    runtime.block_on(fetch_missing_packages(
+        &manifest,
+        &local,
+        project_name,
+        root_path,
+        event_listener,
+    ))?;
+
+    if changed {
+        manifest.save()?;
+    }
+
+    LocalPackages::from(&manifest).save(root_path)?;
+
+    Ok(manifest)
+}
+
+async fn fetch_missing_packages<T>(
+    manifest: &Manifest,
+    local: &LocalPackages,
+    project_name: PackageName,
+    root_path: &Path,
+    event_listener: &T,
+) -> Result<(), Error>
+where
+    T: EventListener,
+{
+    let mut count = 0;
+
+    let mut missing = local
+        .missing_local_packages(manifest, &project_name)
+        .into_iter()
+        .map(|package| {
+            count += 1;
+            package
+        })
+        .peekable();
+
+    if missing.peek().is_some() {
+        let start = Instant::now();
+
+        event_listener.handle_event(Event::DownloadingPackage {
+            name: "packages".to_string(),
+        });
+
+        let downloader = Downloader::new(root_path);
+
+        downloader.download_packages(missing, &project_name).await?;
+
+        event_listener.handle_event(Event::PackagesDownloaded { start, count });
+    }
+
+    Ok(())
 }

crates/project/src/deps/downloader.rs

@@ -0,0 +1,78 @@
+use std::path::Path;
+
+use futures::future;
+use reqwest::Client;
+
+use crate::{config::PackageName, error::Error, paths};
+
+use super::manifest::{Package, PackageSource};
+
+pub struct Downloader<'a> {
+    http: Client,
+    root_path: &'a Path,
+}
+
+impl<'a> Downloader<'a> {
+    pub fn new(root_path: &'a Path) -> Self {
+        Self {
+            http: Client::new(),
+            root_path,
+        }
+    }
+
+    pub async fn download_packages<T>(
+        &self,
+        packages: T,
+        project_name: &PackageName,
+    ) -> Result<(), Error>
+    where
+        T: Iterator<Item = &'a Package>,
+    {
+        let tasks = packages
+            .filter(|package| project_name != &package.name)
+            .map(|package| self.ensure_package_in_build_directory(package));
+
+        let results = future::try_join_all(tasks).await?;
+
+        Ok(())
+    }
+
+    pub async fn ensure_package_in_build_directory(
+        &self,
+        package: &Package,
+    ) -> Result<bool, Error> {
+        self.ensure_package_downloaded(package).await?;
+        self.extract_package_from_cache(&package.name, &package.version)
+    }
+
+    pub async fn ensure_package_downloaded(&self, package: &Package) -> Result<bool, Error> {
+        let zipball_path =
+            paths::package_cache_zipball(&package.name, &package.version.to_string());
+
+        if zipball_path.is_file() {
+            return Ok(false);
+        }
+
+        let url = format!(
+            "https://api.github.com/repos/{}/{}/zipball/{}",
+            package.name.owner, package.name.repo, package.version
+        );
+
+        let response = self.http.get(url).send().await?;
+
+        let PackageSource::Github { url } = &package.source;
+
+        let zipball =
+            hexpm::get_package_tarball_response(response, &outer_checksum.0).map_err(|error| {
+                Error::DownloadPackageError {
+                    package_name: package.name.to_string(),
+                    package_version: package.version.to_string(),
+                    error: error.to_string(),
+                }
+            })?;
+
+        tokio::fs::write(&zipball_path, zipball).await?;
+
+        Ok(true)
+    }
+}

crates/project/src/deps/manifest.rs

@@ -1,17 +1,44 @@
 use std::collections::HashMap;
 
+use crate::{
+    config::{PackageName, Platform},
+    error::Error,
+    telemetry::EventListener,
+};
+
 pub struct Manifest {
     pub requirements: HashMap<String, String>,
     pub packages: Vec<Package>,
 }
 
+impl Manifest {
+    pub fn load<T>(event_listener: &T) -> Result<(Self, bool), Error>
+    where
+        T: EventListener,
+    {
+        todo!()
+    }
+
+    pub fn save(&self) -> Result<(), Error> {
+        todo!()
+    }
+}
+
 pub struct Package {
-    pub name: String,
+    pub name: PackageName,
     pub version: String,
     pub requirements: Vec<String>,
     pub source: PackageSource,
 }
 
 pub enum PackageSource {
-    GitHub { url: String },
+    Github { url: String },
+}
+
+impl From<PackageSource> for Platform {
+    fn from(value: PackageSource) -> Self {
+        match value {
+            PackageSource::Github { .. } => Self::Github,
+        }
+    }
 }

crates/project/src/docs.rs

@@ -192,7 +192,7 @@ fn generate_module(
         documentation: render_markdown(&module.ast.docs.iter().join("\n")),
         modules_prefix: modules_prefix.to_string(),
         modules,
-        project_name: &config.name,
+        project_name: &config.name.to_string(),
         page_title: &format!("{} - {}", module.name, config.name),
         module_name: module.name.clone(),
         project_version: &config.version.to_string(),
@@ -285,8 +285,8 @@ fn generate_readme(
         breadcrumbs: ".",
         modules_prefix: modules_prefix.to_string(),
         modules,
-        project_name: &config.name,
-        page_title: &config.name,
+        project_name: &config.name.to_string(),
+        page_title: &config.name.to_string(),
         project_version: &config.version.to_string(),
         content: render_markdown(&content),
         source,

crates/project/src/error.rs

@@ -33,6 +33,17 @@ pub enum Error {
     #[error(transparent)]
     StandardIo(#[from] io::Error),
 
+    #[error(transparent)]
+    Http(#[from] reqwest::Error),
+
+    #[error("Loading toml")]
+    TomlLoading {
+        path: PathBuf,
+        src: String,
+        named: NamedSource,
+        location: Option<Span>,
+    },
+
     #[error("Cyclical module imports")]
     ImportCycle { modules: Vec<String> },
 
@@ -156,6 +167,7 @@ impl Error {
             Error::FileIo { .. } => None,
             Error::Format { .. } => None,
             Error::StandardIo(_) => None,
+            Error::TomlLoading { path, .. } => Some(path.to_path_buf()),
             Error::ImportCycle { .. } => None,
             Error::List(_) => None,
             Error::Parse { path, .. } => Some(path.to_path_buf()),
@@ -163,6 +175,7 @@ impl Error {
             Error::ValidatorMustReturnBool { path, .. } => Some(path.to_path_buf()),
             Error::WrongValidatorArity { path, .. } => Some(path.to_path_buf()),
             Error::TestFailure { path, .. } => Some(path.to_path_buf()),
+            Error::Http(_) => None,
         }
     }
 
@@ -172,6 +185,7 @@ impl Error {
             Error::FileIo { .. } => None,
             Error::Format { .. } => None,
             Error::StandardIo(_) => None,
+            Error::TomlLoading { src, .. } => Some(src.to_string()),
             Error::ImportCycle { .. } => None,
             Error::List(_) => None,
             Error::Parse { src, .. } => Some(src.to_string()),
@@ -179,6 +193,7 @@ impl Error {
             Error::ValidatorMustReturnBool { src, .. } => Some(src.to_string()),
             Error::WrongValidatorArity { src, .. } => Some(src.to_string()),
             Error::TestFailure { .. } => None,
+            Error::Http(_) => None,
         }
     }
 }
@@ -216,10 +231,12 @@ impl Diagnostic for Error {
             Error::Parse { .. } => Some(Box::new("aiken::parser")),
             Error::Type { .. } => Some(Box::new("aiken::check")),
             Error::StandardIo(_) => None,
+            Error::TomlLoading { .. } => Some(Box::new("aiken::loading::toml")),
             Error::Format { .. } => None,
             Error::ValidatorMustReturnBool { .. } => Some(Box::new("aiken::scripts")),
             Error::WrongValidatorArity { .. } => Some(Box::new("aiken::validators")),
             Error::TestFailure { path, .. } => Some(Box::new(path.to_str().unwrap_or(""))),
+            Error::Http(_) => Some(Box::new("aiken::deps")),
         }
     }
 
@@ -239,6 +256,7 @@ impl Diagnostic for Error {
             Error::Parse { error, .. } => error.kind.help(),
             Error::Type { error, .. } => error.help(),
             Error::StandardIo(_) => None,
+            Error::TomlLoading { .. } => None,
             Error::Format { .. } => None,
             Error::ValidatorMustReturnBool { .. } => Some(Box::new("Try annotating the validator's return type with Bool")),
             Error::WrongValidatorArity { .. } => Some(Box::new("Validators require a minimum number of arguments please add the missing arguments.\nIf you don't need one of the required arguments use an underscore `_datum`.")),
@@ -270,6 +288,7 @@ impl Diagnostic for Error {
                     }
                 }
             },
+            Error::Http(_) => None,
         }
     }
 
@@ -282,6 +301,15 @@ impl Diagnostic for Error {
             Error::Parse { error, .. } => error.labels(),
             Error::Type { error, .. } => error.labels(),
             Error::StandardIo(_) => None,
+            Error::TomlLoading { location, .. } => {
+                if let Some(location) = location {
+                    Some(Box::new(
+                        vec![LabeledSpan::new_with_span(None, *location)].into_iter(),
+                    ))
+                } else {
+                    None
+                }
+            }
             Error::Format { .. } => None,
             Error::ValidatorMustReturnBool { location, .. } => Some(Box::new(
                 vec![LabeledSpan::new_with_span(None, *location)].into_iter(),
@@ -290,6 +318,7 @@ impl Diagnostic for Error {
                 vec![LabeledSpan::new_with_span(None, *location)].into_iter(),
             )),
             Error::TestFailure { .. } => None,
+            Error::Http(_) => None,
         }
     }
 
@@ -302,10 +331,12 @@ impl Diagnostic for Error {
             Error::Parse { named, .. } => Some(named),
             Error::Type { named, .. } => Some(named),
             Error::StandardIo(_) => None,
+            Error::TomlLoading { named, .. } => Some(named),
             Error::Format { .. } => None,
             Error::ValidatorMustReturnBool { named, .. } => Some(named),
             Error::WrongValidatorArity { named, .. } => Some(named),
             Error::TestFailure { .. } => None,
+            Error::Http(_) => None,
         }
     }
 }

crates/project/src/lib.rs

@@ -105,7 +105,7 @@ where
         self.event_listener
             .handle_event(Event::BuildingDocumentation {
                 root: self.root.clone(),
-                name: self.config.name.clone(),
+                name: self.config.name.to_string(),
                 version: self.config.version.clone(),
             });
         self.read_source_files()?;
@@ -151,12 +151,18 @@ where
     }
 
     pub fn compile(&mut self, options: Options) -> Result<(), Error> {
-        let manifest = deps::download(&self.event_listener, None, UseManifest::Yes, &self.root)?;
+        let manifest = deps::download(
+            &self.event_listener,
+            None,
+            UseManifest::Yes,
+            &self.root,
+            &self.config,
+        )?;
 
         self.event_listener
             .handle_event(Event::StartingCompilation {
                 root: self.root.clone(),
-                name: self.config.name.clone(),
+                name: self.config.name.to_string(),
                 version: self.config.version.clone(),
             });
 
@@ -249,7 +255,7 @@ where
                         name,
                         path,
                         extra,
-                        package: self.config.name.clone(),
+                        package: self.config.name.to_string(),
                     };
 
                     if let Some(first) = self
@@ -308,7 +314,7 @@ where
                     .infer(
                         &self.id_gen,
                         kind,
-                        &self.config.name,
+                        &self.config.name.to_string(),
                         &self.module_types,
                         &mut type_warnings,
                     )

crates/project/src/paths.rs

@@ -1,5 +1,36 @@
 use std::path::PathBuf;
 
+use crate::config::PackageName;
+
 pub fn build() -> PathBuf {
     PathBuf::from("build")
 }
+
+pub fn packages() -> PathBuf {
+    build().join("packages")
+}
+
+pub fn packages_toml() -> PathBuf {
+    packages().join("packages.toml")
+}
+
+pub fn build_deps_package(package_name: &PackageName) -> PathBuf {
+    packages().join(format!("{}-{}", package_name.owner, package_name.repo))
+}
+
+pub fn package_cache_zipball(package_name: &PackageName, version: &str) -> PathBuf {
+    packages_cache().join(format!(
+        "{}-{}-{}.zip",
+        package_name.owner, package_name.repo, version
+    ))
+}
+
+fn packages_cache() -> PathBuf {
+    default_aiken_cache().join("packages")
+}
+
+pub fn default_aiken_cache() -> PathBuf {
+    dirs::cache_dir()
+        .expect("Failed to determine user cache directory")
+        .join("aiken")
+}

crates/project/src/telemetry.rs

@@ -36,4 +36,11 @@ pub enum Event {
         tests: Vec<EvalInfo>,
     },
     WaitingForBuildDirLock,
+    DownloadingPackage {
+        name: String,
+    },
+    PackagesDownloaded {
+        start: tokio::time::Instant,
+        count: usize,
+    },
 }

examples/sample/aiken.toml

@@ -1,2 +1,6 @@
 name = "sample"
 version = "0.0.1"
+
+dependencies = [
+  { name = "aiken-lang/stdlib", version = "main", source = "Github" },
+]