From b06bf610b786a1c3d994f9e6bc25c3cc300dc6f4 Mon Sep 17 00:00:00 2001 From: rvcas Date: Mon, 19 Dec 2022 22:32:09 -0500 Subject: [PATCH] feat: dep downloading now works --- Cargo.lock | 98 ++++++++++++++++++++++++++- crates/project/Cargo.toml | 1 + crates/project/src/deps/downloader.rs | 63 +++++++++++++++-- crates/project/src/deps/manifest.rs | 2 - crates/project/src/error.rs | 19 ++++++ crates/project/src/paths.rs | 2 +- 6 files changed, 172 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9a7e388..493be7fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,7 @@ dependencies = [ "toml", "uplc", "walkdir", + "zip-extract", ] [[package]] @@ -216,7 +217,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.5.4", "object", "rustc-demangle", ] @@ -299,6 +300,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +[[package]] +name = "bzip2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.73" @@ -397,6 +419,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -503,6 +534,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide 0.6.2", +] + [[package]] name = "fnv" version = "1.0.7" @@ -646,7 +687,7 @@ checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1031,6 +1072,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.5" @@ -1039,7 +1089,7 @@ checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.42.0", ] @@ -1947,6 +1997,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2221,6 +2282,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2454,3 +2521,28 @@ name = "yansi" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time", +] + +[[package]] +name = "zip-extract" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5cc0309f6e81ab96c2b43d5e935025f8732c886690be8f78f68e06bad1d274" +dependencies = [ + "log", + "thiserror", + "zip", +] diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml index 35c6812a..5890f65c 100644 --- a/crates/project/Cargo.toml +++ b/crates/project/Cargo.toml @@ -31,3 +31,4 @@ tokio = { version = "1.23.0", features = ["full"] } toml = "0.5.9" uplc = { path = '../uplc', version = "0.0.25" } walkdir = "2.3.2" +zip-extract = "0.1.1" diff --git a/crates/project/src/deps/downloader.rs b/crates/project/src/deps/downloader.rs index 653aadc7..ec15cd45 100644 --- a/crates/project/src/deps/downloader.rs +++ b/crates/project/src/deps/downloader.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{io::Cursor, path::Path}; use futures::future; use reqwest::Client; @@ -41,14 +41,20 @@ impl<'a> Downloader<'a> { &self, package: &Package, ) -> Result { - self.ensure_package_downloaded(package).await - // self.extract_package_from_cache(&package.name, &package.version) + self.ensure_package_downloaded(package).await?; + self.extract_package_from_cache(&package.name, &package.version) + .await } pub async fn ensure_package_downloaded(&self, package: &Package) -> Result { + let packages_cache_path = paths::packages_cache(); let zipball_path = paths::package_cache_zipball(&package.name, &package.version.to_string()); + if !packages_cache_path.exists() { + tokio::fs::create_dir_all(packages_cache_path).await?; + } + if zipball_path.is_file() { return Ok(false); } @@ -58,13 +64,56 @@ impl<'a> Downloader<'a> { package.name.owner, package.name.repo, package.version ); - let response = self.http.get(url).send().await?; - - dbg!(response); + let response = self + .http + .get(url) + .header("User-Agent", "aiken-lang") + .send() + .await? + .bytes() + .await?; // let PackageSource::Github { url } = &package.source; - // tokio::fs::write(&zipball_path, zipball).await?; + tokio::fs::write(&zipball_path, response).await?; + + Ok(true) + } + + pub async fn extract_package_from_cache( + &self, + name: &PackageName, + version: &str, + ) -> Result { + let destination = self.root_path.join(paths::build_deps_package(name)); + + // If the directory already exists then there's nothing for us to do + if destination.is_dir() { + return Ok(false); + } + + tokio::fs::create_dir_all(&destination).await?; + + let zipball_path = self + .root_path + .join(paths::package_cache_zipball(name, version)); + + let zipball = tokio::fs::read(zipball_path).await?; + + let result = { + let d = destination.clone(); + + tokio::task::spawn_blocking(move || { + zip_extract::extract(Cursor::new(zipball), &d, true) + }) + .await? + }; + + if result.is_err() { + tokio::fs::remove_dir_all(destination).await?; + } + + result?; Ok(true) } diff --git a/crates/project/src/deps/manifest.rs b/crates/project/src/deps/manifest.rs index a68b1bec..e0e53ced 100644 --- a/crates/project/src/deps/manifest.rs +++ b/crates/project/src/deps/manifest.rs @@ -81,8 +81,6 @@ impl Manifest { "# This file was generated by Aiken\n# You typically do not need to edit this file\n\n", ); - dbg!(&manifest_path.display()); - fs::write(manifest_path, toml)?; Ok(()) diff --git a/crates/project/src/error.rs b/crates/project/src/error.rs index 8264514b..5d2663a3 100644 --- a/crates/project/src/error.rs +++ b/crates/project/src/error.rs @@ -13,6 +13,7 @@ use std::{ path::{Path, PathBuf}, }; use uplc::machine::cost_model::ExBudget; +use zip_extract::ZipExtractError; #[allow(dead_code)] #[derive(thiserror::Error)] @@ -36,6 +37,12 @@ pub enum Error { #[error(transparent)] Http(#[from] reqwest::Error), + #[error(transparent)] + ZipExtract(#[from] ZipExtractError), + + #[error(transparent)] + JoinError(#[from] tokio::task::JoinError), + #[error("{help}")] TomlLoading { path: PathBuf, @@ -177,6 +184,8 @@ impl Error { Error::WrongValidatorArity { path, .. } => Some(path.to_path_buf()), Error::TestFailure { path, .. } => Some(path.to_path_buf()), Error::Http(_) => None, + Error::ZipExtract(_) => None, + Error::JoinError(_) => None, } } @@ -195,6 +204,8 @@ impl Error { Error::WrongValidatorArity { src, .. } => Some(src.to_string()), Error::TestFailure { .. } => None, Error::Http(_) => None, + Error::ZipExtract(_) => None, + Error::JoinError(_) => None, } } } @@ -238,6 +249,8 @@ impl Diagnostic for Error { Error::WrongValidatorArity { .. } => Some(Box::new("aiken::validators")), Error::TestFailure { path, .. } => Some(Box::new(path.to_str().unwrap_or(""))), Error::Http(_) => Some(Box::new("aiken::deps")), + Error::ZipExtract(_) => None, + Error::JoinError(_) => None, } } @@ -290,6 +303,8 @@ impl Diagnostic for Error { } }, Error::Http(_) => None, + Error::ZipExtract(_) => None, + Error::JoinError(_) => None, } } @@ -320,6 +335,8 @@ impl Diagnostic for Error { )), Error::TestFailure { .. } => None, Error::Http(_) => None, + Error::ZipExtract(_) => None, + Error::JoinError(_) => None, } } @@ -338,6 +355,8 @@ impl Diagnostic for Error { Error::WrongValidatorArity { named, .. } => Some(named), Error::TestFailure { .. } => None, Error::Http(_) => None, + Error::ZipExtract(_) => None, + Error::JoinError(_) => None, } } } diff --git a/crates/project/src/paths.rs b/crates/project/src/paths.rs index 0fd21d07..a5ac9332 100644 --- a/crates/project/src/paths.rs +++ b/crates/project/src/paths.rs @@ -29,7 +29,7 @@ pub fn package_cache_zipball(package_name: &PackageName, version: &str) -> PathB )) } -fn packages_cache() -> PathBuf { +pub fn packages_cache() -> PathBuf { default_aiken_cache().join("packages") }