From 09f8ae4444e782625feb5634b00854fdb5246aca Mon Sep 17 00:00:00 2001 From: James Date: Wed, 8 Aug 2018 12:47:32 +1000 Subject: [PATCH] Add tar.xz support; framework for more --- Cargo.lock | 49 +++++++++++++- Cargo.toml | 4 +- build.rs | 17 +++++ config.toml => config.linux.toml | 2 +- config.windows.toml | 2 + src/archives/mod.rs | 106 +++++++++++++++++++++++++++++++ src/main.rs | 5 +- src/tasks/download_pkg.rs | 2 +- src/tasks/install_pkg.rs | 97 +++++++++++++++++----------- src/tasks/mod.rs | 2 +- 10 files changed, 240 insertions(+), 46 deletions(-) rename config.toml => config.linux.toml (68%) create mode 100644 config.windows.toml create mode 100644 src/archives/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 9c6932f..282082b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,16 @@ dependencies = [ "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "filetime" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "flate2" version = "1.0.2" @@ -404,16 +414,18 @@ dependencies = [ "number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-lzma 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "web-view 0.2.1 (git+https://github.com/Boscop/web-view.git?rev=555f422d09cbb94e82a728d47e9e07ca91963f6e)", "winres 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "zip 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "zip 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -794,6 +806,14 @@ dependencies = [ "uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rust-lzma" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pkg-config 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "safemem" version = "0.2.0" @@ -936,6 +956,17 @@ name = "take" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "tar" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tempdir" version = "0.3.7" @@ -1352,9 +1383,17 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "zip" -version = "0.2.8" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1392,6 +1431,7 @@ dependencies = [ "checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd" "checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d" "checksum fern 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "57915fe00a83af935983eb2d00b0ecc62419c4741b28c207ecbf98fd4a1b94c8" +"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f" "checksum flate2 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "37847f133aae7acf82bb9577ccd8bda241df836787642654286e79679826a54b" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" @@ -1456,6 +1496,7 @@ dependencies = [ "checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum reqwest 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e7e237e32c3bfa55c95e29af872c8f481471d70b8a5ec15d85f4d274ffd92dd9" +"checksum rust-lzma 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "032cc29d0d39e6a19bd34b9ef9f23bbcf010d5572d1a4f6b6ed101ba492163f2" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637" "checksum schannel 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "dc1fabf2a7b6483a141426e1afd09ad543520a77ac49bd03c286e7696ccfd77f" @@ -1476,6 +1517,7 @@ dependencies = [ "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum syn 0.14.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e13df71f29f9440b50261a5882c86eac334f1badb3134ec26f0de2f1418e44" "checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" +"checksum tar 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)" = "e8f41ca4a5689f06998f0247fcb60da6c760f1950cc9df2a10d71575ad0b062a" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" @@ -1524,4 +1566,5 @@ dependencies = [ "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum winres 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f07dabda4e79413ecac65bc9a2234ad3d85dc49f9d289f868cd9d8611d88f28d" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -"checksum zip 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "e7341988e4535c60882d5e5f0b7ad0a9a56b080ade8bdb5527cb512f7b2180e0" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +"checksum zip 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "36b9e08fb518a65cf7e08a1e482573eb87a2f4f8c6619316612a3c1f162fe822" diff --git a/Cargo.toml b/Cargo.toml index 7650e4a..90ee281 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,9 @@ semver = {version = "0.9.0", features = ["serde"]} regex = "0.2" dirs = "1.0" -zip = "0.2.8" +zip = "0.4.2" +rust-lzma = "0.3" +tar = "0.4" log = "0.4" fern = "0.5" diff --git a/build.rs b/build.rs index 50fa30b..e63ec4d 100644 --- a/build.rs +++ b/build.rs @@ -15,6 +15,8 @@ use std::io::BufRead; use std::io::BufReader; use std::io::Write; +use std::env::consts::OS; + const FILES_TO_PREPROCESS: &'static [&'static str] = &["helpers.js", "views.js"]; #[cfg(windows)] @@ -32,6 +34,21 @@ fn main() { let output_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let os = OS.to_lowercase(); + + // Find target config + let target_config = PathBuf::from(format!("config.{}.toml", os)); + + if !target_config.exists() { + panic!( + "There is no config file specified for the platform: {:?}. \ + Create a file named \"config.{}.toml\" in the root directory.", + os, os + ); + } + + copy(target_config, output_dir.join("config.toml")).expect("Unable to copy config file"); + // Copy files from static/ to build dir for entry in WalkDir::new("static") { let entry = entry.expect("Unable to read output directory"); diff --git a/config.toml b/config.linux.toml similarity index 68% rename from config.toml rename to config.linux.toml index d633812..481bdcc 100644 --- a/config.toml +++ b/config.linux.toml @@ -1,2 +1,2 @@ name = "yuzu" -target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.v1.toml" +target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.linux.v1.toml" diff --git a/config.windows.toml b/config.windows.toml new file mode 100644 index 0000000..5fe5f24 --- /dev/null +++ b/config.windows.toml @@ -0,0 +1,2 @@ +name = "yuzu" +target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.windows.v1.toml" diff --git a/src/archives/mod.rs b/src/archives/mod.rs new file mode 100644 index 0000000..97955ff --- /dev/null +++ b/src/archives/mod.rs @@ -0,0 +1,106 @@ +//! Provides interfaces to various archives. + +use zip::ZipArchive as UpstreamZipArchive; + +use tar::Archive as UpstreamTarArchive; +use tar::EntryType; + +use std::io::Cursor; +use std::io::Read; +use std::iter::Iterator; +use std::path::PathBuf; + +use lzma::LzmaReader; + +pub trait Archive<'a> { + /// func: iterator value, max size, file name, file contents + fn for_each( + &mut self, + func: &mut FnMut(usize, Option, PathBuf, &mut Read) -> Result<(), String>, + ) -> Result<(), String>; +} + +struct ZipArchive<'a> { + archive: UpstreamZipArchive>, +} + +impl<'a> Archive<'a> for ZipArchive<'a> { + fn for_each( + &mut self, + func: &mut FnMut(usize, Option, PathBuf, &mut Read) -> Result<(), String>, + ) -> Result<(), String> { + let max = self.archive.len(); + + for i in 0..max { + let mut archive = self + .archive + .by_index(i) + .map_err(|v| format!("Error while reading from .zip file: {:?}", v))?; + + if archive.name().ends_with('/') || archive.name().ends_with('\\') { + continue; + } + + func(i, Some(max), archive.sanitized_name(), &mut archive)?; + } + + Ok(()) + } +} + +struct TarArchive<'a> { + archive: UpstreamTarArchive>, +} + +impl<'a> Archive<'a> for TarArchive<'a> { + fn for_each( + &mut self, + func: &mut FnMut(usize, Option, PathBuf, &mut Read) -> Result<(), String>, + ) -> Result<(), String> { + let entries = self + .archive + .entries() + .map_err(|x| format!("Error while reading .tar file: {:?}", x))?; + + for (i, entry) in entries.enumerate() { + let mut entry = + entry.map_err(|v| format!("Failed to read entry from .tar file: {:?}", v))?; + + if entry.header().entry_type() != EntryType::Regular { + continue; + } + + let path = entry + .path() + .map(PathBuf::from) + .map_err(|v| format!("Failed to read entry from .tar file: {:?}", v))?; + + func(i, None, path, &mut entry)?; + } + + Ok(()) + } +} + +/// Reads the named archive with an archive implementation. +pub fn read_archive<'a>(name: &str, data: &'a [u8]) -> Result + 'a>, String> { + if name.ends_with(".zip") { + // Decompress a .zip file + let archive = UpstreamZipArchive::new(Cursor::new(data)) + .map_err(|x| format!("Error while reading .zip file: {:?}", x))?; + + Ok(Box::new(ZipArchive { archive })) + } else if name.ends_with(".tar.xz") { + // Decompress a .tar.xz file + let decompressed_contents: Box = Box::new( + LzmaReader::new_decompressor(Cursor::new(data)) + .map_err(|x| format!("Failed to build decompressor: {:?}", x))?, + ); + + let tar = UpstreamTarArchive::new(decompressed_contents); + + Ok(Box::new(TarArchive { archive: tar })) + } else { + Err(format!("No decompression handler for {:?}.", name)) + } +} diff --git a/src/main.rs b/src/main.rs index 61b9f72..8f080fe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,6 +29,8 @@ extern crate regex; extern crate semver; extern crate dirs; +extern crate lzma; +extern crate tar; extern crate zip; extern crate fern; @@ -39,6 +41,7 @@ extern crate chrono; extern crate clap; +mod archives; mod assets; mod config; mod http; @@ -71,7 +74,7 @@ use log::Level; use config::BaseAttributes; -static RAW_CONFIG: &'static str = include_str!("../config.toml"); +static RAW_CONFIG: &'static str = include_str!(concat!(env!("OUT_DIR"), "/config.toml")); #[derive(Deserialize, Debug)] enum CallbackType { diff --git a/src/tasks/download_pkg.rs b/src/tasks/download_pkg.rs index 4b45627..fad92bd 100644 --- a/src/tasks/download_pkg.rs +++ b/src/tasks/download_pkg.rs @@ -80,7 +80,7 @@ impl Task for DownloadPackageTask { ); })?; - Ok(TaskParamType::FileContents(version, data_storage)) + Ok(TaskParamType::FileContents(version, file, data_storage)) } fn dependencies(&self) -> Vec> { diff --git a/src/tasks/install_pkg.rs b/src/tasks/install_pkg.rs index 591e3c2..5beab6d 100644 --- a/src/tasks/install_pkg.rs +++ b/src/tasks/install_pkg.rs @@ -9,17 +9,18 @@ use config::PackageDescription; use installer::LocalInstallation; use std::fs::create_dir_all; -use std::fs::File; use std::io::copy; -use std::io::Cursor; use tasks::download_pkg::DownloadPackageTask; use tasks::uninstall_pkg::UninstallPackageTask; -use zip::ZipArchive; - use logging::LoggingErrors; +use archives; + +use std::fs::OpenOptions; +use std::path::Path; + pub struct InstallPackageTask { pub name: String, } @@ -68,61 +69,79 @@ impl Task for InstallPackageTask { } let data = input.pop().log_expect("Should have input from resolver!"); - let (file, data) = match data { - TaskParamType::FileContents(file, data) => (file, data), + let (version, file, data) = match data { + TaskParamType::FileContents(version, file, data) => (version, file, data), _ => return Err("Unexpected param type to install package".to_string()), }; - // TODO: Handle files other then zips - let data_cursor = Cursor::new(data.as_slice()); - let mut zip = match ZipArchive::new(data_cursor) { - Ok(v) => v, - Err(v) => return Err(format!("Unable to open .zip file: {:?}", v)), - }; + let mut archive = archives::read_archive(&file.name, data.as_slice())?; - let zip_size = zip.len(); + archive.for_each(&mut |i, archive_size, filename, mut file| { + let string_name = filename + .to_str() + .ok_or("Unable to get str from file name")? + .to_string(); - for i in 0..zip_size { - let mut file = zip.by_index(i).log_expect("Failed to iterate on .zip file"); - - messenger( - &format!("Extracting {} ({} of {})", file.name(), i + 1, zip_size), - (i as f64) / (zip_size as f64), - ); - - let filename = file.name().replace("\\", "/"); + match &archive_size { + Some(size) => { + messenger( + &format!("Extracting {} ({} of {})", string_name, i + 1, size), + (i as f64) / (*size as f64), + ); + } + _ => { + messenger( + &format!("Extracting {} ({} of ??)", string_name, i + 1), + 0.0, + ); + } + } // Ensure that parent directories exist - let mut parent_dir = &filename[..]; - while let Some(v) = parent_dir.rfind('/') { - parent_dir = &parent_dir[0..v + 1]; + let mut parent_dir: &Path = &filename; + while let Some(v) = parent_dir.parent() { + parent_dir = v; - if !installed_files.contains(&parent_dir.to_string()) { - installed_files.push(parent_dir.to_string()); + let string_name = parent_dir + .to_str() + .ok_or("Unable to get str from file name")? + .to_string(); + + if string_name.is_empty() { + continue; + } + + if !installed_files.contains(&string_name) { + info!("Creating dir: {:?}", string_name); + installed_files.push(string_name); } match create_dir_all(path.join(&parent_dir)) { Ok(v) => v, Err(v) => return Err(format!("Unable to create dir: {:?}", v)), } - - parent_dir = &parent_dir[0..v]; } // Create target file let target_path = path.join(&filename); - // Check to make sure this isn't a directory - if filename.ends_with('/') || filename.ends_with('\\') { - // Directory was already created - continue; + info!("Creating file: {:?}", string_name); + + if !installed_files.contains(&string_name) { + installed_files.push(string_name.to_string()); } - info!("Creating file: {:?}", target_path); + let mut file_metadata = OpenOptions::new(); + file_metadata.write(true).create_new(true); - installed_files.push(filename.to_string()); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; - let mut target_file = match File::create(target_path) { + file_metadata.mode(0o770); + } + + let mut target_file = match file_metadata.open(target_path) { Ok(v) => v, Err(v) => return Err(format!("Unable to open file handle: {:?}", v)), }; @@ -132,12 +151,14 @@ impl Task for InstallPackageTask { Ok(v) => v, Err(v) => return Err(format!("Unable to write to file: {:?}", v)), }; - } + + Ok(()) + })?; // Save metadata about this package context.database.push(LocalInstallation { name: package.name.to_owned(), - version: file, + version, files: installed_files, }); diff --git a/src/tasks/mod.rs b/src/tasks/mod.rs index a03bfc5..2022f92 100644 --- a/src/tasks/mod.rs +++ b/src/tasks/mod.rs @@ -25,7 +25,7 @@ pub enum TaskParamType { /// Metadata about a file File(Version, File), /// Downloaded contents of a file - FileContents(Version, Vec), + FileContents(Version, File, Vec), /// Tells the runtime to break parsing other dependencies Break, }