Add tar.xz support; framework for more

Cargo.lock

@@ -212,6 +212,16 @@ dependencies = [
  "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "filetime"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.2"
@@ -404,16 +414,18 @@ dependencies = [
  "number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rust-lzma 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "walkdir 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "web-view 0.2.1 (git+https://github.com/Boscop/web-view.git?rev=555f422d09cbb94e82a728d47e9e07ca91963f6e)",
  "winres 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "zip 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "zip 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -794,6 +806,14 @@ dependencies = [
  "uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rust-lzma"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "pkg-config 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "safemem"
 version = "0.2.0"
@@ -936,6 +956,17 @@ name = "take"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "tar"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "tempdir"
 version = "0.3.7"
@@ -1352,9 +1383,17 @@ dependencies = [
  "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "xattr"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "zip"
-version = "0.2.8"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1392,6 +1431,7 @@ dependencies = [
 "checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd"
 "checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d"
 "checksum fern 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "57915fe00a83af935983eb2d00b0ecc62419c4741b28c207ecbf98fd4a1b94c8"
+"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f"
 "checksum flate2 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "37847f133aae7acf82bb9577ccd8bda241df836787642654286e79679826a54b"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
@@ -1456,6 +1496,7 @@ dependencies = [
 "checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a"
 "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5"
 "checksum reqwest 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e7e237e32c3bfa55c95e29af872c8f481471d70b8a5ec15d85f4d274ffd92dd9"
+"checksum rust-lzma 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "032cc29d0d39e6a19bd34b9ef9f23bbcf010d5572d1a4f6b6ed101ba492163f2"
 "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
 "checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637"
 "checksum schannel 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "dc1fabf2a7b6483a141426e1afd09ad543520a77ac49bd03c286e7696ccfd77f"
@@ -1476,6 +1517,7 @@ dependencies = [
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
 "checksum syn 0.14.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e13df71f29f9440b50261a5882c86eac334f1badb3134ec26f0de2f1418e44"
 "checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5"
+"checksum tar 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)" = "e8f41ca4a5689f06998f0247fcb60da6c760f1950cc9df2a10d71575ad0b062a"
 "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
 "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
@@ -1524,4 +1566,5 @@ dependencies = [
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 "checksum winres 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f07dabda4e79413ecac65bc9a2234ad3d85dc49f9d289f868cd9d8611d88f28d"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
-"checksum zip 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "e7341988e4535c60882d5e5f0b7ad0a9a56b080ade8bdb5527cb512f7b2180e0"
+"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
+"checksum zip 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "36b9e08fb518a65cf7e08a1e482573eb87a2f4f8c6619316612a3c1f162fe822"

Cargo.toml

@@ -32,7 +32,9 @@ semver = {version = "0.9.0", features = ["serde"]}
 regex = "0.2"
 
 dirs = "1.0"
-zip = "0.2.8"
+zip = "0.4.2"
+rust-lzma = "0.3"
+tar = "0.4"
 
 log = "0.4"
 fern = "0.5"

build.rs

@@ -15,6 +15,8 @@ use std::io::BufRead;
 use std::io::BufReader;
 use std::io::Write;
 
+use std::env::consts::OS;
+
 const FILES_TO_PREPROCESS: &'static [&'static str] = &["helpers.js", "views.js"];
 
 #[cfg(windows)]
@@ -32,6 +34,21 @@ fn main() {
 
     let output_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
 
+    let os = OS.to_lowercase();
+
+    // Find target config
+    let target_config = PathBuf::from(format!("config.{}.toml", os));
+
+    if !target_config.exists() {
+        panic!(
+            "There is no config file specified for the platform: {:?}. \
+             Create a file named \"config.{}.toml\" in the root directory.",
+            os, os
+        );
+    }
+
+    copy(target_config, output_dir.join("config.toml")).expect("Unable to copy config file");
+
     // Copy files from static/ to build dir
     for entry in WalkDir::new("static") {
         let entry = entry.expect("Unable to read output directory");

config.linux.toml

@@ -1,2 +1,2 @@
 name = "yuzu"
-target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.v1.toml"
+target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.linux.v1.toml"

config.windows.toml

@@ -0,0 +1,2 @@
+name = "yuzu"
+target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.windows.v1.toml"

src/archives/mod.rs

@@ -0,0 +1,106 @@
+//! Provides interfaces to various archives.
+
+use zip::ZipArchive as UpstreamZipArchive;
+
+use tar::Archive as UpstreamTarArchive;
+use tar::EntryType;
+
+use std::io::Cursor;
+use std::io::Read;
+use std::iter::Iterator;
+use std::path::PathBuf;
+
+use lzma::LzmaReader;
+
+pub trait Archive<'a> {
+    /// func: iterator value, max size, file name, file contents
+    fn for_each(
+        &mut self,
+        func: &mut FnMut(usize, Option<usize>, PathBuf, &mut Read) -> Result<(), String>,
+    ) -> Result<(), String>;
+}
+
+struct ZipArchive<'a> {
+    archive: UpstreamZipArchive<Cursor<&'a [u8]>>,
+}
+
+impl<'a> Archive<'a> for ZipArchive<'a> {
+    fn for_each(
+        &mut self,
+        func: &mut FnMut(usize, Option<usize>, PathBuf, &mut Read) -> Result<(), String>,
+    ) -> Result<(), String> {
+        let max = self.archive.len();
+
+        for i in 0..max {
+            let mut archive = self
+                .archive
+                .by_index(i)
+                .map_err(|v| format!("Error while reading from .zip file: {:?}", v))?;
+
+            if archive.name().ends_with('/') || archive.name().ends_with('\\') {
+                continue;
+            }
+
+            func(i, Some(max), archive.sanitized_name(), &mut archive)?;
+        }
+
+        Ok(())
+    }
+}
+
+struct TarArchive<'a> {
+    archive: UpstreamTarArchive<Box<Read + 'a>>,
+}
+
+impl<'a> Archive<'a> for TarArchive<'a> {
+    fn for_each(
+        &mut self,
+        func: &mut FnMut(usize, Option<usize>, PathBuf, &mut Read) -> Result<(), String>,
+    ) -> Result<(), String> {
+        let entries = self
+            .archive
+            .entries()
+            .map_err(|x| format!("Error while reading .tar file: {:?}", x))?;
+
+        for (i, entry) in entries.enumerate() {
+            let mut entry =
+                entry.map_err(|v| format!("Failed to read entry from .tar file: {:?}", v))?;
+
+            if entry.header().entry_type() != EntryType::Regular {
+                continue;
+            }
+
+            let path = entry
+                .path()
+                .map(PathBuf::from)
+                .map_err(|v| format!("Failed to read entry from .tar file: {:?}", v))?;
+
+            func(i, None, path, &mut entry)?;
+        }
+
+        Ok(())
+    }
+}
+
+/// Reads the named archive with an archive implementation.
+pub fn read_archive<'a>(name: &str, data: &'a [u8]) -> Result<Box<Archive<'a> + 'a>, String> {
+    if name.ends_with(".zip") {
+        // Decompress a .zip file
+        let archive = UpstreamZipArchive::new(Cursor::new(data))
+            .map_err(|x| format!("Error while reading .zip file: {:?}", x))?;
+
+        Ok(Box::new(ZipArchive { archive }))
+    } else if name.ends_with(".tar.xz") {
+        // Decompress a .tar.xz file
+        let decompressed_contents: Box<Read + 'a> = Box::new(
+            LzmaReader::new_decompressor(Cursor::new(data))
+                .map_err(|x| format!("Failed to build decompressor: {:?}", x))?,
+        );
+
+        let tar = UpstreamTarArchive::new(decompressed_contents);
+
+        Ok(Box::new(TarArchive { archive: tar }))
+    } else {
+        Err(format!("No decompression handler for {:?}.", name))
+    }
+}

src/main.rs

@@ -29,6 +29,8 @@ extern crate regex;
 extern crate semver;
 
 extern crate dirs;
+extern crate lzma;
+extern crate tar;
 extern crate zip;
 
 extern crate fern;
@@ -39,6 +41,7 @@ extern crate chrono;
 
 extern crate clap;
 
+mod archives;
 mod assets;
 mod config;
 mod http;
@@ -71,7 +74,7 @@ use log::Level;
 
 use config::BaseAttributes;
 
-static RAW_CONFIG: &'static str = include_str!("../config.toml");
+static RAW_CONFIG: &'static str = include_str!(concat!(env!("OUT_DIR"), "/config.toml"));
 
 #[derive(Deserialize, Debug)]
 enum CallbackType {

src/tasks/download_pkg.rs

@@ -80,7 +80,7 @@ impl Task for DownloadPackageTask {
             );
         })?;
 
-        Ok(TaskParamType::FileContents(version, data_storage))
+        Ok(TaskParamType::FileContents(version, file, data_storage))
     }
 
     fn dependencies(&self) -> Vec<Box<Task>> {

src/tasks/install_pkg.rs

@@ -9,17 +9,18 @@ use config::PackageDescription;
 use installer::LocalInstallation;
 
 use std::fs::create_dir_all;
-use std::fs::File;
 use std::io::copy;
-use std::io::Cursor;
 
 use tasks::download_pkg::DownloadPackageTask;
 use tasks::uninstall_pkg::UninstallPackageTask;
 
-use zip::ZipArchive;
-
 use logging::LoggingErrors;
 
+use archives;
+
+use std::fs::OpenOptions;
+use std::path::Path;
+
 pub struct InstallPackageTask {
     pub name: String,
 }
@@ -68,61 +69,79 @@ impl Task for InstallPackageTask {
         }
 
         let data = input.pop().log_expect("Should have input from resolver!");
-        let (file, data) = match data {
+        let (version, file, data) = match data {
-            TaskParamType::FileContents(file, data) => (file, data),
+            TaskParamType::FileContents(version, file, data) => (version, file, data),
             _ => return Err("Unexpected param type to install package".to_string()),
         };
 
-        // TODO: Handle files other then zips
+        let mut archive = archives::read_archive(&file.name, data.as_slice())?;
-        let data_cursor = Cursor::new(data.as_slice());
-        let mut zip = match ZipArchive::new(data_cursor) {
-            Ok(v) => v,
-            Err(v) => return Err(format!("Unable to open .zip file: {:?}", v)),
-        };
 
-        let zip_size = zip.len();
+        archive.for_each(&mut |i, archive_size, filename, mut file| {
+            let string_name = filename
+                .to_str()
+                .ok_or("Unable to get str from file name")?
+                .to_string();
 
-        for i in 0..zip_size {
+            match &archive_size {
-            let mut file = zip.by_index(i).log_expect("Failed to iterate on .zip file");
+                Some(size) => {
-
+                    messenger(
-            messenger(
+                        &format!("Extracting {} ({} of {})", string_name, i + 1, size),
-                &format!("Extracting {} ({} of {})", file.name(), i + 1, zip_size),
+                        (i as f64) / (*size as f64),
-                (i as f64) / (zip_size as f64),
+                    );
-            );
+                }
-
+                _ => {
-            let filename = file.name().replace("\\", "/");
+                    messenger(
+                        &format!("Extracting {} ({} of ??)", string_name, i + 1),
+                        0.0,
+                    );
+                }
+            }
 
             // Ensure that parent directories exist
-            let mut parent_dir = &filename[..];
+            let mut parent_dir: &Path = &filename;
-            while let Some(v) = parent_dir.rfind('/') {
+            while let Some(v) = parent_dir.parent() {
-                parent_dir = &parent_dir[0..v + 1];
+                parent_dir = v;
 
-                if !installed_files.contains(&parent_dir.to_string()) {
+                let string_name = parent_dir
-                    installed_files.push(parent_dir.to_string());
+                    .to_str()
+                    .ok_or("Unable to get str from file name")?
+                    .to_string();
+
+                if string_name.is_empty() {
+                    continue;
+                }
+
+                if !installed_files.contains(&string_name) {
+                    info!("Creating dir: {:?}", string_name);
+                    installed_files.push(string_name);
                 }
 
                 match create_dir_all(path.join(&parent_dir)) {
                     Ok(v) => v,
                     Err(v) => return Err(format!("Unable to create dir: {:?}", v)),
                 }
-
-                parent_dir = &parent_dir[0..v];
             }
 
             // Create target file
             let target_path = path.join(&filename);
 
-            // Check to make sure this isn't a directory
+            info!("Creating file: {:?}", string_name);
-            if filename.ends_with('/') || filename.ends_with('\\') {
+
-                // Directory was already created
+            if !installed_files.contains(&string_name) {
-                continue;
+                installed_files.push(string_name.to_string());
             }
 
-            info!("Creating file: {:?}", target_path);
+            let mut file_metadata = OpenOptions::new();
+            file_metadata.write(true).create_new(true);
 
-            installed_files.push(filename.to_string());
+            #[cfg(unix)]
+            {
+                use std::os::unix::fs::OpenOptionsExt;
 
-            let mut target_file = match File::create(target_path) {
+                file_metadata.mode(0o770);
+            }
+
+            let mut target_file = match file_metadata.open(target_path) {
                 Ok(v) => v,
                 Err(v) => return Err(format!("Unable to open file handle: {:?}", v)),
             };
@@ -132,12 +151,14 @@ impl Task for InstallPackageTask {
                 Ok(v) => v,
                 Err(v) => return Err(format!("Unable to write to file: {:?}", v)),
             };
-        }
+
+            Ok(())
+        })?;
 
         // Save metadata about this package
         context.database.push(LocalInstallation {
             name: package.name.to_owned(),
-            version: file,
+            version,
             files: installed_files,
         });
 

src/tasks/mod.rs

@@ -25,7 +25,7 @@ pub enum TaskParamType {
     /// Metadata about a file
     File(Version, File),
     /// Downloaded contents of a file
-    FileContents(Version, Vec<u8>),
+    FileContents(Version, File, Vec<u8>),
     /// Tells the runtime to break parsing other dependencies
     Break,
 }