Add tar.xz support; framework for more

This commit is contained in:
James 2018-08-08 12:47:32 +10:00 committed by James Lonie
parent 01baceb039
commit 09f8ae4444
10 changed files with 240 additions and 46 deletions

49
Cargo.lock generated
View File

@ -212,6 +212,16 @@ dependencies = [
"log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "filetime"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "flate2"
version = "1.0.2"
@ -404,16 +414,18 @@ dependencies = [
"number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)",
"rust-lzma 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)",
"tar 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"walkdir 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"web-view 0.2.1 (git+https://github.com/Boscop/web-view.git?rev=555f422d09cbb94e82a728d47e9e07ca91963f6e)",
"winres 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"zip 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"zip 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -794,6 +806,14 @@ dependencies = [
"uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rust-lzma"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"pkg-config 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "safemem"
version = "0.2.0"
@ -936,6 +956,17 @@ name = "take"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tar"
version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempdir"
version = "0.3.7"
@ -1352,9 +1383,17 @@ dependencies = [
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "xattr"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "zip"
version = "0.2.8"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1392,6 +1431,7 @@ dependencies = [
"checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd"
"checksum encoding_rs 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98fd0f24d1fb71a4a6b9330c8ca04cbd4e7cc5d846b54ca74ff376bc7c9f798d"
"checksum fern 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "57915fe00a83af935983eb2d00b0ecc62419c4741b28c207ecbf98fd4a1b94c8"
"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f"
"checksum flate2 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "37847f133aae7acf82bb9577ccd8bda241df836787642654286e79679826a54b"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
@ -1456,6 +1496,7 @@ dependencies = [
"checksum relay 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a"
"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5"
"checksum reqwest 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e7e237e32c3bfa55c95e29af872c8f481471d70b8a5ec15d85f4d274ffd92dd9"
"checksum rust-lzma 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "032cc29d0d39e6a19bd34b9ef9f23bbcf010d5572d1a4f6b6ed101ba492163f2"
"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
"checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637"
"checksum schannel 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "dc1fabf2a7b6483a141426e1afd09ad543520a77ac49bd03c286e7696ccfd77f"
@ -1476,6 +1517,7 @@ dependencies = [
"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
"checksum syn 0.14.7 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e13df71f29f9440b50261a5882c86eac334f1badb3134ec26f0de2f1418e44"
"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5"
"checksum tar 0.4.16 (registry+https://github.com/rust-lang/crates.io-index)" = "e8f41ca4a5689f06998f0247fcb60da6c760f1950cc9df2a10d71575ad0b062a"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"
@ -1524,4 +1566,5 @@ dependencies = [
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum winres 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f07dabda4e79413ecac65bc9a2234ad3d85dc49f9d289f868cd9d8611d88f28d"
"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
"checksum zip 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "e7341988e4535c60882d5e5f0b7ad0a9a56b080ade8bdb5527cb512f7b2180e0"
"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
"checksum zip 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "36b9e08fb518a65cf7e08a1e482573eb87a2f4f8c6619316612a3c1f162fe822"

View File

@ -32,7 +32,9 @@ semver = {version = "0.9.0", features = ["serde"]}
regex = "0.2"
dirs = "1.0"
zip = "0.2.8"
zip = "0.4.2"
rust-lzma = "0.3"
tar = "0.4"
log = "0.4"
fern = "0.5"

View File

@ -15,6 +15,8 @@ use std::io::BufRead;
use std::io::BufReader;
use std::io::Write;
use std::env::consts::OS;
const FILES_TO_PREPROCESS: &'static [&'static str] = &["helpers.js", "views.js"];
#[cfg(windows)]
@ -32,6 +34,21 @@ fn main() {
let output_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
let os = OS.to_lowercase();
// Find target config
let target_config = PathBuf::from(format!("config.{}.toml", os));
if !target_config.exists() {
panic!(
"There is no config file specified for the platform: {:?}. \
Create a file named \"config.{}.toml\" in the root directory.",
os, os
);
}
copy(target_config, output_dir.join("config.toml")).expect("Unable to copy config file");
// Copy files from static/ to build dir
for entry in WalkDir::new("static") {
let entry = entry.expect("Unable to read output directory");

View File

@ -1,2 +1,2 @@
name = "yuzu"
target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.v1.toml"
target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.linux.v1.toml"

2
config.windows.toml Normal file
View File

@ -0,0 +1,2 @@
name = "yuzu"
target_url = "https://raw.githubusercontent.com/j-selby/test-installer/master/config.windows.v1.toml"

106
src/archives/mod.rs Normal file
View File

@ -0,0 +1,106 @@
//! Provides interfaces to various archives.
use zip::ZipArchive as UpstreamZipArchive;
use tar::Archive as UpstreamTarArchive;
use tar::EntryType;
use std::io::Cursor;
use std::io::Read;
use std::iter::Iterator;
use std::path::PathBuf;
use lzma::LzmaReader;
pub trait Archive<'a> {
/// func: iterator value, max size, file name, file contents
fn for_each(
&mut self,
func: &mut FnMut(usize, Option<usize>, PathBuf, &mut Read) -> Result<(), String>,
) -> Result<(), String>;
}
struct ZipArchive<'a> {
archive: UpstreamZipArchive<Cursor<&'a [u8]>>,
}
impl<'a> Archive<'a> for ZipArchive<'a> {
fn for_each(
&mut self,
func: &mut FnMut(usize, Option<usize>, PathBuf, &mut Read) -> Result<(), String>,
) -> Result<(), String> {
let max = self.archive.len();
for i in 0..max {
let mut archive = self
.archive
.by_index(i)
.map_err(|v| format!("Error while reading from .zip file: {:?}", v))?;
if archive.name().ends_with('/') || archive.name().ends_with('\\') {
continue;
}
func(i, Some(max), archive.sanitized_name(), &mut archive)?;
}
Ok(())
}
}
struct TarArchive<'a> {
archive: UpstreamTarArchive<Box<Read + 'a>>,
}
impl<'a> Archive<'a> for TarArchive<'a> {
fn for_each(
&mut self,
func: &mut FnMut(usize, Option<usize>, PathBuf, &mut Read) -> Result<(), String>,
) -> Result<(), String> {
let entries = self
.archive
.entries()
.map_err(|x| format!("Error while reading .tar file: {:?}", x))?;
for (i, entry) in entries.enumerate() {
let mut entry =
entry.map_err(|v| format!("Failed to read entry from .tar file: {:?}", v))?;
if entry.header().entry_type() != EntryType::Regular {
continue;
}
let path = entry
.path()
.map(PathBuf::from)
.map_err(|v| format!("Failed to read entry from .tar file: {:?}", v))?;
func(i, None, path, &mut entry)?;
}
Ok(())
}
}
/// Reads the named archive with an archive implementation.
pub fn read_archive<'a>(name: &str, data: &'a [u8]) -> Result<Box<Archive<'a> + 'a>, String> {
if name.ends_with(".zip") {
// Decompress a .zip file
let archive = UpstreamZipArchive::new(Cursor::new(data))
.map_err(|x| format!("Error while reading .zip file: {:?}", x))?;
Ok(Box::new(ZipArchive { archive }))
} else if name.ends_with(".tar.xz") {
// Decompress a .tar.xz file
let decompressed_contents: Box<Read + 'a> = Box::new(
LzmaReader::new_decompressor(Cursor::new(data))
.map_err(|x| format!("Failed to build decompressor: {:?}", x))?,
);
let tar = UpstreamTarArchive::new(decompressed_contents);
Ok(Box::new(TarArchive { archive: tar }))
} else {
Err(format!("No decompression handler for {:?}.", name))
}
}

View File

@ -29,6 +29,8 @@ extern crate regex;
extern crate semver;
extern crate dirs;
extern crate lzma;
extern crate tar;
extern crate zip;
extern crate fern;
@ -39,6 +41,7 @@ extern crate chrono;
extern crate clap;
mod archives;
mod assets;
mod config;
mod http;
@ -71,7 +74,7 @@ use log::Level;
use config::BaseAttributes;
static RAW_CONFIG: &'static str = include_str!("../config.toml");
static RAW_CONFIG: &'static str = include_str!(concat!(env!("OUT_DIR"), "/config.toml"));
#[derive(Deserialize, Debug)]
enum CallbackType {

View File

@ -80,7 +80,7 @@ impl Task for DownloadPackageTask {
);
})?;
Ok(TaskParamType::FileContents(version, data_storage))
Ok(TaskParamType::FileContents(version, file, data_storage))
}
fn dependencies(&self) -> Vec<Box<Task>> {

View File

@ -9,17 +9,18 @@ use config::PackageDescription;
use installer::LocalInstallation;
use std::fs::create_dir_all;
use std::fs::File;
use std::io::copy;
use std::io::Cursor;
use tasks::download_pkg::DownloadPackageTask;
use tasks::uninstall_pkg::UninstallPackageTask;
use zip::ZipArchive;
use logging::LoggingErrors;
use archives;
use std::fs::OpenOptions;
use std::path::Path;
pub struct InstallPackageTask {
pub name: String,
}
@ -68,61 +69,79 @@ impl Task for InstallPackageTask {
}
let data = input.pop().log_expect("Should have input from resolver!");
let (file, data) = match data {
TaskParamType::FileContents(file, data) => (file, data),
let (version, file, data) = match data {
TaskParamType::FileContents(version, file, data) => (version, file, data),
_ => return Err("Unexpected param type to install package".to_string()),
};
// TODO: Handle files other then zips
let data_cursor = Cursor::new(data.as_slice());
let mut zip = match ZipArchive::new(data_cursor) {
Ok(v) => v,
Err(v) => return Err(format!("Unable to open .zip file: {:?}", v)),
};
let mut archive = archives::read_archive(&file.name, data.as_slice())?;
let zip_size = zip.len();
for i in 0..zip_size {
let mut file = zip.by_index(i).log_expect("Failed to iterate on .zip file");
archive.for_each(&mut |i, archive_size, filename, mut file| {
let string_name = filename
.to_str()
.ok_or("Unable to get str from file name")?
.to_string();
match &archive_size {
Some(size) => {
messenger(
&format!("Extracting {} ({} of {})", file.name(), i + 1, zip_size),
(i as f64) / (zip_size as f64),
&format!("Extracting {} ({} of {})", string_name, i + 1, size),
(i as f64) / (*size as f64),
);
let filename = file.name().replace("\\", "/");
}
_ => {
messenger(
&format!("Extracting {} ({} of ??)", string_name, i + 1),
0.0,
);
}
}
// Ensure that parent directories exist
let mut parent_dir = &filename[..];
while let Some(v) = parent_dir.rfind('/') {
parent_dir = &parent_dir[0..v + 1];
let mut parent_dir: &Path = &filename;
while let Some(v) = parent_dir.parent() {
parent_dir = v;
if !installed_files.contains(&parent_dir.to_string()) {
installed_files.push(parent_dir.to_string());
let string_name = parent_dir
.to_str()
.ok_or("Unable to get str from file name")?
.to_string();
if string_name.is_empty() {
continue;
}
if !installed_files.contains(&string_name) {
info!("Creating dir: {:?}", string_name);
installed_files.push(string_name);
}
match create_dir_all(path.join(&parent_dir)) {
Ok(v) => v,
Err(v) => return Err(format!("Unable to create dir: {:?}", v)),
}
parent_dir = &parent_dir[0..v];
}
// Create target file
let target_path = path.join(&filename);
// Check to make sure this isn't a directory
if filename.ends_with('/') || filename.ends_with('\\') {
// Directory was already created
continue;
info!("Creating file: {:?}", string_name);
if !installed_files.contains(&string_name) {
installed_files.push(string_name.to_string());
}
info!("Creating file: {:?}", target_path);
let mut file_metadata = OpenOptions::new();
file_metadata.write(true).create_new(true);
installed_files.push(filename.to_string());
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
let mut target_file = match File::create(target_path) {
file_metadata.mode(0o770);
}
let mut target_file = match file_metadata.open(target_path) {
Ok(v) => v,
Err(v) => return Err(format!("Unable to open file handle: {:?}", v)),
};
@ -132,12 +151,14 @@ impl Task for InstallPackageTask {
Ok(v) => v,
Err(v) => return Err(format!("Unable to write to file: {:?}", v)),
};
}
Ok(())
})?;
// Save metadata about this package
context.database.push(LocalInstallation {
name: package.name.to_owned(),
version: file,
version,
files: installed_files,
});

View File

@ -25,7 +25,7 @@ pub enum TaskParamType {
/// Metadata about a file
File(Version, File),
/// Downloaded contents of a file
FileContents(Version, Vec<u8>),
FileContents(Version, File, Vec<u8>),
/// Tells the runtime to break parsing other dependencies
Break,
}