From 80d497d47c49dba653ed8be1babe4ae2a3845642 Mon Sep 17 00:00:00 2001 From: grimhilt Date: Thu, 13 Jul 2023 23:36:39 +0200 Subject: [PATCH] creation of a downloader --- Cargo.lock | 81 +++++++++++++++++++++++++++++++++ Cargo.toml | 5 ++- src/commands/clone.rs | 45 ++++++------------- src/services.rs | 1 + src/services/api.rs | 36 ++++++++++++++- src/services/downloader.rs | 89 +++++++++++++++++++++++++++++++++++++ src/services/req_props.rs | 1 - src/services/upload_file.rs | 2 +- 8 files changed, 224 insertions(+), 36 deletions(-) create mode 100644 src/services/downloader.rs diff --git a/Cargo.lock b/Cargo.lock index 779e4f0..0618958 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -151,6 +164,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -247,6 +266,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.28" @@ -267,6 +297,8 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", "futures-io", + "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -449,6 +481,19 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "instant" version = "0.1.12" @@ -524,6 +569,12 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -583,8 +634,11 @@ dependencies = [ "clap", "colored", "dotenv", + "futures-util", "glob", + "indicatif", "lazy_static", + "md5", "regex", "reqwest", "rust-crypto", @@ -612,6 +666,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.17.2" @@ -709,6 +769,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" + [[package]] name = "proc-macro2" version = "1.0.59" @@ -839,10 +905,12 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg", ] @@ -1307,6 +1375,19 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.63" diff --git a/Cargo.toml b/Cargo.toml index 53716c6..f81d6e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -reqwest = { version = "0.11", features = ["blocking", "json", "multipart"] } +reqwest = { version = "0.11", features = ["stream", "json", "multipart"] } tokio = { version = "1", features = ["full"] } dotenv ="0.15.0" clap = "2.33" @@ -18,6 +18,9 @@ lazy_static = "1.4.0" glob = "0.3.1" textwrap = "0.13" chrono = "0.4.26" +indicatif = "0.17.5" +md5 = "0.7.0" +futures-util = "0.3.28" [profile.release] debug = true diff --git a/src/commands/clone.rs b/src/commands/clone.rs index 11843f9..d75e0d6 100644 --- a/src/commands/clone.rs +++ b/src/commands/clone.rs @@ -4,6 +4,7 @@ use std::fs::DirBuilder; use std::path::{Path, PathBuf}; use clap::Values; use regex::Regex; +use crate::services::downloader::Downloader; use crate::utils::api::ApiProps; use crate::global::global::{DIR_PATH, set_dir_path}; use crate::services::api::ApiError; @@ -28,7 +29,7 @@ pub fn clone(remote: Values<'_>) { }; let api_props = ApiProps { host: host.clone(), - username: username, + username, root: dist_path_str.to_string(), }; @@ -58,6 +59,7 @@ pub fn clone(remote: Values<'_>) { let res = ReqProps::new() .set_request(relative_s.as_str(), &api_props) .gethref() + .getcontentlength() .getlastmodified() .send_req_multiple(); @@ -104,7 +106,7 @@ pub fn clone(remote: Values<'_>) { // add tree let path_folder = p.strip_prefix(ref_path.clone()).unwrap(); let lastmodified = folder.lastmodified.unwrap().timestamp_millis(); - if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string()) { + if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string(), false) { eprintln!("err: saving ref of {} ({})", path_folder.display(), err); } } @@ -122,37 +124,18 @@ pub fn clone(remote: Values<'_>) { first_iter = false; } - download_files(ref_path.clone(), files, &api_props); + let downloader = Downloader::new() + .set_api_props(api_props.clone()) + .set_files(files) + .download(ref_path.clone(), Some(&save_blob)); } -fn download_files(ref_p: PathBuf, files: Vec, api_props: &ApiProps) { - for obj in files { - let relative_s = &obj.clone().relative_s.unwrap(); - let res = DownloadFiles::new() - .set_url(&relative_s, api_props) - .save(ref_p.clone()); - - match res { - Ok(()) => { - let relative_p = PathBuf::from(&relative_s); - let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis(); - if let Err(err) = blob::add(relative_p, &lastmodified.to_string()) { - eprintln!("err: saving ref of {} ({})", relative_s.clone(), err); - } - }, - Err(ApiError::Unexpected(_)) => { - eprintln!("err: writing {}", relative_s); - }, - Err(ApiError::IncorrectRequest(err)) => { - eprintln!("fatal: {}", err.status()); - std::process::exit(1); - }, - Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"), - Err(ApiError::RequestError(err)) => { - eprintln!("fatal: {}", err); - std::process::exit(1); - } - } +fn save_blob(obj: ObjProps) { + let relative_s = &obj.clone().relative_s.unwrap(); + let relative_p = PathBuf::from(&relative_s); + let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis(); + if let Err(err) = blob::add(relative_p, &lastmodified.to_string(), false) { + eprintln!("err: saving ref of {} ({})", relative_s.clone(), err); } } diff --git a/src/services.rs b/src/services.rs index 70ecab3..9c85e86 100644 --- a/src/services.rs +++ b/src/services.rs @@ -4,3 +4,4 @@ pub mod download_files; pub mod req_props; pub mod upload_file; pub mod delete_path; +pub mod downloader; diff --git a/src/services/api.rs b/src/services/api.rs index 8182145..39cc8e4 100644 --- a/src/services/api.rs +++ b/src/services/api.rs @@ -2,6 +2,7 @@ use std::env; use dotenv::dotenv; use reqwest::Client; use reqwest::RequestBuilder; +use reqwest::multipart::Form; use reqwest::{Response, Error, Method}; use reqwest::header::{HeaderValue, CONTENT_TYPE, HeaderMap, IntoHeaderName}; use crate::utils::api::ApiProps; @@ -31,6 +32,23 @@ impl ApiBuilder { } } + pub fn set_url(&mut self, method: Method, url: &str) -> &mut ApiBuilder { + let remote = match config::get("remote") { + Some(r) => r, + None => { + eprintln!("fatal: unable to find a remote"); + std::process::exit(1); + } + }; + + let (host, _, _) = get_url_props(&remote); + let mut u = String::from(host); + u.push_str(url); + self.request = Some(self.client.request(method, u)); + self + + } + pub fn build_request(&mut self, method: Method, path: &str) -> &mut ApiBuilder { let remote = match config::get("remote") { Some(r) => r, @@ -93,6 +111,20 @@ impl ApiBuilder { self } + pub fn set_multipart(&mut self, form: Form) -> &mut ApiBuilder { + match self.request.take() { + None => { + eprintln!("fatal: incorrect request"); + std::process::exit(1); + }, + Some(req) => { + self.request = Some(req.multipart(form)); + self.set_header(CONTENT_TYPE, HeaderValue::from_static("multipart/related")); + } + } + self + } + pub fn set_header(&mut self, key: K, val: HeaderValue) -> &mut ApiBuilder { let map = self.headers.get_or_insert(HeaderMap::new()); map.insert(key, val); @@ -110,7 +142,6 @@ impl ApiBuilder { } } self - } pub async fn send(&mut self) -> Result { @@ -122,7 +153,8 @@ impl ApiBuilder { }, Some(req) => { if let Some(headers) = &self.headers { - req.headers(headers.clone()).send().await.map_err(Error::from) + req.headers(headers.clone()) + .send().await.map_err(Error::from) } else { req.send().await.map_err(Error::from) } diff --git a/src/services/downloader.rs b/src/services/downloader.rs new file mode 100644 index 0000000..26da4c4 --- /dev/null +++ b/src/services/downloader.rs @@ -0,0 +1,89 @@ +use std::path::PathBuf; +use crate::utils::api::ApiProps; +use crate::services::api::ApiError; +use crate::services::download_files::DownloadFiles; +use crate::services::req_props::ObjProps; + +const SIZE_TO_STREAM: u64 = 2 * 1024 * 1024; + +pub struct Downloader { + files: Vec, + should_log: bool, + api_props: Option, +} + +impl Downloader { + pub fn new() -> Self { + Downloader { + files: vec![], + should_log: false, + api_props: None, + } + } + + pub fn should_log(&mut self) -> &mut Downloader { + self.should_log = true; + self + } + + pub fn set_api_props(&mut self, api_props: ApiProps) -> &mut Downloader { + self.api_props = Some(api_props); + self + } + + pub fn set_files(&mut self, files: Vec) -> &mut Downloader { + self.files = files; + self + } + + pub fn add_file(&mut self, file: ObjProps) -> &mut Downloader { + self.files.push(file); + self + } + + pub fn download(&mut self, ref_p: PathBuf, callback: Option<&dyn Fn(ObjProps)>) { + for file in self.files.clone() { + let relative_s = &file.clone().relative_s.unwrap(); + let mut download = DownloadFiles::new(); + download.set_url(&relative_s, &self.api_props.clone().unwrap()); + + let res = { + if let Some(size) = file.contentlength { + if size > SIZE_TO_STREAM { + download.save_stream(ref_p.clone()) + } else { + download.save(ref_p.clone()) + } + } else { + download.save(ref_p.clone()) + } + }; + + + match res { + Ok(()) => { + if let Some(fct) = callback { + fct(file); + } + //let relative_p = PathBuf::from(&relative_s); + //let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis(); + //if let Err(err) = blob::add(relative_p, &lastmodified.to_string()) { + // eprintln!("err: saving ref of {} ({})", relative_s.clone(), err); + //} + }, + Err(ApiError::Unexpected(_)) => { + eprintln!("err: writing {}", relative_s); + }, + Err(ApiError::IncorrectRequest(err)) => { + eprintln!("fatal: {}", err.status()); + std::process::exit(1); + }, + Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"), + Err(ApiError::RequestError(err)) => { + eprintln!("fatal: {}", err); + std::process::exit(1); + } + } + } + } +} diff --git a/src/services/req_props.rs b/src/services/req_props.rs index cf73eaa..dbde423 100644 --- a/src/services/req_props.rs +++ b/src/services/req_props.rs @@ -1,5 +1,4 @@ use std::io::Cursor; -use futures_util::StreamExt; use chrono::{Utc, DateTime}; use reqwest::{Method, Response, Error}; use xml::reader::{EventReader, XmlEvent}; diff --git a/src/services/upload_file.rs b/src/services/upload_file.rs index ef3f6e9..fced9a5 100644 --- a/src/services/upload_file.rs +++ b/src/services/upload_file.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::{Read}; +use std::io::Read; use std::path::PathBuf; use reqwest::{Method, Response, Error}; use crate::services::api::{ApiBuilder, ApiError};