creation of a downloader

Cargo.lock

@@ -129,6 +129,19 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "console"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "unicode-width",
+ "windows-sys 0.45.0",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -151,6 +164,12 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.32"
@@ -247,6 +266,17 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
 
+[[package]]
+name = "futures-macro"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.28"
@@ -267,6 +297,8 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
 dependencies = [
  "futures-core",
  "futures-io",
+ "futures-macro",
+ "futures-sink",
  "futures-task",
  "memchr",
  "pin-project-lite",
@@ -449,6 +481,19 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.17.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057"
+dependencies = [
+ "console",
+ "instant",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -524,6 +569,12 @@ version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
 
+[[package]]
+name = "md5"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
+
 [[package]]
 name = "memchr"
 version = "2.5.0"
@@ -583,8 +634,11 @@ dependencies = [
  "clap",
  "colored",
  "dotenv",
+ "futures-util",
  "glob",
+ "indicatif",
  "lazy_static",
+ "md5",
  "regex",
  "reqwest",
  "rust-crypto",
@@ -612,6 +666,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "once_cell"
 version = "1.17.2"
@@ -709,6 +769,12 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "portable-atomic"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.59"
@@ -839,10 +905,12 @@ dependencies = [
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-streams",
  "web-sys",
  "winreg",
 ]
@@ -1307,6 +1375,19 @@ version = "0.2.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93"
 
+[[package]]
+name = "wasm-streams"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.63"

Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { version = "0.11", features = ["blocking", "json", "multipart"] }
+reqwest = { version = "0.11", features = ["stream", "json", "multipart"] }
 tokio = { version = "1", features = ["full"] }
 dotenv ="0.15.0" 
 clap = "2.33"
@@ -18,6 +18,9 @@ lazy_static = "1.4.0"
 glob = "0.3.1"
 textwrap = "0.13"
 chrono = "0.4.26"
+indicatif = "0.17.5"
+md5 = "0.7.0"
+futures-util = "0.3.28"
 
 [profile.release]
 debug = true

src/commands/clone.rs

@@ -4,6 +4,7 @@ use std::fs::DirBuilder;
 use std::path::{Path, PathBuf};
 use clap::Values;
 use regex::Regex;
+use crate::services::downloader::Downloader;
 use crate::utils::api::ApiProps;
 use crate::global::global::{DIR_PATH, set_dir_path};
 use crate::services::api::ApiError;
@@ -28,7 +29,7 @@ pub fn clone(remote: Values<'_>) {
     };
     let api_props = ApiProps {
         host: host.clone(),
-        username: username,
+        username,
         root: dist_path_str.to_string(),
     };
 
@@ -58,6 +59,7 @@ pub fn clone(remote: Values<'_>) {
         let res = ReqProps::new()
             .set_request(relative_s.as_str(), &api_props)
             .gethref()
+            .getcontentlength()
             .getlastmodified()
             .send_req_multiple();
 
@@ -104,7 +106,7 @@ pub fn clone(remote: Values<'_>) {
             // add tree
             let path_folder = p.strip_prefix(ref_path.clone()).unwrap();
             let lastmodified = folder.lastmodified.unwrap().timestamp_millis();
-            if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string()) {
+            if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string(), false) {
                 eprintln!("err: saving ref of {} ({})", path_folder.display(), err);
             }
         }
@@ -122,37 +124,18 @@ pub fn clone(remote: Values<'_>) {
         first_iter = false;
     }
 
-    download_files(ref_path.clone(), files, &api_props);
+    let downloader = Downloader::new()
+        .set_api_props(api_props.clone())
+        .set_files(files)
+        .download(ref_path.clone(), Some(&save_blob));
 }
 
-fn download_files(ref_p: PathBuf, files: Vec<ObjProps>, api_props: &ApiProps) {
+fn save_blob(obj: ObjProps) {
-    for obj in files {
+    let relative_s = &obj.clone().relative_s.unwrap();
-        let relative_s = &obj.clone().relative_s.unwrap();
+    let relative_p = PathBuf::from(&relative_s);
-        let res = DownloadFiles::new()
+    let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
-            .set_url(&relative_s, api_props)
+    if let Err(err) = blob::add(relative_p, &lastmodified.to_string(), false) {
-            .save(ref_p.clone());
+        eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
-
-        match res {
-            Ok(()) => {
-                let relative_p = PathBuf::from(&relative_s);
-                let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
-                if let Err(err) = blob::add(relative_p, &lastmodified.to_string()) {
-                    eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
-                }
-            },
-            Err(ApiError::Unexpected(_)) => {
-                eprintln!("err: writing {}", relative_s);
-            },
-            Err(ApiError::IncorrectRequest(err)) => {
-                eprintln!("fatal: {}", err.status());
-                std::process::exit(1);
-            },
-            Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"),
-            Err(ApiError::RequestError(err)) => {
-                eprintln!("fatal: {}", err);
-                std::process::exit(1);
-            }
-        }
     }
 }
 

src/services.rs

@@ -4,3 +4,4 @@ pub mod download_files;
 pub mod req_props;
 pub mod upload_file;
 pub mod delete_path;
+pub mod downloader;

src/services/api.rs

@@ -2,6 +2,7 @@ use std::env;
 use dotenv::dotenv;
 use reqwest::Client;
 use reqwest::RequestBuilder;
+use reqwest::multipart::Form;
 use reqwest::{Response, Error, Method};
 use reqwest::header::{HeaderValue, CONTENT_TYPE, HeaderMap, IntoHeaderName};
 use crate::utils::api::ApiProps;
@@ -31,6 +32,23 @@ impl ApiBuilder {
         }
     }
 
+    pub fn set_url(&mut self, method: Method, url: &str) -> &mut ApiBuilder {
+        let remote = match config::get("remote") {
+            Some(r) => r,
+            None => {
+                eprintln!("fatal: unable to find a remote");
+                std::process::exit(1);
+            }
+        };
+
+        let (host, _, _) = get_url_props(&remote);
+        let mut u = String::from(host);
+        u.push_str(url);
+        self.request = Some(self.client.request(method, u));
+        self
+        
+    }
+
     pub fn build_request(&mut self, method: Method, path: &str) -> &mut ApiBuilder {
         let remote = match config::get("remote") {
             Some(r) => r,
@@ -93,6 +111,20 @@ impl ApiBuilder {
         self
     }
 
+    pub fn set_multipart(&mut self, form: Form) -> &mut ApiBuilder {
+        match self.request.take() {
+            None => {
+                eprintln!("fatal: incorrect request");
+                std::process::exit(1);
+            },
+            Some(req) => {
+                self.request = Some(req.multipart(form));
+                self.set_header(CONTENT_TYPE, HeaderValue::from_static("multipart/related"));
+            }
+        }
+        self
+    }
+
     pub fn set_header<K: IntoHeaderName>(&mut self, key: K, val: HeaderValue) -> &mut ApiBuilder {
         let map = self.headers.get_or_insert(HeaderMap::new());
         map.insert(key, val);
@@ -110,7 +142,6 @@ impl ApiBuilder {
             }
         }
         self
-
     }
 
     pub async fn send(&mut self) -> Result<Response, Error> {
@@ -122,7 +153,8 @@ impl ApiBuilder {
             },
             Some(req) => {
                 if let Some(headers) = &self.headers {
-                    req.headers(headers.clone()).send().await.map_err(Error::from)
+                    req.headers(headers.clone())
+                        .send().await.map_err(Error::from)
                 } else {
                     req.send().await.map_err(Error::from)
                 }

src/services/downloader.rs

@@ -0,0 +1,89 @@
+use std::path::PathBuf;
+use crate::utils::api::ApiProps;
+use crate::services::api::ApiError;
+use crate::services::download_files::DownloadFiles;
+use crate::services::req_props::ObjProps;
+
+const SIZE_TO_STREAM: u64 = 2 * 1024 * 1024;
+
+pub struct Downloader {
+    files: Vec<ObjProps>,
+    should_log: bool,
+    api_props: Option<ApiProps>,
+}
+
+impl Downloader {
+    pub fn new() -> Self {
+        Downloader { 
+            files: vec![],
+            should_log: false,
+            api_props: None,
+        }
+    } 
+
+    pub fn should_log(&mut self) -> &mut Downloader {
+        self.should_log = true;
+        self
+    }
+
+    pub fn set_api_props(&mut self, api_props: ApiProps) -> &mut Downloader {
+        self.api_props = Some(api_props);
+        self
+    }
+
+    pub fn set_files(&mut self, files: Vec<ObjProps>) -> &mut Downloader {
+        self.files = files;
+        self
+    }
+
+    pub fn add_file(&mut self, file: ObjProps) -> &mut Downloader {
+        self.files.push(file);
+        self
+    }
+
+    pub fn download(&mut self, ref_p: PathBuf, callback: Option<&dyn Fn(ObjProps)>) {
+        for file in self.files.clone() {
+            let relative_s = &file.clone().relative_s.unwrap();
+            let mut download = DownloadFiles::new();
+            download.set_url(&relative_s, &self.api_props.clone().unwrap());
+
+            let res = {
+                if let Some(size) = file.contentlength {
+                    if size > SIZE_TO_STREAM {
+                        download.save_stream(ref_p.clone())
+                    } else {
+                        download.save(ref_p.clone())
+                    }
+                } else {
+                    download.save(ref_p.clone())
+                }
+            };
+
+            
+            match res {
+                Ok(()) => {
+                    if let Some(fct) = callback {
+                        fct(file);
+                    }
+                    //let relative_p = PathBuf::from(&relative_s);
+                    //let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
+                    //if let Err(err) = blob::add(relative_p, &lastmodified.to_string()) {
+                    //    eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
+                    //}
+                },
+                Err(ApiError::Unexpected(_)) => {
+                    eprintln!("err: writing {}", relative_s);
+                },
+                Err(ApiError::IncorrectRequest(err)) => {
+                    eprintln!("fatal: {}", err.status());
+                    std::process::exit(1);
+                },
+                Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"),
+                Err(ApiError::RequestError(err)) => {
+                    eprintln!("fatal: {}", err);
+                    std::process::exit(1);
+                }
+            }
+        }
+    }
+}

src/services/req_props.rs

@@ -1,5 +1,4 @@
 use std::io::Cursor;
-use futures_util::StreamExt;
 use chrono::{Utc, DateTime};
 use reqwest::{Method, Response, Error};
 use xml::reader::{EventReader, XmlEvent};

src/services/upload_file.rs

@@ -1,5 +1,5 @@
 use std::fs::File;
-use std::io::{Read};
+use std::io::Read;
 use std::path::PathBuf;
 use reqwest::{Method, Response, Error};
 use crate::services::api::{ApiBuilder, ApiError};