Compare commits
7 Commits
5bc9ef0035
...
dfd42389f3
Author | SHA1 | Date | |
---|---|---|---|
|
dfd42389f3 | ||
|
80d497d47c | ||
|
70fb733b05 | ||
|
60e0bf76a0 | ||
|
b4cb78c676 | ||
|
0922066baa | ||
|
2d4905f506 |
81
Cargo.lock
generated
81
Cargo.lock
generated
@ -129,6 +129,19 @@ dependencies = [
|
|||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "console"
|
||||||
|
version = "0.15.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
|
||||||
|
dependencies = [
|
||||||
|
"encode_unicode",
|
||||||
|
"lazy_static",
|
||||||
|
"libc",
|
||||||
|
"unicode-width",
|
||||||
|
"windows-sys 0.45.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation"
|
name = "core-foundation"
|
||||||
version = "0.9.3"
|
version = "0.9.3"
|
||||||
@ -151,6 +164,12 @@ version = "0.15.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
|
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "encode_unicode"
|
||||||
|
version = "0.3.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "encoding_rs"
|
name = "encoding_rs"
|
||||||
version = "0.8.32"
|
version = "0.8.32"
|
||||||
@ -247,6 +266,17 @@ version = "0.3.28"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
|
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-macro"
|
||||||
|
version = "0.3.28"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-sink"
|
name = "futures-sink"
|
||||||
version = "0.3.28"
|
version = "0.3.28"
|
||||||
@ -267,6 +297,8 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
|
"futures-macro",
|
||||||
|
"futures-sink",
|
||||||
"futures-task",
|
"futures-task",
|
||||||
"memchr",
|
"memchr",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
@ -449,6 +481,19 @@ dependencies = [
|
|||||||
"hashbrown",
|
"hashbrown",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "indicatif"
|
||||||
|
version = "0.17.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057"
|
||||||
|
dependencies = [
|
||||||
|
"console",
|
||||||
|
"instant",
|
||||||
|
"number_prefix",
|
||||||
|
"portable-atomic",
|
||||||
|
"unicode-width",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "instant"
|
name = "instant"
|
||||||
version = "0.1.12"
|
version = "0.1.12"
|
||||||
@ -524,6 +569,12 @@ version = "0.4.18"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
|
checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "md5"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "memchr"
|
name = "memchr"
|
||||||
version = "2.5.0"
|
version = "2.5.0"
|
||||||
@ -583,8 +634,11 @@ dependencies = [
|
|||||||
"clap",
|
"clap",
|
||||||
"colored",
|
"colored",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
|
"futures-util",
|
||||||
"glob",
|
"glob",
|
||||||
|
"indicatif",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
|
"md5",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"rust-crypto",
|
"rust-crypto",
|
||||||
@ -612,6 +666,12 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "number_prefix"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.17.2"
|
version = "1.17.2"
|
||||||
@ -709,6 +769,12 @@ version = "0.3.27"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
|
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "portable-atomic"
|
||||||
|
version = "1.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro2"
|
name = "proc-macro2"
|
||||||
version = "1.0.59"
|
version = "1.0.59"
|
||||||
@ -839,10 +905,12 @@ dependencies = [
|
|||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-native-tls",
|
"tokio-native-tls",
|
||||||
|
"tokio-util",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
"url",
|
"url",
|
||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
"wasm-bindgen-futures",
|
"wasm-bindgen-futures",
|
||||||
|
"wasm-streams",
|
||||||
"web-sys",
|
"web-sys",
|
||||||
"winreg",
|
"winreg",
|
||||||
]
|
]
|
||||||
@ -1307,6 +1375,19 @@ version = "0.2.86"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93"
|
checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-streams"
|
||||||
|
version = "0.2.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
|
||||||
|
dependencies = [
|
||||||
|
"futures-util",
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"wasm-bindgen-futures",
|
||||||
|
"web-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "web-sys"
|
name = "web-sys"
|
||||||
version = "0.3.63"
|
version = "0.3.63"
|
||||||
|
@ -6,7 +6,7 @@ edition = "2021"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
reqwest = { version = "0.11", features = ["blocking", "json", "multipart"] }
|
reqwest = { version = "0.11", features = ["stream", "json", "multipart"] }
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
dotenv ="0.15.0"
|
dotenv ="0.15.0"
|
||||||
clap = "2.33"
|
clap = "2.33"
|
||||||
@ -18,6 +18,9 @@ lazy_static = "1.4.0"
|
|||||||
glob = "0.3.1"
|
glob = "0.3.1"
|
||||||
textwrap = "0.13"
|
textwrap = "0.13"
|
||||||
chrono = "0.4.26"
|
chrono = "0.4.26"
|
||||||
|
indicatif = "0.17.5"
|
||||||
|
md5 = "0.7.0"
|
||||||
|
futures-util = "0.3.28"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
debug = true
|
debug = true
|
||||||
|
@ -4,11 +4,11 @@ use std::fs::DirBuilder;
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use clap::Values;
|
use clap::Values;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
|
use crate::services::downloader::Downloader;
|
||||||
use crate::utils::api::ApiProps;
|
use crate::utils::api::ApiProps;
|
||||||
use crate::global::global::{DIR_PATH, set_dir_path};
|
use crate::global::global::{DIR_PATH, set_dir_path};
|
||||||
use crate::services::api::ApiError;
|
use crate::services::api::ApiError;
|
||||||
use crate::services::req_props::{ReqProps, ObjProps};
|
use crate::services::req_props::{ReqProps, ObjProps};
|
||||||
use crate::services::download_files::DownloadFiles;
|
|
||||||
use crate::store::object::{tree, blob};
|
use crate::store::object::{tree, blob};
|
||||||
use crate::commands::config;
|
use crate::commands::config;
|
||||||
use crate::commands::init;
|
use crate::commands::init;
|
||||||
@ -28,7 +28,7 @@ pub fn clone(remote: Values<'_>) {
|
|||||||
};
|
};
|
||||||
let api_props = ApiProps {
|
let api_props = ApiProps {
|
||||||
host: host.clone(),
|
host: host.clone(),
|
||||||
username: username,
|
username,
|
||||||
root: dist_path_str.to_string(),
|
root: dist_path_str.to_string(),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -58,6 +58,7 @@ pub fn clone(remote: Values<'_>) {
|
|||||||
let res = ReqProps::new()
|
let res = ReqProps::new()
|
||||||
.set_request(relative_s.as_str(), &api_props)
|
.set_request(relative_s.as_str(), &api_props)
|
||||||
.gethref()
|
.gethref()
|
||||||
|
.getcontentlength()
|
||||||
.getlastmodified()
|
.getlastmodified()
|
||||||
.send_req_multiple();
|
.send_req_multiple();
|
||||||
|
|
||||||
@ -104,7 +105,7 @@ pub fn clone(remote: Values<'_>) {
|
|||||||
// add tree
|
// add tree
|
||||||
let path_folder = p.strip_prefix(ref_path.clone()).unwrap();
|
let path_folder = p.strip_prefix(ref_path.clone()).unwrap();
|
||||||
let lastmodified = folder.lastmodified.unwrap().timestamp_millis();
|
let lastmodified = folder.lastmodified.unwrap().timestamp_millis();
|
||||||
if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string()) {
|
if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string(), false) {
|
||||||
eprintln!("err: saving ref of {} ({})", path_folder.display(), err);
|
eprintln!("err: saving ref of {} ({})", path_folder.display(), err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -122,38 +123,20 @@ pub fn clone(remote: Values<'_>) {
|
|||||||
first_iter = false;
|
first_iter = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
download_files(ref_path.clone(), files, &api_props);
|
let downloader = Downloader::new()
|
||||||
|
.set_api_props(api_props.clone())
|
||||||
|
.set_files(files)
|
||||||
|
.should_log()
|
||||||
|
.download(ref_path.clone(), Some(&save_blob));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn download_files(ref_p: PathBuf, files: Vec<ObjProps>, api_props: &ApiProps) {
|
fn save_blob(obj: ObjProps) {
|
||||||
for obj in files {
|
|
||||||
let relative_s = &obj.clone().relative_s.unwrap();
|
let relative_s = &obj.clone().relative_s.unwrap();
|
||||||
let res = DownloadFiles::new()
|
|
||||||
.set_url(&relative_s, api_props)
|
|
||||||
.save(ref_p.clone());
|
|
||||||
|
|
||||||
match res {
|
|
||||||
Ok(()) => {
|
|
||||||
let relative_p = PathBuf::from(&relative_s);
|
let relative_p = PathBuf::from(&relative_s);
|
||||||
let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
|
let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
|
||||||
if let Err(err) = blob::add(relative_p, &lastmodified.to_string()) {
|
if let Err(err) = blob::add(relative_p, &lastmodified.to_string(), false) {
|
||||||
eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
|
eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
|
||||||
}
|
}
|
||||||
},
|
|
||||||
Err(ApiError::Unexpected(_)) => {
|
|
||||||
eprintln!("err: writing {}", relative_s);
|
|
||||||
},
|
|
||||||
Err(ApiError::IncorrectRequest(err)) => {
|
|
||||||
eprintln!("fatal: {}", err.status());
|
|
||||||
std::process::exit(1);
|
|
||||||
},
|
|
||||||
Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"),
|
|
||||||
Err(ApiError::RequestError(err)) => {
|
|
||||||
eprintln!("fatal: {}", err);
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_url_props(url: &str) -> (String, Option<&str>, &str) {
|
pub fn get_url_props(url: &str) -> (String, Option<&str>, &str) {
|
||||||
|
@ -68,7 +68,7 @@ impl PushChange for New {
|
|||||||
let lastmodified = prop.lastmodified.unwrap().timestamp_millis();
|
let lastmodified = prop.lastmodified.unwrap().timestamp_millis();
|
||||||
|
|
||||||
// update blob
|
// update blob
|
||||||
blob::add(obj.path.clone(), &lastmodified.to_string())?;
|
blob::add(obj.path.clone(), &lastmodified.to_string(), true)?;
|
||||||
|
|
||||||
// remove index
|
// remove index
|
||||||
index::rm_line(obj.path.to_str().unwrap())?;
|
index::rm_line(obj.path.to_str().unwrap())?;
|
||||||
|
@ -74,7 +74,7 @@ impl PushChange for NewDir {
|
|||||||
let lastmodified = prop.lastmodified.unwrap().timestamp_millis();
|
let lastmodified = prop.lastmodified.unwrap().timestamp_millis();
|
||||||
|
|
||||||
// update tree
|
// update tree
|
||||||
tree::add(obj.path.clone(), &lastmodified.to_string())?;
|
tree::add(obj.path.clone(), &lastmodified.to_string(), true)?;
|
||||||
|
|
||||||
// remove index
|
// remove index
|
||||||
index::rm_line(obj.path.to_str().unwrap())?;
|
index::rm_line(obj.path.to_str().unwrap())?;
|
||||||
|
@ -4,3 +4,4 @@ pub mod download_files;
|
|||||||
pub mod req_props;
|
pub mod req_props;
|
||||||
pub mod upload_file;
|
pub mod upload_file;
|
||||||
pub mod delete_path;
|
pub mod delete_path;
|
||||||
|
pub mod downloader;
|
||||||
|
@ -2,6 +2,7 @@ use std::env;
|
|||||||
use dotenv::dotenv;
|
use dotenv::dotenv;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
use reqwest::RequestBuilder;
|
use reqwest::RequestBuilder;
|
||||||
|
use reqwest::multipart::Form;
|
||||||
use reqwest::{Response, Error, Method};
|
use reqwest::{Response, Error, Method};
|
||||||
use reqwest::header::{HeaderValue, CONTENT_TYPE, HeaderMap, IntoHeaderName};
|
use reqwest::header::{HeaderValue, CONTENT_TYPE, HeaderMap, IntoHeaderName};
|
||||||
use crate::utils::api::ApiProps;
|
use crate::utils::api::ApiProps;
|
||||||
@ -31,6 +32,23 @@ impl ApiBuilder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_url(&mut self, method: Method, url: &str) -> &mut ApiBuilder {
|
||||||
|
let remote = match config::get("remote") {
|
||||||
|
Some(r) => r,
|
||||||
|
None => {
|
||||||
|
eprintln!("fatal: unable to find a remote");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let (host, _, _) = get_url_props(&remote);
|
||||||
|
let mut u = String::from(host);
|
||||||
|
u.push_str(url);
|
||||||
|
self.request = Some(self.client.request(method, u));
|
||||||
|
self
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
pub fn build_request(&mut self, method: Method, path: &str) -> &mut ApiBuilder {
|
pub fn build_request(&mut self, method: Method, path: &str) -> &mut ApiBuilder {
|
||||||
let remote = match config::get("remote") {
|
let remote = match config::get("remote") {
|
||||||
Some(r) => r,
|
Some(r) => r,
|
||||||
@ -93,6 +111,20 @@ impl ApiBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_multipart(&mut self, form: Form) -> &mut ApiBuilder {
|
||||||
|
match self.request.take() {
|
||||||
|
None => {
|
||||||
|
eprintln!("fatal: incorrect request");
|
||||||
|
std::process::exit(1);
|
||||||
|
},
|
||||||
|
Some(req) => {
|
||||||
|
self.request = Some(req.multipart(form));
|
||||||
|
self.set_header(CONTENT_TYPE, HeaderValue::from_static("multipart/related"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn set_header<K: IntoHeaderName>(&mut self, key: K, val: HeaderValue) -> &mut ApiBuilder {
|
pub fn set_header<K: IntoHeaderName>(&mut self, key: K, val: HeaderValue) -> &mut ApiBuilder {
|
||||||
let map = self.headers.get_or_insert(HeaderMap::new());
|
let map = self.headers.get_or_insert(HeaderMap::new());
|
||||||
map.insert(key, val);
|
map.insert(key, val);
|
||||||
@ -110,7 +142,6 @@ impl ApiBuilder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
self
|
self
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn send(&mut self) -> Result<Response, Error> {
|
pub async fn send(&mut self) -> Result<Response, Error> {
|
||||||
@ -122,7 +153,8 @@ impl ApiBuilder {
|
|||||||
},
|
},
|
||||||
Some(req) => {
|
Some(req) => {
|
||||||
if let Some(headers) = &self.headers {
|
if let Some(headers) = &self.headers {
|
||||||
req.headers(headers.clone()).send().await.map_err(Error::from)
|
req.headers(headers.clone())
|
||||||
|
.send().await.map_err(Error::from)
|
||||||
} else {
|
} else {
|
||||||
req.send().await.map_err(Error::from)
|
req.send().await.map_err(Error::from)
|
||||||
}
|
}
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use futures_util::StreamExt;
|
||||||
|
use std::fs::File;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
use std::io::{self, Write};
|
use std::io::{self, Write};
|
||||||
use reqwest::{Method, Response, Error};
|
use reqwest::{Method, Response, Error};
|
||||||
@ -20,7 +22,7 @@ impl DownloadFiles {
|
|||||||
|
|
||||||
pub fn set_url(&mut self, relative_ps: &str, api_props: &ApiProps) -> &mut DownloadFiles {
|
pub fn set_url(&mut self, relative_ps: &str, api_props: &ApiProps) -> &mut DownloadFiles {
|
||||||
self.relative_ps = relative_ps.to_string();
|
self.relative_ps = relative_ps.to_string();
|
||||||
self.api_builder.set_req(Method::from_bytes(b"PROPFIND").unwrap(), relative_ps, api_props);
|
self.api_builder.set_req(Method::GET, relative_ps, api_props);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -38,13 +40,40 @@ impl DownloadFiles {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn save_stream(&mut self, ref_p: PathBuf, callback: Option<impl Fn(u64)>) -> Result<(), ApiError> {
|
||||||
|
let abs_p = ref_p.join(PathBuf::from(self.relative_ps.clone()));
|
||||||
|
let mut file = File::create(abs_p).unwrap();
|
||||||
|
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(async {
|
||||||
|
let res = self.send().await.map_err(ApiError::RequestError)?;
|
||||||
|
if res.status().is_success() {
|
||||||
|
let mut stream = res.bytes_stream();
|
||||||
|
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let unwrap_chunk = chunk.unwrap();
|
||||||
|
// save chunk inside file
|
||||||
|
if let Err(err) = file.write_all(&unwrap_chunk) {
|
||||||
|
return Err(ApiError::Unexpected(err.to_string()));
|
||||||
|
} else if let Some(fct) = &callback {
|
||||||
|
// call callback with size of this chunk
|
||||||
|
fct(unwrap_chunk.len().try_into().unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(ApiError::IncorrectRequest(res))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn save(&mut self, ref_p: PathBuf) -> Result<(), ApiError> {
|
pub fn save(&mut self, ref_p: PathBuf) -> Result<(), ApiError> {
|
||||||
tokio::runtime::Runtime::new().unwrap().block_on(async {
|
tokio::runtime::Runtime::new().unwrap().block_on(async {
|
||||||
let p = ref_p.join(PathBuf::from(self.relative_ps.clone()));
|
let p = ref_p.join(PathBuf::from(self.relative_ps.clone()));
|
||||||
let res = self.send().await.map_err(ApiError::RequestError)?;
|
let res = self.send().await.map_err(ApiError::RequestError)?;
|
||||||
if res.status().is_success() {
|
if res.status().is_success() {
|
||||||
let body = res.bytes().await.map_err(ApiError::EmptyError)?;
|
let body = res.bytes().await.map_err(ApiError::EmptyError)?;
|
||||||
match DownloadFiles::write_file(p, &body.to_vec()) {
|
match Self::write_file(p, &body.to_vec()) {
|
||||||
Err(_) => Err(ApiError::Unexpected(String::from(""))),
|
Err(_) => Err(ApiError::Unexpected(String::from(""))),
|
||||||
Ok(_) => Ok(()),
|
Ok(_) => Ok(()),
|
||||||
}
|
}
|
||||||
|
171
src/services/downloader.rs
Normal file
171
src/services/downloader.rs
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
use std::path::PathBuf;
|
||||||
|
use indicatif::{ProgressBar, MultiProgress, ProgressStyle, HumanBytes};
|
||||||
|
|
||||||
|
use crate::utils::api::ApiProps;
|
||||||
|
use crate::services::api::ApiError;
|
||||||
|
use crate::services::download_files::DownloadFiles;
|
||||||
|
use crate::services::req_props::ObjProps;
|
||||||
|
|
||||||
|
const SIZE_TO_STREAM: u64 = 2 * 1024 * 1024;
|
||||||
|
|
||||||
|
pub struct Downloader {
|
||||||
|
files: Vec<ObjProps>,
|
||||||
|
should_log: bool,
|
||||||
|
api_props: Option<ApiProps>,
|
||||||
|
progress_bars: Vec<ProgressBar>,
|
||||||
|
multi_progress: Option<MultiProgress>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Downloader {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Downloader {
|
||||||
|
files: vec![],
|
||||||
|
should_log: false,
|
||||||
|
api_props: None,
|
||||||
|
progress_bars: vec![],
|
||||||
|
multi_progress: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn should_log(&mut self) -> &mut Downloader {
|
||||||
|
self.should_log = true;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_api_props(&mut self, api_props: ApiProps) -> &mut Downloader {
|
||||||
|
self.api_props = Some(api_props);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_files(&mut self, files: Vec<ObjProps>) -> &mut Downloader {
|
||||||
|
self.files = files;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_file(&mut self, file: ObjProps) -> &mut Downloader {
|
||||||
|
self.files.push(file);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn init_log(&mut self, nb_objs: u64, total_size: u64) {
|
||||||
|
self.multi_progress = Some(MultiProgress::new());
|
||||||
|
|
||||||
|
self.progress_bars.push(
|
||||||
|
self.multi_progress
|
||||||
|
.clone()
|
||||||
|
.unwrap()
|
||||||
|
.add(ProgressBar::new(nb_objs).with_message("Objects")));
|
||||||
|
|
||||||
|
let msg = format!("0B/{}", HumanBytes(total_size).to_string());
|
||||||
|
self.progress_bars.push(
|
||||||
|
self.multi_progress
|
||||||
|
.clone()
|
||||||
|
.unwrap()
|
||||||
|
.add(ProgressBar::new(total_size).with_message(msg)));
|
||||||
|
|
||||||
|
self.progress_bars[0].set_style(
|
||||||
|
ProgressStyle::with_template("{_:>10} [{bar:40}] {pos}/{len} {msg}")
|
||||||
|
.unwrap()
|
||||||
|
.progress_chars("=> "));
|
||||||
|
|
||||||
|
self.progress_bars[1].set_style(
|
||||||
|
ProgressStyle::with_template("[{elapsed_precise}] [{bar:40}] {msg}")
|
||||||
|
.unwrap()
|
||||||
|
.progress_chars("=> "));
|
||||||
|
|
||||||
|
self.progress_bars[0].tick();
|
||||||
|
self.progress_bars[1].tick();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_bytes_bar(&self, size: u64) {
|
||||||
|
let bytes_bar = &self.progress_bars[1];
|
||||||
|
bytes_bar.inc(size);
|
||||||
|
let msg = format!(
|
||||||
|
"{}/{}",
|
||||||
|
HumanBytes(bytes_bar.position()).to_string(),
|
||||||
|
HumanBytes(bytes_bar.length().unwrap()).to_string());
|
||||||
|
bytes_bar.set_message(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn download(&mut self, ref_p: PathBuf, callback: Option<&dyn Fn(ObjProps)>) {
|
||||||
|
if self.should_log {
|
||||||
|
let mut total_size = 0;
|
||||||
|
let nb_objs = self.files.len();
|
||||||
|
|
||||||
|
self.files
|
||||||
|
.iter()
|
||||||
|
.for_each(|f|
|
||||||
|
if let Some(size) = f.contentlength {
|
||||||
|
total_size += size
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
self.init_log(nb_objs.try_into().unwrap(), total_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
for file in self.files.clone() {
|
||||||
|
let relative_s = &file.clone().relative_s.unwrap();
|
||||||
|
let mut download = DownloadFiles::new();
|
||||||
|
download.set_url(&relative_s, &self.api_props.clone().unwrap());
|
||||||
|
|
||||||
|
let should_use_stream = {
|
||||||
|
if let Some(size) = file.contentlength {
|
||||||
|
if size > SIZE_TO_STREAM {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// download
|
||||||
|
let res = {
|
||||||
|
if should_use_stream {
|
||||||
|
// todo should increment here
|
||||||
|
download.save_stream(ref_p.clone(), Some(|a| self.update_bytes_bar(a)))
|
||||||
|
} else {
|
||||||
|
download.save(ref_p.clone())
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// deal with error
|
||||||
|
match res {
|
||||||
|
Ok(()) => {
|
||||||
|
if let Some(fct) = callback {
|
||||||
|
fct(file.clone());
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(ApiError::Unexpected(_)) => {
|
||||||
|
eprintln!("err: writing {}", relative_s);
|
||||||
|
},
|
||||||
|
Err(ApiError::IncorrectRequest(err)) => {
|
||||||
|
eprintln!("fatal: {}", err.status());
|
||||||
|
std::process::exit(1);
|
||||||
|
},
|
||||||
|
Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"),
|
||||||
|
Err(ApiError::RequestError(err)) => {
|
||||||
|
eprintln!("fatal: {}", err);
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// increment loading bars
|
||||||
|
if self.should_log {
|
||||||
|
self.progress_bars[0].inc(1); // increment object
|
||||||
|
|
||||||
|
// increment bytes only if
|
||||||
|
// not incremented continuously by stream
|
||||||
|
if !should_use_stream {
|
||||||
|
self.update_bytes_bar(file.contentlength.unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// finish all bars
|
||||||
|
for bar in &self.progress_bars {
|
||||||
|
bar.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -12,6 +12,7 @@ pub struct ObjProps {
|
|||||||
pub href: Option<String>,
|
pub href: Option<String>,
|
||||||
pub relative_s: Option<String>,
|
pub relative_s: Option<String>,
|
||||||
pub lastmodified: Option<DateTime<Utc>>,
|
pub lastmodified: Option<DateTime<Utc>>,
|
||||||
|
pub contentlength: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Clone for ObjProps {
|
impl Clone for ObjProps {
|
||||||
@ -20,6 +21,7 @@ impl Clone for ObjProps {
|
|||||||
href: self.href.clone(),
|
href: self.href.clone(),
|
||||||
relative_s: self.relative_s.clone(),
|
relative_s: self.relative_s.clone(),
|
||||||
lastmodified: self.lastmodified.clone(),
|
lastmodified: self.lastmodified.clone(),
|
||||||
|
contentlength: self.contentlength.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -30,6 +32,7 @@ impl ObjProps {
|
|||||||
href: None,
|
href: None,
|
||||||
relative_s: None,
|
relative_s: None,
|
||||||
lastmodified: None,
|
lastmodified: None,
|
||||||
|
contentlength: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -73,7 +76,7 @@ impl ReqProps {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn _getcontentlenght(&mut self) -> &mut ReqProps {
|
pub fn getcontentlength(&mut self) -> &mut ReqProps {
|
||||||
self.xml_balises.push(String::from("getcontentlength"));
|
self.xml_balises.push(String::from("getcontentlength"));
|
||||||
self.xml_payload.push_str(r#"<d:getcontentlength/>"#);
|
self.xml_payload.push_str(r#"<d:getcontentlength/>"#);
|
||||||
self
|
self
|
||||||
@ -122,19 +125,22 @@ impl ReqProps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn send_with_err(&mut self) -> Result<String, ApiError> {
|
pub fn send_with_err(&mut self) -> Result<String, ApiError> {
|
||||||
let res = tokio::runtime::Runtime::new().unwrap().block_on(async {
|
tokio::runtime::Runtime::new().unwrap().block_on(async {
|
||||||
self.send().await
|
match self.send().await {
|
||||||
}).map_err(ApiError::RequestError)?;
|
Err(res) => Err(ApiError::RequestError(res)),
|
||||||
|
Ok(res) if res.status().is_success() => {
|
||||||
if res.status().is_success() {
|
let body = res
|
||||||
let body = tokio::runtime::Runtime::new().unwrap().block_on(async {
|
.text()
|
||||||
res.text().await
|
.await
|
||||||
}).map_err(ApiError::EmptyError)?;
|
.map_err(|err| ApiError::EmptyError(err))?;
|
||||||
Ok(body)
|
Ok(body)
|
||||||
} else {
|
},
|
||||||
|
Ok(res) => {
|
||||||
Err(ApiError::IncorrectRequest(res))
|
Err(ApiError::IncorrectRequest(res))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn send_req_multiple(&mut self) -> Result<Vec<ObjProps>, ApiError> {
|
pub fn send_req_multiple(&mut self) -> Result<Vec<ObjProps>, ApiError> {
|
||||||
match self.send_with_err() {
|
match self.send_with_err() {
|
||||||
@ -160,54 +166,60 @@ impl ReqProps {
|
|||||||
let cursor = Cursor::new(xml);
|
let cursor = Cursor::new(xml);
|
||||||
let parser = EventReader::new(cursor);
|
let parser = EventReader::new(cursor);
|
||||||
|
|
||||||
let mut should_get = false;
|
|
||||||
let mut values: Vec<ObjProps> = vec![];
|
let mut values: Vec<ObjProps> = vec![];
|
||||||
|
|
||||||
let mut iter = self.xml_balises.iter();
|
let mut should_get = false;
|
||||||
let mut val = iter.next();
|
let mut val: String = String::from("");
|
||||||
let mut content = ObjProps::new();
|
let mut content = ObjProps::new();
|
||||||
|
|
||||||
for event in parser {
|
for event in parser {
|
||||||
match event {
|
match event {
|
||||||
Ok(XmlEvent::StartElement { name, .. }) => {
|
Ok(XmlEvent::StartElement { name, .. }) => {
|
||||||
if let Some(v) = val.clone() {
|
should_get = {
|
||||||
should_get = &name.local_name == v;
|
if self.xml_balises.clone().contains(&name.local_name) {
|
||||||
|
val = name.local_name.clone();
|
||||||
|
true
|
||||||
} else {
|
} else {
|
||||||
// end of balises to get then start over for
|
false
|
||||||
// next object if want multiple
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Ok(XmlEvent::Characters(text)) => {
|
||||||
|
if !text.trim().is_empty() && should_get {
|
||||||
|
match val.as_str() {
|
||||||
|
"href" => {
|
||||||
|
content.href = Some(text.clone());
|
||||||
|
content.relative_s = Some(
|
||||||
|
get_relative_s(text, &(self.api_props
|
||||||
|
.clone()
|
||||||
|
.unwrap())));
|
||||||
|
},
|
||||||
|
"getlastmodified" => {
|
||||||
|
content.lastmodified = Some(
|
||||||
|
parse_timestamp(&text).unwrap());
|
||||||
|
},
|
||||||
|
"getcontentlength" => {
|
||||||
|
content.contentlength = Some(
|
||||||
|
text.clone().parse().unwrap());
|
||||||
|
},
|
||||||
|
_ => (),
|
||||||
|
}
|
||||||
|
should_get = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(XmlEvent::EndElement { name, .. }) => {
|
||||||
|
if name.local_name == "response" {
|
||||||
values.push(content.clone());
|
values.push(content.clone());
|
||||||
if multiple {
|
if multiple {
|
||||||
iter = self.xml_balises.iter();
|
|
||||||
val = iter.next();
|
|
||||||
content = ObjProps::new();
|
content = ObjProps::new();
|
||||||
if let Some(v) = val.clone() {
|
|
||||||
should_get = &name.local_name == v;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Ok(XmlEvent::Characters(text)) => {
|
|
||||||
if !text.trim().is_empty() && should_get {
|
|
||||||
match val.unwrap().as_str() {
|
|
||||||
"href" => {
|
|
||||||
content.href = Some(text.clone());
|
|
||||||
content.relative_s = Some(get_relative_s(text, &(self.api_props.clone().unwrap())));
|
|
||||||
},
|
|
||||||
"getlastmodified" => {
|
|
||||||
content.lastmodified = Some(parse_timestamp(&text).unwrap());
|
|
||||||
},
|
|
||||||
_ => (),
|
|
||||||
}
|
|
||||||
val = iter.next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(XmlEvent::EndElement { .. }) => {
|
|
||||||
should_get = false;
|
should_get = false;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("Error: {}", e);
|
eprintln!("err: parsing xml: {}", e);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{Read};
|
use std::io::Read;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use reqwest::{Method, Response, Error};
|
use reqwest::{Method, Response, Error};
|
||||||
use crate::services::api::{ApiBuilder, ApiError};
|
use crate::services::api::{ApiBuilder, ApiError};
|
||||||
|
@ -5,7 +5,7 @@ use crate::utils::path;
|
|||||||
use crate::store::head;
|
use crate::store::head;
|
||||||
use crate::store::object::{update_dates, parse_path, add_node, create_obj, rm_node};
|
use crate::store::object::{update_dates, parse_path, add_node, create_obj, rm_node};
|
||||||
|
|
||||||
pub fn add(path: PathBuf, date: &str) -> io::Result<()> {
|
pub fn add(path: PathBuf, date: &str, up_parent: bool) -> io::Result<()> {
|
||||||
let (line, hash, name) = parse_path(path.clone(), true);
|
let (line, hash, name) = parse_path(path.clone(), true);
|
||||||
// add blob reference to parent
|
// add blob reference to parent
|
||||||
if path.iter().count() == 1 {
|
if path.iter().count() == 1 {
|
||||||
@ -22,7 +22,9 @@ pub fn add(path: PathBuf, date: &str) -> io::Result<()> {
|
|||||||
create_obj(hash, &content)?;
|
create_obj(hash, &content)?;
|
||||||
|
|
||||||
// update date for all parent
|
// update date for all parent
|
||||||
|
if up_parent {
|
||||||
update_dates(path, date)?;
|
update_dates(path, date)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
@ -5,7 +5,7 @@ use crate::utils::{read, path};
|
|||||||
use crate::store::head;
|
use crate::store::head;
|
||||||
use crate::store::object::{self, update_dates, parse_path, hash_obj, add_node, create_obj};
|
use crate::store::object::{self, update_dates, parse_path, hash_obj, add_node, create_obj};
|
||||||
|
|
||||||
pub fn add(path: PathBuf, date: &str) -> io::Result<()> {
|
pub fn add(path: PathBuf, date: &str, up_parent: bool) -> io::Result<()> {
|
||||||
let (line, hash, name) = parse_path(path.clone(), false);
|
let (line, hash, name) = parse_path(path.clone(), false);
|
||||||
|
|
||||||
// add tree reference to parent
|
// add tree reference to parent
|
||||||
@ -22,7 +22,9 @@ pub fn add(path: PathBuf, date: &str) -> io::Result<()> {
|
|||||||
create_obj(hash, &content)?;
|
create_obj(hash, &content)?;
|
||||||
|
|
||||||
// update date for all parent
|
// update date for all parent
|
||||||
|
if up_parent {
|
||||||
update_dates(path, date)?;
|
update_dates(path, date)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user