Compare commits

..

No commits in common. "dfd42389f3fdaa15a0cb56c4db0d3212c15b0f5b" and "5bc9ef0035bd736f1884ee447427bbc8939b16fc" have entirely different histories.

13 changed files with 81 additions and 397 deletions

81
Cargo.lock generated
View File

@ -129,19 +129,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "console"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"unicode-width",
"windows-sys 0.45.0",
]
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -164,12 +151,6 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.32"
@ -266,17 +247,6 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
[[package]]
name = "futures-macro"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.28"
@ -297,8 +267,6 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
dependencies = [
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
@ -481,19 +449,6 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "indicatif"
version = "0.17.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057"
dependencies = [
"console",
"instant",
"number_prefix",
"portable-atomic",
"unicode-width",
]
[[package]]
name = "instant"
version = "0.1.12"
@ -569,12 +524,6 @@ version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.5.0"
@ -634,11 +583,8 @@ dependencies = [
"clap",
"colored",
"dotenv",
"futures-util",
"glob",
"indicatif",
"lazy_static",
"md5",
"regex",
"reqwest",
"rust-crypto",
@ -666,12 +612,6 @@ dependencies = [
"libc",
]
[[package]]
name = "number_prefix"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.17.2"
@ -769,12 +709,6 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "portable-atomic"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794"
[[package]]
name = "proc-macro2"
version = "1.0.59"
@ -905,12 +839,10 @@ dependencies = [
"serde_urlencoded",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"winreg",
]
@ -1375,19 +1307,6 @@ version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93"
[[package]]
name = "wasm-streams"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.63"

View File

@ -6,7 +6,7 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
reqwest = { version = "0.11", features = ["stream", "json", "multipart"] }
reqwest = { version = "0.11", features = ["blocking", "json", "multipart"] }
tokio = { version = "1", features = ["full"] }
dotenv ="0.15.0"
clap = "2.33"
@ -18,9 +18,6 @@ lazy_static = "1.4.0"
glob = "0.3.1"
textwrap = "0.13"
chrono = "0.4.26"
indicatif = "0.17.5"
md5 = "0.7.0"
futures-util = "0.3.28"
[profile.release]
debug = true

View File

@ -4,11 +4,11 @@ use std::fs::DirBuilder;
use std::path::{Path, PathBuf};
use clap::Values;
use regex::Regex;
use crate::services::downloader::Downloader;
use crate::utils::api::ApiProps;
use crate::global::global::{DIR_PATH, set_dir_path};
use crate::services::api::ApiError;
use crate::services::req_props::{ReqProps, ObjProps};
use crate::services::download_files::DownloadFiles;
use crate::store::object::{tree, blob};
use crate::commands::config;
use crate::commands::init;
@ -28,7 +28,7 @@ pub fn clone(remote: Values<'_>) {
};
let api_props = ApiProps {
host: host.clone(),
username,
username: username,
root: dist_path_str.to_string(),
};
@ -58,7 +58,6 @@ pub fn clone(remote: Values<'_>) {
let res = ReqProps::new()
.set_request(relative_s.as_str(), &api_props)
.gethref()
.getcontentlength()
.getlastmodified()
.send_req_multiple();
@ -105,7 +104,7 @@ pub fn clone(remote: Values<'_>) {
// add tree
let path_folder = p.strip_prefix(ref_path.clone()).unwrap();
let lastmodified = folder.lastmodified.unwrap().timestamp_millis();
if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string(), false) {
if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string()) {
eprintln!("err: saving ref of {} ({})", path_folder.display(), err);
}
}
@ -123,19 +122,37 @@ pub fn clone(remote: Values<'_>) {
first_iter = false;
}
let downloader = Downloader::new()
.set_api_props(api_props.clone())
.set_files(files)
.should_log()
.download(ref_path.clone(), Some(&save_blob));
download_files(ref_path.clone(), files, &api_props);
}
fn save_blob(obj: ObjProps) {
let relative_s = &obj.clone().relative_s.unwrap();
let relative_p = PathBuf::from(&relative_s);
let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
if let Err(err) = blob::add(relative_p, &lastmodified.to_string(), false) {
eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
fn download_files(ref_p: PathBuf, files: Vec<ObjProps>, api_props: &ApiProps) {
for obj in files {
let relative_s = &obj.clone().relative_s.unwrap();
let res = DownloadFiles::new()
.set_url(&relative_s, api_props)
.save(ref_p.clone());
match res {
Ok(()) => {
let relative_p = PathBuf::from(&relative_s);
let lastmodified = obj.clone().lastmodified.unwrap().timestamp_millis();
if let Err(err) = blob::add(relative_p, &lastmodified.to_string()) {
eprintln!("err: saving ref of {} ({})", relative_s.clone(), err);
}
},
Err(ApiError::Unexpected(_)) => {
eprintln!("err: writing {}", relative_s);
},
Err(ApiError::IncorrectRequest(err)) => {
eprintln!("fatal: {}", err.status());
std::process::exit(1);
},
Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"),
Err(ApiError::RequestError(err)) => {
eprintln!("fatal: {}", err);
std::process::exit(1);
}
}
}
}

View File

@ -68,7 +68,7 @@ impl PushChange for New {
let lastmodified = prop.lastmodified.unwrap().timestamp_millis();
// update blob
blob::add(obj.path.clone(), &lastmodified.to_string(), true)?;
blob::add(obj.path.clone(), &lastmodified.to_string())?;
// remove index
index::rm_line(obj.path.to_str().unwrap())?;

View File

@ -74,7 +74,7 @@ impl PushChange for NewDir {
let lastmodified = prop.lastmodified.unwrap().timestamp_millis();
// update tree
tree::add(obj.path.clone(), &lastmodified.to_string(), true)?;
tree::add(obj.path.clone(), &lastmodified.to_string())?;
// remove index
index::rm_line(obj.path.to_str().unwrap())?;

View File

@ -4,4 +4,3 @@ pub mod download_files;
pub mod req_props;
pub mod upload_file;
pub mod delete_path;
pub mod downloader;

View File

@ -2,7 +2,6 @@ use std::env;
use dotenv::dotenv;
use reqwest::Client;
use reqwest::RequestBuilder;
use reqwest::multipart::Form;
use reqwest::{Response, Error, Method};
use reqwest::header::{HeaderValue, CONTENT_TYPE, HeaderMap, IntoHeaderName};
use crate::utils::api::ApiProps;
@ -32,23 +31,6 @@ impl ApiBuilder {
}
}
pub fn set_url(&mut self, method: Method, url: &str) -> &mut ApiBuilder {
let remote = match config::get("remote") {
Some(r) => r,
None => {
eprintln!("fatal: unable to find a remote");
std::process::exit(1);
}
};
let (host, _, _) = get_url_props(&remote);
let mut u = String::from(host);
u.push_str(url);
self.request = Some(self.client.request(method, u));
self
}
pub fn build_request(&mut self, method: Method, path: &str) -> &mut ApiBuilder {
let remote = match config::get("remote") {
Some(r) => r,
@ -111,20 +93,6 @@ impl ApiBuilder {
self
}
pub fn set_multipart(&mut self, form: Form) -> &mut ApiBuilder {
match self.request.take() {
None => {
eprintln!("fatal: incorrect request");
std::process::exit(1);
},
Some(req) => {
self.request = Some(req.multipart(form));
self.set_header(CONTENT_TYPE, HeaderValue::from_static("multipart/related"));
}
}
self
}
pub fn set_header<K: IntoHeaderName>(&mut self, key: K, val: HeaderValue) -> &mut ApiBuilder {
let map = self.headers.get_or_insert(HeaderMap::new());
map.insert(key, val);
@ -142,6 +110,7 @@ impl ApiBuilder {
}
}
self
}
pub async fn send(&mut self) -> Result<Response, Error> {
@ -153,8 +122,7 @@ impl ApiBuilder {
},
Some(req) => {
if let Some(headers) = &self.headers {
req.headers(headers.clone())
.send().await.map_err(Error::from)
req.headers(headers.clone()).send().await.map_err(Error::from)
} else {
req.send().await.map_err(Error::from)
}

View File

@ -1,6 +1,4 @@
use std::path::PathBuf;
use futures_util::StreamExt;
use std::fs::File;
use std::fs::OpenOptions;
use std::io::{self, Write};
use reqwest::{Method, Response, Error};
@ -22,7 +20,7 @@ impl DownloadFiles {
pub fn set_url(&mut self, relative_ps: &str, api_props: &ApiProps) -> &mut DownloadFiles {
self.relative_ps = relative_ps.to_string();
self.api_builder.set_req(Method::GET, relative_ps, api_props);
self.api_builder.set_req(Method::from_bytes(b"PROPFIND").unwrap(), relative_ps, api_props);
self
}
@ -40,40 +38,13 @@ impl DownloadFiles {
}
}
pub fn save_stream(&mut self, ref_p: PathBuf, callback: Option<impl Fn(u64)>) -> Result<(), ApiError> {
let abs_p = ref_p.join(PathBuf::from(self.relative_ps.clone()));
let mut file = File::create(abs_p).unwrap();
tokio::runtime::Runtime::new().unwrap().block_on(async {
let res = self.send().await.map_err(ApiError::RequestError)?;
if res.status().is_success() {
let mut stream = res.bytes_stream();
while let Some(chunk) = stream.next().await {
let unwrap_chunk = chunk.unwrap();
// save chunk inside file
if let Err(err) = file.write_all(&unwrap_chunk) {
return Err(ApiError::Unexpected(err.to_string()));
} else if let Some(fct) = &callback {
// call callback with size of this chunk
fct(unwrap_chunk.len().try_into().unwrap());
}
}
Ok(())
} else {
Err(ApiError::IncorrectRequest(res))
}
})
}
pub fn save(&mut self, ref_p: PathBuf) -> Result<(), ApiError> {
tokio::runtime::Runtime::new().unwrap().block_on(async {
let p = ref_p.join(PathBuf::from(self.relative_ps.clone()));
let res = self.send().await.map_err(ApiError::RequestError)?;
if res.status().is_success() {
let body = res.bytes().await.map_err(ApiError::EmptyError)?;
match Self::write_file(p, &body.to_vec()) {
match DownloadFiles::write_file(p, &body.to_vec()) {
Err(_) => Err(ApiError::Unexpected(String::from(""))),
Ok(_) => Ok(()),
}

View File

@ -1,171 +0,0 @@
use std::path::PathBuf;
use indicatif::{ProgressBar, MultiProgress, ProgressStyle, HumanBytes};
use crate::utils::api::ApiProps;
use crate::services::api::ApiError;
use crate::services::download_files::DownloadFiles;
use crate::services::req_props::ObjProps;
const SIZE_TO_STREAM: u64 = 2 * 1024 * 1024;
pub struct Downloader {
files: Vec<ObjProps>,
should_log: bool,
api_props: Option<ApiProps>,
progress_bars: Vec<ProgressBar>,
multi_progress: Option<MultiProgress>,
}
impl Downloader {
pub fn new() -> Self {
Downloader {
files: vec![],
should_log: false,
api_props: None,
progress_bars: vec![],
multi_progress: None,
}
}
pub fn should_log(&mut self) -> &mut Downloader {
self.should_log = true;
self
}
pub fn set_api_props(&mut self, api_props: ApiProps) -> &mut Downloader {
self.api_props = Some(api_props);
self
}
pub fn set_files(&mut self, files: Vec<ObjProps>) -> &mut Downloader {
self.files = files;
self
}
pub fn add_file(&mut self, file: ObjProps) -> &mut Downloader {
self.files.push(file);
self
}
fn init_log(&mut self, nb_objs: u64, total_size: u64) {
self.multi_progress = Some(MultiProgress::new());
self.progress_bars.push(
self.multi_progress
.clone()
.unwrap()
.add(ProgressBar::new(nb_objs).with_message("Objects")));
let msg = format!("0B/{}", HumanBytes(total_size).to_string());
self.progress_bars.push(
self.multi_progress
.clone()
.unwrap()
.add(ProgressBar::new(total_size).with_message(msg)));
self.progress_bars[0].set_style(
ProgressStyle::with_template("{_:>10} [{bar:40}] {pos}/{len} {msg}")
.unwrap()
.progress_chars("=> "));
self.progress_bars[1].set_style(
ProgressStyle::with_template("[{elapsed_precise}] [{bar:40}] {msg}")
.unwrap()
.progress_chars("=> "));
self.progress_bars[0].tick();
self.progress_bars[1].tick();
}
fn update_bytes_bar(&self, size: u64) {
let bytes_bar = &self.progress_bars[1];
bytes_bar.inc(size);
let msg = format!(
"{}/{}",
HumanBytes(bytes_bar.position()).to_string(),
HumanBytes(bytes_bar.length().unwrap()).to_string());
bytes_bar.set_message(msg);
}
pub fn download(&mut self, ref_p: PathBuf, callback: Option<&dyn Fn(ObjProps)>) {
if self.should_log {
let mut total_size = 0;
let nb_objs = self.files.len();
self.files
.iter()
.for_each(|f|
if let Some(size) = f.contentlength {
total_size += size
}
);
self.init_log(nb_objs.try_into().unwrap(), total_size);
}
for file in self.files.clone() {
let relative_s = &file.clone().relative_s.unwrap();
let mut download = DownloadFiles::new();
download.set_url(&relative_s, &self.api_props.clone().unwrap());
let should_use_stream = {
if let Some(size) = file.contentlength {
if size > SIZE_TO_STREAM {
true
} else {
false
}
} else {
false
}
};
// download
let res = {
if should_use_stream {
// todo should increment here
download.save_stream(ref_p.clone(), Some(|a| self.update_bytes_bar(a)))
} else {
download.save(ref_p.clone())
}
};
// deal with error
match res {
Ok(()) => {
if let Some(fct) = callback {
fct(file.clone());
}
},
Err(ApiError::Unexpected(_)) => {
eprintln!("err: writing {}", relative_s);
},
Err(ApiError::IncorrectRequest(err)) => {
eprintln!("fatal: {}", err.status());
std::process::exit(1);
},
Err(ApiError::EmptyError(_)) => eprintln!("Failed to get body"),
Err(ApiError::RequestError(err)) => {
eprintln!("fatal: {}", err);
std::process::exit(1);
}
}
// increment loading bars
if self.should_log {
self.progress_bars[0].inc(1); // increment object
// increment bytes only if
// not incremented continuously by stream
if !should_use_stream {
self.update_bytes_bar(file.contentlength.unwrap());
}
}
}
// finish all bars
for bar in &self.progress_bars {
bar.finish();
}
}
}

View File

@ -12,7 +12,6 @@ pub struct ObjProps {
pub href: Option<String>,
pub relative_s: Option<String>,
pub lastmodified: Option<DateTime<Utc>>,
pub contentlength: Option<u64>,
}
impl Clone for ObjProps {
@ -21,7 +20,6 @@ impl Clone for ObjProps {
href: self.href.clone(),
relative_s: self.relative_s.clone(),
lastmodified: self.lastmodified.clone(),
contentlength: self.contentlength.clone(),
}
}
}
@ -32,7 +30,6 @@ impl ObjProps {
href: None,
relative_s: None,
lastmodified: None,
contentlength: None,
}
}
}
@ -76,7 +73,7 @@ impl ReqProps {
self
}
pub fn getcontentlength(&mut self) -> &mut ReqProps {
pub fn _getcontentlenght(&mut self) -> &mut ReqProps {
self.xml_balises.push(String::from("getcontentlength"));
self.xml_payload.push_str(r#"<d:getcontentlength/>"#);
self
@ -125,21 +122,18 @@ impl ReqProps {
}
pub fn send_with_err(&mut self) -> Result<String, ApiError> {
tokio::runtime::Runtime::new().unwrap().block_on(async {
match self.send().await {
Err(res) => Err(ApiError::RequestError(res)),
Ok(res) if res.status().is_success() => {
let body = res
.text()
.await
.map_err(|err| ApiError::EmptyError(err))?;
Ok(body)
},
Ok(res) => {
Err(ApiError::IncorrectRequest(res))
}
}
})
let res = tokio::runtime::Runtime::new().unwrap().block_on(async {
self.send().await
}).map_err(ApiError::RequestError)?;
if res.status().is_success() {
let body = tokio::runtime::Runtime::new().unwrap().block_on(async {
res.text().await
}).map_err(ApiError::EmptyError)?;
Ok(body)
} else {
Err(ApiError::IncorrectRequest(res))
}
}
pub fn send_req_multiple(&mut self) -> Result<Vec<ObjProps>, ApiError> {
@ -166,60 +160,54 @@ impl ReqProps {
let cursor = Cursor::new(xml);
let parser = EventReader::new(cursor);
let mut should_get = false;
let mut values: Vec<ObjProps> = vec![];
let mut should_get = false;
let mut val: String = String::from("");
let mut iter = self.xml_balises.iter();
let mut val = iter.next();
let mut content = ObjProps::new();
for event in parser {
match event {
Ok(XmlEvent::StartElement { name, .. }) => {
should_get = {
if self.xml_balises.clone().contains(&name.local_name) {
val = name.local_name.clone();
true
if let Some(v) = val.clone() {
should_get = &name.local_name == v;
} else {
// end of balises to get then start over for
// next object if want multiple
values.push(content.clone());
if multiple {
iter = self.xml_balises.iter();
val = iter.next();
content = ObjProps::new();
if let Some(v) = val.clone() {
should_get = &name.local_name == v;
}
} else {
false
break;
}
};
}
}
Ok(XmlEvent::Characters(text)) => {
if !text.trim().is_empty() && should_get {
match val.as_str() {
match val.unwrap().as_str() {
"href" => {
content.href = Some(text.clone());
content.relative_s = Some(
get_relative_s(text, &(self.api_props
.clone()
.unwrap())));
content.relative_s = Some(get_relative_s(text, &(self.api_props.clone().unwrap())));
},
"getlastmodified" => {
content.lastmodified = Some(
parse_timestamp(&text).unwrap());
},
"getcontentlength" => {
content.contentlength = Some(
text.clone().parse().unwrap());
content.lastmodified = Some(parse_timestamp(&text).unwrap());
},
_ => (),
}
should_get = false;
val = iter.next()
}
}
Ok(XmlEvent::EndElement { name, .. }) => {
if name.local_name == "response" {
values.push(content.clone());
if multiple {
content = ObjProps::new();
} else {
break;
}
}
Ok(XmlEvent::EndElement { .. }) => {
should_get = false;
}
Err(e) => {
eprintln!("err: parsing xml: {}", e);
eprintln!("Error: {}", e);
break;
}
_ => {}

View File

@ -1,5 +1,5 @@
use std::fs::File;
use std::io::Read;
use std::io::{Read};
use std::path::PathBuf;
use reqwest::{Method, Response, Error};
use crate::services::api::{ApiBuilder, ApiError};

View File

@ -5,7 +5,7 @@ use crate::utils::path;
use crate::store::head;
use crate::store::object::{update_dates, parse_path, add_node, create_obj, rm_node};
pub fn add(path: PathBuf, date: &str, up_parent: bool) -> io::Result<()> {
pub fn add(path: PathBuf, date: &str) -> io::Result<()> {
let (line, hash, name) = parse_path(path.clone(), true);
// add blob reference to parent
if path.iter().count() == 1 {
@ -22,9 +22,7 @@ pub fn add(path: PathBuf, date: &str, up_parent: bool) -> io::Result<()> {
create_obj(hash, &content)?;
// update date for all parent
if up_parent {
update_dates(path, date)?;
}
update_dates(path, date)?;
Ok(())
}

View File

@ -5,7 +5,7 @@ use crate::utils::{read, path};
use crate::store::head;
use crate::store::object::{self, update_dates, parse_path, hash_obj, add_node, create_obj};
pub fn add(path: PathBuf, date: &str, up_parent: bool) -> io::Result<()> {
pub fn add(path: PathBuf, date: &str) -> io::Result<()> {
let (line, hash, name) = parse_path(path.clone(), false);
// add tree reference to parent
@ -22,9 +22,7 @@ pub fn add(path: PathBuf, date: &str, up_parent: bool) -> io::Result<()> {
create_obj(hash, &content)?;
// update date for all parent
if up_parent {
update_dates(path, date)?;
}
update_dates(path, date)?;
Ok(())
}