optimize clone by allowing to fetch with a different depth

src/commands/clone.rs

@@ -14,10 +14,17 @@ use crate::store::object::{tree, blob};
 use crate::commands::config;
 use crate::commands::init;
 
-pub fn clone(remote: Values<'_>) {
+pub const DEPTH: &str = "3";
+
+pub struct CloneArgs<'a> {
+    pub remote: Values<'a>,
+    pub depth: Option<String>,
+}
+
+pub fn clone(args: CloneArgs) {
     let d = DIR_PATH.lock().unwrap().clone();
 
-    let url = remote.clone().next().unwrap();
+    let url = args.remote.clone().next().unwrap();
     let (host, tmp_user, dist_path_str) = get_url_props(url);
     let username = match tmp_user {
         Some(u) => u.to_string(),
@@ -59,7 +66,12 @@ pub fn clone(remote: Values<'_>) {
         }
     }
 
-    let (folders, files) = enumerate_remote(|a| req(&api_props, a));
+    let depth = &args.depth.clone().unwrap_or(DEPTH.to_string());
+    let (folders, files) = enumerate_remote(|a| req(
+            &api_props,
+            depth,
+            a
+            ), Some(depth));
 
     for folder in folders {
         // create folder
@@ -92,9 +104,10 @@ fn save_blob(obj: ObjProps) {
     }
 }
 
-fn req(api_props: &ApiProps, relative_s: &str) -> Result<Vec<ObjProps>, ApiError> {
+fn req(api_props: &ApiProps, depth: &str, relative_s: &str) -> Result<Vec<ObjProps>, ApiError> {
     ReqProps::new()
         .set_request(relative_s, &api_props)
+        .set_depth(depth)
         .gethref()
         .getcontentlength()
         .getlastmodified()

src/main.rs

@@ -3,6 +3,7 @@ use textwrap::{fill, Options};
 
 use crate::commands::add::AddArgs;
 use crate::commands::remote_diff::RemoteDiffArgs;
+use crate::commands::clone::{self, CloneArgs};
 
 mod commands;
 mod utils;
@@ -25,7 +26,18 @@ fn main() {
                 .value_name("REMOTE")
                 .help(&fill(
                         "The repository to clone from. See the NEXTSYNC URLS section below for more information on specifying repositories.",
-                        Options::new(80).width,
+                        Options::new(70).width,
+                        ))
+                )
+            .arg(
+                Arg::with_name("depth")
+                .short("d")
+                .long("depth")
+                .required(false)
+                .takes_value(true)
+                .help(&fill(
+                        &format!("Depth of the recursive fetch of object properties. This value should be lower when there are a lot of files per directory and higher when there are a lot of subdirectories with fewer files. (Default: {})", clone::DEPTH),
+                        Options::new(70).width,
                         ))
                 )
             .arg(
@@ -127,7 +139,7 @@ fn main() {
     } else if let Some(matches) = matches.subcommand_matches("add") {
         if let Some(files) = matches.values_of("files") {
             commands::add::add(AddArgs {
-                files: files,
+                files,
                 force: matches.is_present("force"),
             });
         }
@@ -138,7 +150,12 @@ fn main() {
             global::global::set_dir_path(String::from(val.clone().next().unwrap()));
         }
         if let Some(remote) = matches.values_of("remote") {
-            commands::clone::clone(remote);
+            commands::clone::clone(CloneArgs {
+                remote,
+                depth: matches.values_of("depth").map(
+                    |mut val| val.next().unwrap().to_owned()
+                    ),
+            });
         }
     } else if let Some(_matches) = matches.subcommand_matches("push") {
         commands::push::push();

src/services/downloader.rs

@@ -123,7 +123,6 @@ impl Downloader {
             // download
             let res = {
                 if should_use_stream {
-                    // todo should increment here
                     download.save_stream(ref_p.clone(), Some(|a| self.update_bytes_bar(a)))
                 } else {
                     download.save(ref_p.clone())

src/utils/remote.rs

@@ -1,6 +1,6 @@
 use crate::services::{req_props::ObjProps, api::ApiError};
 
-pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>) -> (Vec<ObjProps>, Vec<ObjProps>) {
+pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>, depth: Option<&str>) -> (Vec<ObjProps>, Vec<ObjProps>) {
     let mut folders: Vec<ObjProps> = vec![ObjProps::new()];
     let mut all_folders:  Vec<ObjProps> = vec![];
     let mut files: Vec<ObjProps> = vec![];
@@ -33,12 +33,18 @@ pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>) -
             Err(ApiError::Unexpected(_)) => todo!()
         };
 
-        // find folders and files in response
+        // separate folders and files in response
+        
         let mut iter = objs.iter();
-        iter.next(); // jump first element which is the folder cloned
+        // first element is not used as it is the fetched folder
+        let default_depth = calc_depth(iter.next().unwrap());
+        let d = depth.unwrap_or("0").parse::<u16>().unwrap();
         for object in iter {
             if object.is_dir() {
-                folders.push(object.clone());
+                // should get content of this folder if it is not already in this reponse
+                if calc_depth(object) - default_depth == d {
+                    folders.push(object.clone());
+                }
                 all_folders.push(object.clone());
             } else {
                 files.push(object.clone());
@@ -48,3 +54,8 @@ pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>) -
 
     (all_folders, files)
 }
+
+fn calc_depth(obj: &ObjProps) -> u16 {
+    obj.relative_s.clone().unwrap_or(String::from("")).split("/").count() as u16
+}
+