remote diff with new opti remote enumerater

README.md

@@ -2,7 +2,7 @@
 
 A git-like command line tool to interact with Nextcloud.
 
-This is **work in progress**.
+This is **in working progress**.
 
 This should work pretty much like git with some adaptations to be more debuggable (for now) and easier to code. There is no history and with that no need to commit, to upload new files you have to add and push them.
 

src/commands.rs

@@ -5,3 +5,4 @@ pub mod reset;
 pub mod clone;
 pub mod push;
 pub mod config;
+pub mod remote_diff;

src/commands/clone.rs

@@ -6,6 +6,7 @@ use clap::Values;
 use regex::Regex;
 use crate::services::downloader::Downloader;
 use crate::utils::api::ApiProps;
+use crate::utils::remote::{enumerate_remote, EnumerateOptions};
 use crate::global::global::{DIR_PATH, set_dir_path};
 use crate::services::api::ApiError;
 use crate::services::req_props::{ReqProps, ObjProps};
@@ -13,10 +14,17 @@ use crate::store::object::{tree, blob};
 use crate::commands::config;
 use crate::commands::init;
 
-pub fn clone(remote: Values<'_>) {
+pub const DEPTH: &str = "3";
+
+pub struct CloneArgs<'a> {
+    pub remote: Values<'a>,
+    pub depth: Option<String>,
+}
+
+pub fn clone(args: CloneArgs) {
     let d = DIR_PATH.lock().unwrap().clone();
 
-    let url = remote.clone().next().unwrap();
+    let url = args.remote.clone().next().unwrap();
     let (host, tmp_user, dist_path_str) = get_url_props(url);
     let username = match tmp_user {
         Some(u) => u.to_string(),
@@ -43,84 +51,43 @@ pub fn clone(remote: Values<'_>) {
         },
     };
 
-    let mut folders: Vec<ObjProps> = vec![ObjProps::new()];
+    // try to create root folder 
-    let mut files: Vec<ObjProps> = vec![];
+    if DirBuilder::new().recursive(true).create(ref_path.clone()).is_err() {
-    let mut first_iter = true;
+        eprintln!("fatal: unable to create the destination directory");
-    while folders.len() > 0 {
+        std::process::exit(1);
-        let folder = folders.pop().unwrap();
+    } else {
+        init::init();
+        let mut remote_config = api_props.username.clone();
+        remote_config.push_str("@");
+        remote_config.push_str(api_props.host.strip_prefix("https://").unwrap());
+        remote_config.push_str(&api_props.root);
+        if config::set("remote", &remote_config).is_err() {
+            eprintln!("err: not able to save remote");
+        }
+    }
 
-        let relative_s = match folder.relative_s {
+    let depth = &args.depth.clone().unwrap_or(DEPTH.to_string());
-            Some(relative_s) => relative_s,
+    let (folders, files) = enumerate_remote(
-            None => String::from(""),
+        |a| req(&api_props, depth, a),
-        };
+        &should_skip,
+        EnumerateOptions {
+            depth: Some(depth.to_owned()),
+            relative_s: None
+        });
 
-        // request folder content
+    for folder in folders {
-        let res = ReqProps::new()
+        // create folder
-            .set_request(relative_s.as_str(), &api_props)
+        let p = ref_path.clone().join(Path::new(&folder.relative_s.unwrap()));
-            .gethref()
+        if let Err(err) = DirBuilder::new().recursive(true).create(p.clone()) {
-            .getcontentlength()
+            eprintln!("err: cannot create directory {} ({})", p.display(), err);
-            .getlastmodified()
-            .send_req_multiple();
-
-        let objs = match res {
-            Ok(o) => o,
-            Err(ApiError::IncorrectRequest(err)) => {
-                eprintln!("fatal: {}", err.status());
-                std::process::exit(1);
-            },
-            Err(ApiError::EmptyError(_)) => {
-                eprintln!("Failed to get body");
-                vec![]
-            }
-            Err(ApiError::RequestError(err)) => {
-                eprintln!("fatal: {}", err);
-                std::process::exit(1);
-            },
-            Err(ApiError::Unexpected(_)) => todo!()
-        };
-
-        // create object
-        if first_iter {
-            // root folder, init and config
-            if DirBuilder::new().recursive(true).create(ref_path.clone()).is_err() {
-                eprintln!("fatal: unable to create the destination directory");
-                std::process::exit(1);
-            } else {
-                init::init();
-                let mut remote_config = api_props.username.clone();
-                remote_config.push_str("@");
-                remote_config.push_str(api_props.host.strip_prefix("https://").unwrap());
-                remote_config.push_str(&api_props.root);
-                if config::set("remote", &remote_config).is_err() {
-                    eprintln!("err: not able to save remote");
-                }
-            }
-        } else {
-            // create folder
-            let p = ref_path.clone().join(Path::new(&relative_s));
-            if let Err(err) = DirBuilder::new().recursive(true).create(p.clone()) {
-                eprintln!("err: cannot create directory {} ({})", p.display(), err);
-            }
-
-            // add tree
-            let path_folder = p.strip_prefix(ref_path.clone()).unwrap();
-            let lastmodified = folder.lastmodified.unwrap().timestamp_millis();
-            if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string(), false) {
-                eprintln!("err: saving ref of {} ({})", path_folder.display(), err);
-            }
         }
 
-        // find folders and files in response
+        // add tree
-        let mut iter = objs.iter();
+        let path_folder = p.strip_prefix(ref_path.clone()).unwrap();
-        iter.next(); // jump first element which is the folder cloned
+        let lastmodified = folder.lastmodified.unwrap().timestamp_millis();
-        for object in iter {
+        if let Err(err) = tree::add(path_folder.to_path_buf(), &lastmodified.to_string(), false) {
-            if object.href.clone().unwrap().chars().last().unwrap() == '/' {
+            eprintln!("err: saving ref of {} ({})", path_folder.display(), err);
-                folders.push(object.clone());
-            } else {
-                files.push(object.clone());
-            }
         }
-        first_iter = false;
     }
 
     let downloader = Downloader::new()
@@ -139,6 +106,20 @@ fn save_blob(obj: ObjProps) {
     }
 }
 
+fn should_skip(_: ObjProps) -> bool {
+    return false;
+}
+
+fn req(api_props: &ApiProps, depth: &str, relative_s: &str) -> Result<Vec<ObjProps>, ApiError> {
+    ReqProps::new()
+        .set_request(relative_s, &api_props)
+        .set_depth(depth)
+        .gethref()
+        .getcontentlength()
+        .getlastmodified()
+        .send_req_multiple()
+}
+
 pub fn get_url_props(url: &str) -> (String, Option<&str>, &str) {
     let mut username = None;
     let mut domain = "";

src/commands/remote_diff.rs

@@ -0,0 +1,82 @@
+use crate::services::api::ApiError;
+use crate::services::req_props::{ReqProps, ObjProps};
+use crate::store::object::{Object, self};
+use crate::utils::api::{ApiProps, get_api_props};
+use crate::utils::path;
+use crate::utils::remote::{enumerate_remote, EnumerateOptions};
+use std::fs::canonicalize;
+use std::path::PathBuf;
+
+pub struct RemoteDiffArgs {
+    pub path: Option<String>,
+}
+
+pub fn remote_diff(args: RemoteDiffArgs) {
+    let path = {
+        if let Some(path) = args.path {
+            let mut cur = path::current().unwrap();
+            cur.push(path);
+            let canonic = canonicalize(cur).ok().unwrap();
+            dbg!(&canonic);
+            dbg!(path::repo_root());
+            let ok = canonic.strip_prefix(path::repo_root());
+                dbg!(&ok);
+            
+                // todo
+            PathBuf::from("/")
+        } else {
+            PathBuf::from("/")
+        }
+    };
+
+    let mut folders: Vec<ObjProps> = vec![ObjProps {
+        contentlength: None,
+        href: None,
+        lastmodified: None,
+        relative_s: Some(path.to_str().unwrap().to_owned()),
+    }];
+    let mut files: Vec<ObjProps> = vec![];
+
+    let depth = "2"; // todo
+                     // todo origin
+    let api_props = get_api_props();
+    let (folders, files) = enumerate_remote(
+        |a| req(&api_props, depth, a),
+        &should_skip,
+        EnumerateOptions {
+            depth: Some(depth.to_owned()),
+            relative_s: Some(path.to_str().unwrap().to_owned())
+        });
+
+    for folder in folders {
+            println!("should pull {}", folder.clone().relative_s.unwrap());
+    }
+    for file in files {
+            println!("should pull {}", file.clone().relative_s.unwrap());
+    }
+
+}
+
+fn should_skip(obj: ObjProps) -> bool {
+    let mut o = Object::new(&obj.clone().relative_s.unwrap());
+    let exist = o.exists();
+
+    // if doesn't exist locally when cannot skip it as we need to pull it
+    if !exist {
+        return false;
+    }
+
+    // if local directory is older there is changes on the remote we cannot
+    // skip this folder
+    !o.read().is_older(obj.lastmodified.unwrap().timestamp())
+}
+
+fn req(api_props: &ApiProps, depth: &str, relative_s: &str) -> Result<Vec<ObjProps>, ApiError> {
+    ReqProps::new()
+        .set_request(relative_s, &api_props)
+        .set_depth(depth)
+        .gethref()
+        .getlastmodified()
+        .send_req_multiple()
+}
+

src/main.rs

@@ -1,6 +1,9 @@
 use clap::{App, Arg, SubCommand};
 use textwrap::{fill, Options};
+
 use crate::commands::add::AddArgs;
+use crate::commands::remote_diff::RemoteDiffArgs;
+use crate::commands::clone::{self, CloneArgs};
 
 mod commands;
 mod utils;
@@ -23,7 +26,18 @@ fn main() {
                 .value_name("REMOTE")
                 .help(&fill(
                         "The repository to clone from. See the NEXTSYNC URLS section below for more information on specifying repositories.",
-                        Options::new(80).width,
+                        Options::new(70).width,
+                        ))
+                )
+            .arg(
+                Arg::with_name("depth")
+                .short("d")
+                .long("depth")
+                .required(false)
+                .takes_value(true)
+                .help(&fill(
+                        &format!("Depth of the recursive fetch of object properties. This value should be lower when there are a lot of files per directory and higher when there are a lot of subdirectories with fewer files. (Default: {})", clone::DEPTH),
+                        Options::new(70).width,
                         ))
                 )
             .arg(
@@ -96,6 +110,20 @@ fn main() {
                 .value_name("VALUE")
                 )
             )
+        .subcommand(
+            SubCommand::with_name("remote-diff")
+            .arg(
+                Arg::with_name("path")
+                .required(false)
+                .takes_value(true)
+                .value_name("PATH")
+                .help("The path to pull."),
+                )
+            .about("Fetch new and modifed files from the nextcloud server.")
+            )
+        .subcommand(
+            SubCommand::with_name("test")
+            )
         .get_matches();
 
     if let Some(matches) = matches.subcommand_matches("init") {
@@ -111,7 +139,7 @@ fn main() {
     } else if let Some(matches) = matches.subcommand_matches("add") {
         if let Some(files) = matches.values_of("files") {
             commands::add::add(AddArgs {
-                files: files,
+                files,
                 force: matches.is_present("force"),
             });
         }
@@ -122,7 +150,12 @@ fn main() {
             global::global::set_dir_path(String::from(val.clone().next().unwrap()));
         }
         if let Some(remote) = matches.values_of("remote") {
-            commands::clone::clone(remote);
+            commands::clone::clone(CloneArgs {
+                remote,
+                depth: matches.values_of("depth").map(
+                    |mut val| val.next().unwrap().to_owned()
+                    ),
+            });
         }
     } else if let Some(_matches) = matches.subcommand_matches("push") {
         commands::push::push();
@@ -134,6 +167,21 @@ fn main() {
                 }
             }
         }
+    } else if let Some(matches) = matches.subcommand_matches("remote-diff") {
+        commands::remote_diff::remote_diff(RemoteDiffArgs {
+            path: {
+                if let Some(mut path) = matches.values_of("path") {
+                    match path.next() {
+                        Some(p) => Some(String::from(p)),
+                        None => None,
+                    }
+                } else {
+                    None
+                }
+            },
+        });
+    } else if let Some(_) = matches.subcommand_matches("test") {
+
     }
 }
 

src/services/api.rs

@@ -63,7 +63,9 @@ impl ApiBuilder {
         url.push_str(username.unwrap());
         url.push_str(&root);
         url.push_str("/");
-        url.push_str(path);
+        if path !=  "/" {
+            url.push_str(path);
+        } 
         self.request = Some(self.client.request(method, url));
         self
     }
@@ -75,7 +77,9 @@ impl ApiBuilder {
         url.push_str(&api_props.username);
         url.push_str(&api_props.root);
         url.push_str("/");
-        url.push_str(p);
+        if p !=  "/" {
+            url.push_str(p);
+        } 
         self.request = Some(self.client.request(meth, url));
         self
     }

src/services/downloader.rs

@@ -123,7 +123,6 @@ impl Downloader {
             // download
             let res = {
                 if should_use_stream {
-                    // todo should increment here
                     download.save_stream(ref_p.clone(), Some(|a| self.update_bytes_bar(a)))
                 } else {
                     download.save(ref_p.clone())

src/services/req_props.rs

@@ -3,6 +3,8 @@ use chrono::{Utc, DateTime};
 use reqwest::{Method, Response, Error};
 use xml::reader::{EventReader, XmlEvent};
 use reqwest::header::HeaderValue;
+use crate::commands::clone::get_url_props;
+use crate::commands::config;
 use crate::utils::time::parse_timestamp;
 use crate::utils::api::{get_relative_s, ApiProps};
 use crate::services::api::{ApiBuilder, ApiError};
@@ -35,6 +37,15 @@ impl ObjProps {
             contentlength: None,
         }
     }
+
+    pub fn is_dir(&self) -> bool {
+        if let Some(href) = &self.href {
+            href.chars().last().unwrap() == '/'
+        } else {
+            eprintln!("err: cannot determine object type wihout href");
+            false
+        }
+    }
 }
 
 pub struct ReqProps {
@@ -55,6 +66,19 @@ impl ReqProps {
     }
 
     pub fn set_url(&mut self, url: &str) -> &mut ReqProps {
+        let remote = match config::get("remote") {
+            Some(r) => r,
+            None => {
+                eprintln!("fatal: unable to find a remote");
+                std::process::exit(1);
+            }
+        };
+        let (host, username, root) = get_url_props(&remote);
+        self.api_props = Some(ApiProps {
+            host,
+            username: username.unwrap().to_owned(),
+            root: root.to_owned(),
+        });
         self.api_builder.build_request(Method::from_bytes(b"PROPFIND").unwrap(), url);
         self
     }
@@ -66,6 +90,7 @@ impl ReqProps {
     }
 
     pub fn gethref(&mut self) -> &mut ReqProps {
+        // not an actual property but used to prevent getting anything else
         self.xml_balises.push(String::from("href"));
         self
     }
@@ -112,6 +137,7 @@ impl ReqProps {
     }
 
     fn validate_xml(&mut self) -> &mut ReqProps {
+        self.gethref();
         let mut xml = String::from(r#"<?xml version="1.0" encoding="UTF-8"?><d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns"><d:prop>"#);
         xml.push_str(&self.xml_payload.clone());
         xml.push_str(r#"</d:prop></d:propfind>"#);

src/store/head.rs

@@ -1,10 +1,16 @@
 use std::fs::OpenOptions;
+use std::path::PathBuf;
 use std::io::{self, Write};
 use crate::utils::{read, path};
 
-pub fn add_line(line: String) -> io::Result<()> {
+pub fn path() -> PathBuf {
     let mut root = path::nextsync();
     root.push("HEAD");
+    root 
+}
+
+pub fn add_line(line: String) -> io::Result<()> {
+    let root = path();
 
     let mut file = OpenOptions::new()
         .read(true)
@@ -17,8 +23,7 @@ pub fn add_line(line: String) -> io::Result<()> {
 }
 
 pub fn rm_line(line: &str) -> io::Result<()> {
-    let mut root = path::nextsync();
+    let root = path();
-    root.push("HEAD");
     read::rm_line(root, line)?;
     Ok(())
 }

src/store/object.rs

@@ -4,11 +4,85 @@ use std::fs::{self, OpenOptions};
 use crypto::sha1::Sha1;
 use crypto::digest::Digest;
 use std::io::{Seek, SeekFrom, Read};
+use crate::utils::time::parse_timestamp;
+use crate::store::head;
 use crate::utils::{read, path};
 
 pub mod tree;
 pub mod blob;
 
+pub struct Object {
+    path: PathBuf,
+    hash: String,
+    obj_p: PathBuf,
+    ts: Option<i64>
+}
+
+impl Object {
+    pub fn new(path: &str) -> Object {
+        let path = match path.chars().next_back() == "/".chars().next() {
+            true => {
+                let mut new = path.chars();
+                new.next_back();
+                new.as_str()
+            },
+            false => path,
+        };
+        if path == "" {
+            return Object {
+                path: PathBuf::from("/"),
+                hash: String::from(""),
+                obj_p: head::path(),
+                ts: None,
+            }
+        }
+
+        let mut hasher = Sha1::new();
+        hasher.input_str(path);
+        let hash = hasher.result_str();
+
+        let (dir, res) = hash.split_at(2);
+
+        let mut obj_p = path::objects();
+        obj_p.push(dir);
+        obj_p.push(res);
+
+        Object {
+            path: PathBuf::from(path),
+            hash,
+            obj_p,
+            ts: None,
+        } 
+    }
+
+    pub fn read(&mut self) -> &mut Object {
+        match read::read_lines(&self.obj_p) {
+            Ok(mut reader) => {
+                if let Some(Ok(line)) = reader.next() {
+                    let mut data = line.rsplit(' ');
+                    if data.clone().count() >= 2 {
+                        self.ts = Some(data.next().unwrap().parse::<i64>().unwrap())
+                    }
+                }
+            },
+            Err(err) => {
+                eprintln!("error reading object {}: {}", self.obj_p.display(), err);
+            },
+        };
+        self
+    }
+
+    pub fn exists(&mut self) -> bool {
+       self.obj_p.exists()
+    }
+
+    /// return true if the local file is older than the remote one
+    pub fn is_older(&mut self, ts: i64) -> bool {
+        // todo be aware of the diff of ts format
+        ts > self.ts.expect("Should be read before used") / 1000
+    }
+}
+
 /// Returns (line, hash, name)
 ///
 /// # Examples

src/utils.rs

@@ -3,3 +3,4 @@ pub mod read;
 pub mod nextsyncignore;
 pub mod api;
 pub mod time;
+pub mod remote;

src/utils/remote.rs

@@ -0,0 +1,93 @@
+use crate::services::{req_props::ObjProps, api::ApiError};
+
+pub struct EnumerateOptions {
+    pub depth: Option<String>,
+    pub relative_s: Option<String>,
+}
+
+pub fn enumerate_remote(
+    req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>,
+    should_skip: &dyn Fn(ObjProps) -> bool,
+    options: EnumerateOptions
+    ) -> (Vec<ObjProps>, Vec<ObjProps>) {
+
+    let mut folders: Vec<ObjProps> = vec![ObjProps::new()];
+    let mut all_folders:  Vec<ObjProps> = vec![];
+    let mut files: Vec<ObjProps> = vec![];
+
+    while folders.len() > 0 {
+        let folder = folders.pop().unwrap();
+
+        let relative_s = match folder.relative_s {
+            Some(relative_s) => relative_s,
+            None => options.relative_s.clone().unwrap_or(String::from("")),
+        };
+
+        // request folder content
+        let res = req(relative_s.as_str());
+
+        let objs = match res {
+            Ok(o) => o,
+            Err(ApiError::IncorrectRequest(err)) => {
+                eprintln!("fatal: {}", err.status());
+                std::process::exit(1);
+            },
+            Err(ApiError::EmptyError(_)) => {
+                eprintln!("Failed to get body");
+                vec![]
+            }
+            Err(ApiError::RequestError(err)) => {
+                eprintln!("fatal: {}", err);
+                std::process::exit(1);
+            },
+            Err(ApiError::Unexpected(_)) => todo!()
+        };
+
+        // separate folders and files in response
+        let mut iter = objs.iter();
+        // first element is not used as it is the fetched folder
+        let default_depth = calc_depth(iter.next().unwrap());
+        let d = options.depth.clone().unwrap_or("0".to_owned()).parse::<u16>().unwrap();
+        let mut skip_depth = 0;
+        for object in iter {
+            if object.is_dir() {
+                let current_depth = calc_depth(object);
+                // skip children of skiped folder
+                if skip_depth != 0 && skip_depth < current_depth {
+                   continue; 
+                }
+
+                let should_skip = should_skip(object.clone());
+                if should_skip {
+                    skip_depth = current_depth;
+                } else {
+                    skip_depth = 0;
+                    all_folders.push(object.clone());
+                }
+
+                // should get content of this folder if it is not already in this reponse
+                if current_depth - default_depth == d && !should_skip {
+                    folders.push(object.clone());
+                }
+            } else {
+                let current_depth = calc_depth(object);
+                // skip children of skiped folder
+                if skip_depth != 0 && skip_depth < current_depth {
+                   continue; 
+                }
+
+                if !should_skip(object.clone()) {
+                    skip_depth = 0;
+                    files.push(object.clone());
+                }
+            }
+        }
+    }
+
+    (all_folders, files)
+}
+
+fn calc_depth(obj: &ObjProps) -> u16 {
+    obj.relative_s.clone().unwrap_or(String::from("")).split("/").count() as u16
+}
+