grimhilt/nextsync-rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remote diff with new opti remote enumerater

Browse Source
This commit is contained in:
grimhilt
2023-07-21 17:10:57 +02:00
parent f01983b29d
commit 4e20ec94f9
5 changed files with 97 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/commands/clone.rs
View File

@@ -6,7 +6,7 @@ use clap::Values;
use regex::Regex;
use crate::services::downloader::Downloader;
use crate::utils::api::ApiProps;
use crate::utils::remote::enumerate_remote;
use crate::utils::remote::{enumerate_remote, EnumerateOptions};
use crate::global::global::{DIR_PATH, set_dir_path};
use crate::services::api::ApiError;
use crate::services::req_props::{ReqProps, ObjProps};
@@ -67,11 +67,13 @@ pub fn clone(args: CloneArgs) {
}
let depth = &args.depth.clone().unwrap_or(DEPTH.to_string());
let (folders, files) = enumerate_remote(|a| req(
&api_props,
depth,
a
), Some(depth));
let (folders, files) = enumerate_remote(
|a| req(&api_props, depth, a),
&should_skip,
EnumerateOptions {
depth: Some(depth.to_owned()),
relative_s: None
});
for folder in folders {
// create folder
@@ -104,6 +106,10 @@ fn save_blob(obj: ObjProps) {
}
}
fn should_skip(_: ObjProps) -> bool {
return false;
}
fn req(api_props: &ApiProps, depth: &str, relative_s: &str) -> Result<Vec<ObjProps>, ApiError> {
ReqProps::new()
.set_request(relative_s, &api_props)

100
src/commands/remote_diff.rs
View File

@@ -1,7 +1,9 @@
use crate::services::api::ApiError;
use crate::services::req_props::{ReqProps, ObjProps};
use crate::store::object::Object;
use crate::store::object::{Object, self};
use crate::utils::api::{ApiProps, get_api_props};
use crate::utils::path;
use crate::utils::remote::{enumerate_remote, EnumerateOptions};
use std::fs::canonicalize;
use std::path::PathBuf;
@@ -16,9 +18,11 @@ pub fn remote_diff(args: RemoteDiffArgs) {
cur.push(path);
let canonic = canonicalize(cur).ok().unwrap();
dbg!(&canonic);
dbg!(path::repo_root());
let ok = canonic.strip_prefix(path::repo_root());
dbg!(&ok);
// todo
PathBuf::from("/")
} else {
PathBuf::from("/")
@@ -33,62 +37,46 @@ pub fn remote_diff(args: RemoteDiffArgs) {
}];
let mut files: Vec<ObjProps> = vec![];
while folders.len() > 0 {
let folder = folders.pop().unwrap();
let res = ReqProps::new()
.set_url(&folder.relative_s.unwrap())
.gethref()
.getlastmodified()
.send_req_multiple();
let objs = match res {
Ok(o) => o,
Err(ApiError::IncorrectRequest(err)) => {
eprintln!("fatal: {}", err.status());
std::process::exit(1);
},
Err(ApiError::EmptyError(_)) => {
eprintln!("Failed to get body");
vec![]
}
Err(ApiError::RequestError(err)) => {
eprintln!("fatal: {}", err);
std::process::exit(1);
},
Err(ApiError::Unexpected(_)) => todo!()
};
let mut iter = objs.iter();
// todo opti store date of root
let root = iter.next();
for obj in iter {
let mut o = Object::new(&obj.clone().relative_s.unwrap());
let exist = o.exists();
let should_pull = {
if exist {
o.read()
.is_older(obj.lastmodified.unwrap().timestamp())
} else {
true
}
};
if should_pull {
println!("should pull {}", obj.clone().relative_s.unwrap());
if obj.href.clone().unwrap().chars().last().unwrap() == '/' {
folders.push(obj.clone());
} else {
files.push(obj.clone());
}
}
}
let depth = "2"; // todo
// todo origin
let api_props = get_api_props();
let (folders, files) = enumerate_remote(
|a| req(&api_props, depth, a),
&should_skip,
EnumerateOptions {
depth: Some(depth.to_owned()),
relative_s: Some(path.to_str().unwrap().to_owned())
});
for folder in folders {
println!("should pull {}", folder.clone().relative_s.unwrap());
}
for file in files {
println!("should pull {}", file.clone().relative_s.unwrap());
}
}
fn should_skip(obj: ObjProps) -> bool {
let mut o = Object::new(&obj.clone().relative_s.unwrap());
let exist = o.exists();
// if doesn't exist locally when cannot skip it as we need to pull it
if !exist {
return false;
}
// if local directory is older there is changes on the remote we cannot
// skip this folder
!o.read().is_older(obj.lastmodified.unwrap().timestamp())
}
fn req(api_props: &ApiProps, depth: &str, relative_s: &str) -> Result<Vec<ObjProps>, ApiError> {
ReqProps::new()
.set_request(relative_s, &api_props)
.set_depth(depth)
.gethref()
.getlastmodified()
.send_req_multiple()
}

1
src/store/object.rs
View File

@@ -76,6 +76,7 @@ impl Object {
self.obj_p.exists()
}
/// return true if the local file is older than the remote one
pub fn is_older(&mut self, ts: i64) -> bool {
// todo be aware of the diff of ts format
ts > self.ts.expect("Should be read before used") / 1000

46
src/utils/remote.rs
View File

@@ -1,6 +1,16 @@
use crate::services::{req_props::ObjProps, api::ApiError};
pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>, depth: Option<&str>) -> (Vec<ObjProps>, Vec<ObjProps>) {
pub struct EnumerateOptions {
pub depth: Option<String>,
pub relative_s: Option<String>,
}
pub fn enumerate_remote(
req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>,
should_skip: &dyn Fn(ObjProps) -> bool,
options: EnumerateOptions
) -> (Vec<ObjProps>, Vec<ObjProps>) {
let mut folders: Vec<ObjProps> = vec![ObjProps::new()];
let mut all_folders: Vec<ObjProps> = vec![];
let mut files: Vec<ObjProps> = vec![];
@@ -10,7 +20,7 @@ pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>, d
let relative_s = match folder.relative_s {
Some(relative_s) => relative_s,
None => String::from(""),
None => options.relative_s.clone().unwrap_or(String::from("")),
};
// request folder content
@@ -34,20 +44,42 @@ pub fn enumerate_remote(req: impl Fn(&str) -> Result<Vec<ObjProps>, ApiError>, d
};
// separate folders and files in response
let mut iter = objs.iter();
// first element is not used as it is the fetched folder
let default_depth = calc_depth(iter.next().unwrap());
let d = depth.unwrap_or("0").parse::<u16>().unwrap();
let d = options.depth.clone().unwrap_or("0".to_owned()).parse::<u16>().unwrap();
let mut skip_depth = 0;
for object in iter {
if object.is_dir() {
let current_depth = calc_depth(object);
// skip children of skiped folder
if skip_depth != 0 && skip_depth < current_depth {
continue;
}
let should_skip = should_skip(object.clone());
if should_skip {
skip_depth = current_depth;
} else {
skip_depth = 0;
all_folders.push(object.clone());
}
// should get content of this folder if it is not already in this reponse
if calc_depth(object) - default_depth == d {
if current_depth - default_depth == d && !should_skip {
folders.push(object.clone());
}
all_folders.push(object.clone());
} else {
files.push(object.clone());
let current_depth = calc_depth(object);
// skip children of skiped folder
if skip_depth != 0 && skip_depth < current_depth {
continue;
}
if !should_skip(object.clone()) {
skip_depth = 0;
files.push(object.clone());
}
}
}
}