From d895c98d42b992db7104aab3562a136bb6e3c14c Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 13 Aug 2018 19:43:37 -0400 Subject: [PATCH] feat: handle initial scan in parallel and use a temp directory --- Cargo.lock | 49 +++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 +++ src/config/job.rs | 6 ++++++ src/main.rs | 25 +++++++++++++++--------- src/state.rs | 4 +++- 5 files changed, 77 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60f89a4..40d2eec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -188,6 +188,20 @@ dependencies = [ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "gif" version = "0.10.0" @@ -445,6 +459,16 @@ dependencies = [ "proc-macro2 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rayon" version = "1.0.2" @@ -491,6 +515,14 @@ dependencies = [ "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.9" @@ -522,11 +554,14 @@ dependencies = [ "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", "serde_regex 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -589,6 +624,15 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "thread_local" version = "0.3.5" @@ -714,6 +758,8 @@ dependencies = [ "checksum filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f" "checksum fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "c4bbbf71584aeed076100b5665ac14e3d85eeb31fdbb45fbd41ef9a682b5ec05" "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum gif 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff3414b424657317e708489d2857d9575f4403698428b040b609b9d1c1a84a2c" "checksum image 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ebdff791af04e30089bde8ad2a632b86af433b40c04db8d70ad4b21487db7a6a" "checksum inflate 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6f53b811ee8e2057ccf9643ca6b4277de90efaf5e61e55fd5254576926bb4245" @@ -743,11 +789,13 @@ dependencies = [ "checksum png 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f54b9600d584d3b8a739e1662a595fab051329eff43f20e7d8cc22872962145b" "checksum proc-macro2 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "7a17a4d77bc20d344179de803a34694c0ac7a0b3fb4384bee99783215a8e0410" "checksum quote 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ed7d650913520df631972f21e104a4fa2f9c82a14afc65d17b388a2e29731e7c" +"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" "checksum rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "df7a791f788cb4c516f0e091301a29c2b71ef680db5e644a7d68835c8ae6dbfa" "checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5bbbea44c5490a1e84357ff28b7d518b4619a159fed5d25f6c1de2d19cc42814" "checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" "checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" @@ -759,6 +807,7 @@ dependencies = [ "checksum slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d807fd58c4181bbabed77cb3b891ba9748241a552bcc5be698faaebefc54f46e" "checksum syn 0.14.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b7bfcbb0c068d0f642a0ffbd5c604965a360a61f99e8add013cef23a838614f3" "checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7" +"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" diff --git a/Cargo.toml b/Cargo.toml index 33eff7a..171cd09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,3 +17,6 @@ serde_derive = "1" serde_json = "1" serde_regex = "0.3" failure = "0.1" +tempdir = "0.3" +num_cpus = "1" +rayon = "1" diff --git a/src/config/job.rs b/src/config/job.rs index 8ef8e64..59c1d52 100644 --- a/src/config/job.rs +++ b/src/config/job.rs @@ -37,7 +37,13 @@ impl Job { for f in &mut state.file_paths { let i = image::open(&f)?; + let old_file_name = match f.file_name() { + Some(f) => f, + None => bail!("missing file name"), + }; + let old_f = f.clone(); + *f = state.temp_dir.join(old_file_name); f.set_extension(to.extension()); let mut dest = OpenOptions::new() diff --git a/src/main.rs b/src/main.rs index f4a8af4..5098499 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,11 @@ use failure::Error; use notify::{DebouncedEvent, RecursiveMode, Watcher}; -use std::{fs, path::PathBuf, sync::mpsc}; +use rayon::prelude::*; + +use tempdir::TempDir; + +use std::{fs::{self, DirEntry}, path::PathBuf, sync::mpsc}; pub type Result = std::result::Result; @@ -33,19 +37,22 @@ fn main() -> Result<()> { let (tx, rx) = mpsc::channel(); - for entry in std::fs::read_dir(&screenshots_dir)? { - let entry = entry?; - if let Err(e) = handle(&config, entry.path()) { + let temp_dir = TempDir::new("fso-")?; + let temp_path = temp_dir.path().to_owned(); + + let existing_files: Vec = std::fs::read_dir(&screenshots_dir)?.collect::>()?; + + existing_files.into_par_iter().for_each(|entry| { + if let Err(e) = handle(&config, temp_path.clone(), entry.path()) { eprintln!("{}", e); } - } + }); let mut watcher = notify::watcher( tx, Duration::milliseconds(config.options.event_delay).to_std().unwrap(), )?; - watcher.watch( screenshots_dir.to_string_lossy().to_string(), RecursiveMode::NonRecursive @@ -53,7 +60,7 @@ fn main() -> Result<()> { loop { match rx.recv() { - Ok(DebouncedEvent::Create(p)) => if let Err(e) = handle(&config, p) { + Ok(DebouncedEvent::Create(p)) => if let Err(e) = handle(&config, temp_path.clone(), p) { eprintln!("{}", e); }, Ok(_) => {}, @@ -62,7 +69,7 @@ fn main() -> Result<()> { } } -fn handle(config: &Config, p: PathBuf) -> Result<()> { +fn handle(config: &Config, temp_dir: PathBuf, p: PathBuf) -> Result<()> { let screenshots_dir = config.options.screenshots_dir.canonicalize()?; // if the path doesn't exist, ignore @@ -89,7 +96,7 @@ fn handle(config: &Config, p: PathBuf) -> Result<()> { }; // execute the jobs in the pipeline - let mut state = State::new(p, time); + let mut state = State::new(p, time, temp_dir); for job in &config.pipeline { job.execute(&config, &mut state)?; } diff --git a/src/state.rs b/src/state.rs index cc562a9..8308eb4 100644 --- a/src/state.rs +++ b/src/state.rs @@ -6,12 +6,14 @@ use std::path::PathBuf; pub struct State { pub file_paths: Vec, pub datetime: DateTime, + pub temp_dir: PathBuf, } impl State { - pub fn new(file_path: PathBuf, datetime: DateTime,) -> Self { + pub fn new(file_path: PathBuf, datetime: DateTime, temp_dir: PathBuf) -> Self { State { datetime, + temp_dir, file_paths: vec![file_path], } }