feat: handle initial scan in parallel and use a temp directory
This commit is contained in:
parent
8acae5757d
commit
d895c98d42
|
|
@ -188,6 +188,20 @@ dependencies = [
|
|||
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-zircon"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fuchsia-zircon-sys"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "gif"
|
||||
version = "0.10.0"
|
||||
|
|
@ -445,6 +459,16 @@ dependencies = [
|
|||
"proc-macro2 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.0.2"
|
||||
|
|
@ -491,6 +515,14 @@ dependencies = [
|
|||
"ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.9"
|
||||
|
|
@ -522,11 +554,14 @@ dependencies = [
|
|||
"failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"image 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"notify 4.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.71 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_regex 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -589,6 +624,15 @@ dependencies = [
|
|||
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempdir"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "0.3.5"
|
||||
|
|
@ -714,6 +758,8 @@ dependencies = [
|
|||
"checksum filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f"
|
||||
"checksum fsevent 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "c4bbbf71584aeed076100b5665ac14e3d85eeb31fdbb45fbd41ef9a682b5ec05"
|
||||
"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
|
||||
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
||||
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
|
||||
"checksum gif 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff3414b424657317e708489d2857d9575f4403698428b040b609b9d1c1a84a2c"
|
||||
"checksum image 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ebdff791af04e30089bde8ad2a632b86af433b40c04db8d70ad4b21487db7a6a"
|
||||
"checksum inflate 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6f53b811ee8e2057ccf9643ca6b4277de90efaf5e61e55fd5254576926bb4245"
|
||||
|
|
@ -743,11 +789,13 @@ dependencies = [
|
|||
"checksum png 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f54b9600d584d3b8a739e1662a595fab051329eff43f20e7d8cc22872962145b"
|
||||
"checksum proc-macro2 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "7a17a4d77bc20d344179de803a34694c0ac7a0b3fb4384bee99783215a8e0410"
|
||||
"checksum quote 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ed7d650913520df631972f21e104a4fa2f9c82a14afc65d17b388a2e29731e7c"
|
||||
"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
|
||||
"checksum rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "df7a791f788cb4c516f0e091301a29c2b71ef680db5e644a7d68835c8ae6dbfa"
|
||||
"checksum rayon-core 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b055d1e92aba6877574d8fe604a63c8b5df60f60e5982bf7ccbb1338ea527356"
|
||||
"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1"
|
||||
"checksum regex 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5bbbea44c5490a1e84357ff28b7d518b4619a159fed5d25f6c1de2d19cc42814"
|
||||
"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d"
|
||||
"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5"
|
||||
"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395"
|
||||
"checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637"
|
||||
"checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
|
||||
|
|
@ -759,6 +807,7 @@ dependencies = [
|
|||
"checksum slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d807fd58c4181bbabed77cb3b891ba9748241a552bcc5be698faaebefc54f46e"
|
||||
"checksum syn 0.14.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b7bfcbb0c068d0f642a0ffbd5c604965a360a61f99e8add013cef23a838614f3"
|
||||
"checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7"
|
||||
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
|
||||
"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963"
|
||||
"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b"
|
||||
"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
|
||||
|
|
|
|||
|
|
@ -17,3 +17,6 @@ serde_derive = "1"
|
|||
serde_json = "1"
|
||||
serde_regex = "0.3"
|
||||
failure = "0.1"
|
||||
tempdir = "0.3"
|
||||
num_cpus = "1"
|
||||
rayon = "1"
|
||||
|
|
|
|||
|
|
@ -37,7 +37,13 @@ impl Job {
|
|||
for f in &mut state.file_paths {
|
||||
let i = image::open(&f)?;
|
||||
|
||||
let old_file_name = match f.file_name() {
|
||||
Some(f) => f,
|
||||
None => bail!("missing file name"),
|
||||
};
|
||||
|
||||
let old_f = f.clone();
|
||||
*f = state.temp_dir.join(old_file_name);
|
||||
f.set_extension(to.extension());
|
||||
|
||||
let mut dest = OpenOptions::new()
|
||||
|
|
|
|||
25
src/main.rs
25
src/main.rs
|
|
@ -16,7 +16,11 @@ use failure::Error;
|
|||
|
||||
use notify::{DebouncedEvent, RecursiveMode, Watcher};
|
||||
|
||||
use std::{fs, path::PathBuf, sync::mpsc};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use tempdir::TempDir;
|
||||
|
||||
use std::{fs::{self, DirEntry}, path::PathBuf, sync::mpsc};
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
|
|
@ -33,19 +37,22 @@ fn main() -> Result<()> {
|
|||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
|
||||
for entry in std::fs::read_dir(&screenshots_dir)? {
|
||||
let entry = entry?;
|
||||
if let Err(e) = handle(&config, entry.path()) {
|
||||
let temp_dir = TempDir::new("fso-")?;
|
||||
let temp_path = temp_dir.path().to_owned();
|
||||
|
||||
let existing_files: Vec<DirEntry> = std::fs::read_dir(&screenshots_dir)?.collect::<std::result::Result<_, _>>()?;
|
||||
|
||||
existing_files.into_par_iter().for_each(|entry| {
|
||||
if let Err(e) = handle(&config, temp_path.clone(), entry.path()) {
|
||||
eprintln!("{}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut watcher = notify::watcher(
|
||||
tx,
|
||||
Duration::milliseconds(config.options.event_delay).to_std().unwrap(),
|
||||
)?;
|
||||
|
||||
|
||||
watcher.watch(
|
||||
screenshots_dir.to_string_lossy().to_string(),
|
||||
RecursiveMode::NonRecursive
|
||||
|
|
@ -53,7 +60,7 @@ fn main() -> Result<()> {
|
|||
|
||||
loop {
|
||||
match rx.recv() {
|
||||
Ok(DebouncedEvent::Create(p)) => if let Err(e) = handle(&config, p) {
|
||||
Ok(DebouncedEvent::Create(p)) => if let Err(e) = handle(&config, temp_path.clone(), p) {
|
||||
eprintln!("{}", e);
|
||||
},
|
||||
Ok(_) => {},
|
||||
|
|
@ -62,7 +69,7 @@ fn main() -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
fn handle(config: &Config, p: PathBuf) -> Result<()> {
|
||||
fn handle(config: &Config, temp_dir: PathBuf, p: PathBuf) -> Result<()> {
|
||||
let screenshots_dir = config.options.screenshots_dir.canonicalize()?;
|
||||
|
||||
// if the path doesn't exist, ignore
|
||||
|
|
@ -89,7 +96,7 @@ fn handle(config: &Config, p: PathBuf) -> Result<()> {
|
|||
};
|
||||
|
||||
// execute the jobs in the pipeline
|
||||
let mut state = State::new(p, time);
|
||||
let mut state = State::new(p, time, temp_dir);
|
||||
for job in &config.pipeline {
|
||||
job.execute(&config, &mut state)?;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ use std::path::PathBuf;
|
|||
pub struct State {
|
||||
pub file_paths: Vec<PathBuf>,
|
||||
pub datetime: DateTime<Utc>,
|
||||
pub temp_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl State {
|
||||
pub fn new(file_path: PathBuf, datetime: DateTime<Utc>,) -> Self {
|
||||
pub fn new(file_path: PathBuf, datetime: DateTime<Utc>, temp_dir: PathBuf) -> Self {
|
||||
State {
|
||||
datetime,
|
||||
temp_dir,
|
||||
file_paths: vec![file_path],
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue