Make Val Thread Safe

[xorhex]

Rust n00b here. Trying to multithread jaq queries using rayon and ran into this - guessing the trait Send needs to be implemented for jaq_json::Val:

    json_files
        .par_iter()
        .flat_map(|file| {
            let json = match fs::read_to_string(file) {
                Ok(c) => c,
                Err(e) => {
                    eprintln!("Error reading {}: {}", file.display(), e);
                    return Vec::new();
                }
            };

            let parsed: serde_json::Value = match serde_json::from_str(&json) {
                Ok(v) => v,
                Err(e) => {
                    eprintln!("Error parsing {}: {}", file.display(), e);
                    return Vec::new();
                }
            };

            let program = File {
                code: filter,
                path: (),
            };
            let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs()));
            let arena = Arena::default();

            let modules = loader.load(&arena, program).unwrap();

            let filter = jaq_core::Compiler::default()
                .with_funs(jaq_std::funs().chain(jaq_json::funs()))
                .compile(modules)
                .unwrap();

            let inputs = RcIter::new(core::iter::empty());
            let mut output = Vec::new();

            for res in filter.run((Ctx::new([], &inputs), Val::from(parsed.to_string()))) {
                match res {
                    Ok(v) => output.push(v),
                    Err(e) => eprintln!("Runtime error in {:?}: {:?}", file, e),
                }
            }

            output
        })
        .collect()

[01mf02]

Hi @xorhex!

jaq 3.0 (alpha) should have what you need --- see jaq_json's sync feature.

By the way, if you want better file loading performance, then I recommend loading JSON files directly with jaq_json::json::parse_many, perhaps even loading the JSON file via memmap2. That avoids the detour via serde_json.

[01mf02]

If you just want to process multiple files in parallel, you could also just use something like GNU Parallel.