From 361670e8557abfcc8d0176d3e4376d6ecfe7537e Mon Sep 17 00:00:00 2001 From: Kevin Amado Date: Wed, 26 Jan 2022 22:17:58 -0500 Subject: [PATCH] perf: parallel execution - Now we format nixpkgs in 10 seconds in my machine :) --- Cargo.lock | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + README.md | 37 ++++++++++++++++--- src/main.rs | 15 ++++++-- 4 files changed, 148 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abec61c..2c87955 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,7 @@ version = "0.1.0" dependencies = [ "clap", "indoc", + "rayon", "rnix", "rowan 0.15.3", "walkdir", @@ -45,6 +46,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "3.0.13" @@ -72,6 +79,56 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03746e0c6dd9b5d2d9132ffe0bede35fb5f815604fd371bb42599fd37bc8e483" +[[package]] +name = "crossbeam-channel" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "hashbrown" version = "0.9.1" @@ -118,6 +175,12 @@ dependencies = [ "unindent", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.114" @@ -148,6 +211,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "os_str_bytes" version = "6.0.0" @@ -157,6 +230,31 @@ dependencies = [ "memchr", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "rnix" version = "0.10.1" @@ -209,6 +307,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.136" diff --git a/Cargo.toml b/Cargo.toml index 2b473ad..9a41d1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [dependencies] clap = "3" + rayon = "1.5" rnix = "0.10" rowan = "0.15" walkdir = "2" diff --git a/README.md b/README.md index a5307a9..4991292 100644 --- a/README.md +++ b/README.md @@ -57,25 +57,31 @@ Let's get Alejandra on our systems: $ alejandra --help ``` -## Goals +## Features - ✔️ **Fast** - It's written in rust + It's written in [Rust](https://www.rust-lang.org/) and formats [Nixpkgs](https://github.com/NixOS/nixpkgs) - in under 1 minute. + in just a few seconds[^benchmark-specs]. - That's 55000 lines of Nix code per second. + | Cores | Seconds | + |:-----:|:--------: + | 1 | 40 | + | 2 | 21 | + | 4 | 15 | + | 8 | 11 | + | 16 | 10 | - ✔️ **Highly tested** Coverage currently > 80%, - aiming to a 💯% soon. + aiming to 💯% soon. - ✔️ **Comprehensive** All elements in the Nix grammar have a rule, - so there won't be portions of code unformatted. + so there won't be portions of code without style. - ✔️ **Tolerant to syntax errors** @@ -98,3 +104,22 @@ Let's get Alejandra on our systems: ## Do I need to configure anything? - No. + +## References + +- [RFC 0101 - Nix formatting](https://github.com/NixOS/rfcs/pull/101) +- [rnix-parser](https://github.com/nix-community/rnix-parser) + +## Alternatives + +- [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt) +- [nixfmt](https://github.com/serokell/nixfmt) + +[^benchmark-specs]: + + Running on a [machine](https://github.com/kamadorueda/machine) with: + + - CPU: 16 x Intel(R) Core(TM) i7-10700K + - MHz: 3800.00 + - BogoMips: 7599.80 + - Cache Size: 16384 KB diff --git a/src/main.rs b/src/main.rs index ea7aee5..2d5d98b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use rayon::prelude::*; use std::io::Read; fn main() -> std::io::Result<()> { @@ -17,9 +18,17 @@ fn main() -> std::io::Result<()> { alejandra::find::nix_files(paths.collect()); eprintln!("Formatting {} files.", paths.len()); - for path in paths { - alejandra::format::file(&config, &path)?; - eprintln!("Formatting: {}", path); + + for result in paths + .par_iter() + .map(|path| -> std::io::Result<()> { + eprintln!("Formatting: {}", &path); + alejandra::format::file(&config, &path)?; + Ok(()) + }) + .collect::>>() + { + result?; } } None => {