idea: Introduce `cargo xtask` for our develop workflow

[Xuanwo]

Feature Description

https://github.com/matklad/cargo-xtask

cargo-xtask is a fantastic tool that integrates our workflow in Rust, the language we love and know best. We can implement whatever we want in rust. I propose to use name odev as our worklfows name.

This feature will allow us to implement our develop workflow like cargo odev release or cargo odev check-all.

Problem and Solution

OpenDAL doesn't define a develop workflow for the whole project so far. We have mant scripts located at https://github.com/apache/opendal/tree/main/scripts, and only few committers know how to use them.

In the future, we want to introduce https://github.com/apache/opendal/issues/4279, which will add more complex workflow in our developement.

We need a solid solution for us to maintain.

Additional Context

No response

Are you willing to contribute to the development of this feature?

• [X] Yes, I am willing to contribute to the development of this feature.

[tisonkun]

Here is how I practice xtask:

Some common tools can be reused.

// Copyright 2024 tison <[email protected]>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::process::Command as StdCommand;

use clap::Parser;
use clap::Subcommand;

#[derive(Parser)]
struct Command {
    #[clap(subcommand)]
    sub: SubCommand,
}

impl Command {
    fn run(self) {
        match self.sub {
            SubCommand::Build(cmd) => cmd.run(),
            SubCommand::Lint(cmd) => cmd.run(),
            SubCommand::UnitTest(cmd) => cmd.run(),
        }
    }
}

#[derive(Subcommand)]
enum SubCommand {
    #[clap(about = "Compile workspace packages.")]
    Build(CommandBuild),
    #[clap(about = "Run format and clippy checks.")]
    Lint(CommandLint),
    #[clap(name = "ut", about = "Run unit tests.")]
    UnitTest(CommandUnitTest),
}

#[derive(Parser)]
struct CommandBuild {}

impl CommandBuild {
    fn run(self) {
        run_command(make_build_cmd());
    }
}

#[derive(Parser)]
struct CommandUnitTest {
    #[arg(long, help = "Run tests serially and do not capture output.")]
    no_capture: bool,
}

impl CommandUnitTest {
    fn run(self) {
        run_command(make_test_cmd(self.no_capture));
    }
}

#[derive(Parser)]
#[clap(name = "lint")]
struct CommandLint {
    #[arg(long, help = "Automatically apply lint suggestions.")]
    fix: bool,
}

impl CommandLint {
    fn run(self) {
        run_command(make_clippy_cmd(self.fix));
        run_command(make_format_cmd(self.fix));
        run_command(make_taplo_cmd(self.fix));
        run_command(make_typos_cmd());
        run_command(make_hawkeye_cmd(self.fix));
    }
}

fn find_command(cmd: &str) -> StdCommand {
    let output = StdCommand::new("which")
        .arg(cmd)
        .output()
        .expect("broken command: which");
    if output.status.success() {
        let result = String::from_utf8_lossy(&output.stdout);
        let mut cmd = StdCommand::new(result.trim());
        cmd.current_dir(env!("CARGO_WORKSPACE_DIR"));
        cmd
    } else {
        let stdout = String::from_utf8_lossy(&output.stdout);
        let stderr = String::from_utf8_lossy(&output.stderr);
        panic!("{cmd} not found.\nstdout: {}\nstderr: {}", stdout, stderr);
    }
}

fn ensure_installed(bin: &str, crate_name: &str) {
    let output = StdCommand::new("which")
        .arg(bin)
        .output()
        .expect("broken command: which");
    if !output.status.success() {
        let mut cmd = find_command("cargo");
        cmd.args(["install", crate_name]);
        run_command(cmd);
    }
}

fn run_command(mut cmd: StdCommand) {
    println!("{cmd:?}");
    let status = cmd.status().expect("failed to execute process");
    assert!(status.success(), "command failed: {status}");
}

fn make_build_cmd() -> StdCommand {
    let mut cmd = find_command("cargo");
    cmd.args([
        "build",
        "--workspace",
        "--all-features",
        "--tests",
        "--examples",
        "--benches",
        "--bins",
    ]);
    cmd
}

fn make_test_cmd(no_capture: bool) -> StdCommand {
    ensure_installed("cargo-nextest", "cargo-nextest");
    let mut cmd = find_command("cargo");
    cmd.args(["nextest", "run", "--workspace"]);
    if no_capture {
        cmd.arg("--no-capture");
    }
    cmd
}

fn make_format_cmd(fix: bool) -> StdCommand {
    let mut cmd = find_command("cargo");
    cmd.args(["fmt", "--all"]);
    if !fix {
        cmd.arg("--check");
    }
    cmd
}

fn make_clippy_cmd(fix: bool) -> StdCommand {
    let mut cmd = find_command("cargo");
    cmd.args([
        "clippy",
        "--tests",
        "--all-features",
        "--all-targets",
        "--workspace",
    ]);
    if fix {
        cmd.args(["--allow-staged", "--allow-dirty", "--fix"]);
    } else {
        cmd.args(["--", "-D", "warnings"]);
    }
    cmd
}

fn make_hawkeye_cmd(fix: bool) -> StdCommand {
    ensure_installed("hawkeye", "hawkeye");
    let mut cmd = find_command("hawkeye");
    if fix {
        cmd.args(["format", "--fail-if-updated=false"]);
    } else {
        cmd.args(["check"]);
    }
    cmd
}

fn make_typos_cmd() -> StdCommand {
    ensure_installed("typos", "typos-cli");
    find_command("typos")
}

fn make_taplo_cmd(fix: bool) -> StdCommand {
    ensure_installed("taplo", "taplo-cli");
    let mut cmd = find_command("taplo");
    if fix {
        cmd.args(["format"]);
    } else {
        cmd.args(["format", "--check"]);
    }
    cmd
}

fn main() {
    let cmd = Command::parse();
    cmd.run()
}

[Xuanwo]

We can first migrate our scripts into cargo o.