diff --git a/.github/workflows/scripts_test.yml b/.github/workflows/scripts_test.yml new file mode 100644 index 000000000..be40ac561 --- /dev/null +++ b/.github/workflows/scripts_test.yml @@ -0,0 +1,31 @@ +name: Scripts tests +permissions: read-all +on: + push: + branches: + - main + pull_request: + branches: + - main +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true +jobs: + build-and-test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # pin@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # pin@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Run tests + run: | + python -m pytest diff --git a/.gitignore b/.gitignore index 318187095..b72020704 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ tests/tfhe_rust_bool/end_to_end_fpga/tfhe-rs # vscode .vscode/** + +# for python files +__pycache__ diff --git a/docs/content/en/docs/getting_started.md b/docs/content/en/docs/getting_started.md index d2261c268..84f7fb76b 100644 --- a/docs/content/en/docs/getting_started.md +++ b/docs/content/en/docs/getting_started.md @@ -40,7 +40,11 @@ Like above, run the following to skip tests that depend on Yosys: bazel test --define=HEIR_NO_YOSYS=1 --test_tag_filters=-yosys @heir//... ``` -## Optional: Run heir-opt on an mlir file +## Run the hello-world example + +The `hello-world` example is a simple program that + +## Optional: Run a custom `heir-opt` pipeline HEIR comes with two central binaries, `heir-opt` for running optimization passes and dialect conversions, and `heir-translate` for backend code generation. To @@ -63,6 +67,24 @@ bazel run //tools:heir-opt -- \ $PWD/tests/comb_to_cggi/add_one.mlir ``` +To convert an existing lit test to a `bazel run` command for manual tweaking +and introspection (e.g., adding `--debug` or `--mlir-print-ir-after-all` to see +how he IR changes with each pass), use `python scripts/lit_to_bazel.py`. + +```bash +# after pip installing requirements-dev.txt +python scripts/lit_to_bazel.py tests/simd/box_blur_64x64.mlir +``` + +Which outputs + +```bash +bazel run --noallow_analysis_cache_discard //tools:heir-opt -- \ +--secretize=entry-function=box_blur --wrap-generic --canonicalize --cse --full-loop-unroll \ +--insert-rotate --cse --canonicalize --collapse-insertion-chains \ +--canonicalize --cse /path/to/heir/tests/simd/box_blur_64x64.mlir +``` + ## Developing in HEIR We use [pre-commit](https://pre-commit.com/) to manage a series of git @@ -91,15 +113,18 @@ pre-commit run --all-files ## Creating a New Pass -The `templates` folder contains Python scripts to create boilerplate for new -conversion or (dialect-specific) transform passes. +The `scripts/templates` folder contains Python scripts to create boilerplate +for new conversion or (dialect-specific) transform passes. These should be used +when the tablegen files containing existing pass definitions in the expected +filepaths are not already present. Otherwise, you should modify the existing +tablegen files directly. ### Conversion Pass To create a new conversion pass, run a command similar to the following: ``` -python templates/templates.py new_conversion_pass \ +python scripts/templates/templates.py new_conversion_pass \ --source_dialect_name=CGGI \ --source_dialect_namespace=cggi \ --source_dialect_mnemonic=cggi \ @@ -117,7 +142,7 @@ To create a transform or rewrite pass that operates on a dialect, run a command similar to the following: ``` -python templates/templates.py new_dialect_transform \ +python scripts/templates/templates.py new_dialect_transform \ --pass_name=ForgetSecrets \ --pass_flag=forget-secrets \ --dialect_name=Secret \ @@ -128,7 +153,7 @@ python templates/templates.py new_dialect_transform \ If the transform does not operate from and to a specific dialect, use ``` -python templates/templates.py new_transform \ +python scripts/templates/templates.py new_transform \ --pass_name=ForgetSecrets \ --pass_flag=forget-secrets \ --force=false diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..173b912c8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "__pycache__", + "bazel", + "bazel-bin", + "bazel-heir", + "bazel-out", + "bazel-testlogs", + "docs", + "external", + "tests", + "tools", + "venv", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 29ff28d91..62f7cf6df 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ fire==0.5.0 jinja2==3.1.3 pre-commit==v3.3.3 +pytest==8.1.1 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/lit_to_bazel.py b/scripts/lit_to_bazel.py new file mode 100644 index 000000000..a57a4e2d5 --- /dev/null +++ b/scripts/lit_to_bazel.py @@ -0,0 +1,105 @@ +import os +import pathlib +from collections import deque + +import fire + + +# a sentinel for a bash pipe +PIPE = "|" +OUT_REDIRECT = ">" +IN_REDIRECT = "<" +RUN_PREFIX = "// RUN:" + +def strip_run_prefix(line): + if RUN_PREFIX in line: + return line.split(RUN_PREFIX)[1] + return line + + +def convert_to_run_commands(run_lines): + run_lines = deque(run_lines) + cmds = [] + current_command = "" + while run_lines: + line = run_lines.popleft() + if RUN_PREFIX not in line: + continue + + line = strip_run_prefix(line) + + if '|' in line: + first, second = line.split('|', maxsplit=1) + current_command += " " + first.strip() + cmds.append(current_command.strip()) + current_command = "" + cmds.append(PIPE) + run_lines.appendleft(RUN_PREFIX + " " + second.strip()) + continue + + # redirecting to a file implicitly ends the command on that line + if OUT_REDIRECT in line or IN_REDIRECT in line: + cmds.append(line.strip()) + current_command = "" + continue + + if line.strip().endswith('\\'): + current_command += " " + line.replace('\\', '').strip() + continue + + current_command += line + cmds.append(current_command.strip()) + current_command = "" + + return cmds + + +def lit_to_bazel( + lit_test_file: str, +): + """A helper CLI that converts MLIR test files to bazel run commands. + + Args: + lit_test_file: The lit test file that should be converted to a bazel run + command. + """ + + git_root = pathlib.Path(__file__).parent.parent + if not os.path.isdir(git_root / ".git"): + raise RuntimeError(f"Could not find git root, looked at {git_root}") + + if not lit_test_file: + raise ValueError("lit_test_file must be provided") + + if not os.path.isfile(lit_test_file): + raise ValueError("Unable to find lit_test_file '%s'" % lit_test_file) + + run_lines = [] + with open(lit_test_file, "r") as f: + for line in f: + if "// RUN:" in line: + run_lines.append(line) + + commands = convert_to_run_commands(run_lines) + commands = [x for x in commands if 'FileCheck' not in x] + # remove consecutive and trailing pipes + if commands[-1] == PIPE: + commands.pop() + deduped_commands = [] + for command in commands: + if command == PIPE and deduped_commands[-1] == PIPE: + continue + deduped_commands.append(command) + + joined = " ".join(deduped_commands) + # I would consider using bazel-bin/tools/heir-opt, but the yosys + # requirement requires additional env vars to be set for the yosys and ABC + # paths, which is not yet worth doing for this script. + joined = joined.replace("heir-opt", "bazel run --noallow_analysis_cache_discard //tools:heir-opt --") + joined = joined.replace("heir-translate", f"{git_root}/bazel-bin/tools/heir-translate") + joined = joined.replace("%s", str(pathlib.Path(lit_test_file).absolute())) + print(joined) + + +if __name__ == "__main__": + fire.Fire(lit_to_bazel) diff --git a/templates/Conversion/include/BUILD.jinja b/scripts/templates/Conversion/include/BUILD.jinja similarity index 100% rename from templates/Conversion/include/BUILD.jinja rename to scripts/templates/Conversion/include/BUILD.jinja diff --git a/templates/Conversion/include/ConversionPass.h.jinja b/scripts/templates/Conversion/include/ConversionPass.h.jinja similarity index 100% rename from templates/Conversion/include/ConversionPass.h.jinja rename to scripts/templates/Conversion/include/ConversionPass.h.jinja diff --git a/templates/Conversion/include/ConversionPass.td.jinja b/scripts/templates/Conversion/include/ConversionPass.td.jinja similarity index 100% rename from templates/Conversion/include/ConversionPass.td.jinja rename to scripts/templates/Conversion/include/ConversionPass.td.jinja diff --git a/templates/Conversion/lib/BUILD.jinja b/scripts/templates/Conversion/lib/BUILD.jinja similarity index 100% rename from templates/Conversion/lib/BUILD.jinja rename to scripts/templates/Conversion/lib/BUILD.jinja diff --git a/templates/Conversion/lib/ConversionPass.cpp.jinja b/scripts/templates/Conversion/lib/ConversionPass.cpp.jinja similarity index 100% rename from templates/Conversion/lib/ConversionPass.cpp.jinja rename to scripts/templates/Conversion/lib/ConversionPass.cpp.jinja diff --git a/templates/Dialect/include/Attributes.h.jinja b/scripts/templates/Dialect/include/Attributes.h.jinja similarity index 100% rename from templates/Dialect/include/Attributes.h.jinja rename to scripts/templates/Dialect/include/Attributes.h.jinja diff --git a/templates/Dialect/include/Attributes.td.jinja b/scripts/templates/Dialect/include/Attributes.td.jinja similarity index 100% rename from templates/Dialect/include/Attributes.td.jinja rename to scripts/templates/Dialect/include/Attributes.td.jinja diff --git a/templates/Dialect/include/BUILD.jinja b/scripts/templates/Dialect/include/BUILD.jinja similarity index 100% rename from templates/Dialect/include/BUILD.jinja rename to scripts/templates/Dialect/include/BUILD.jinja diff --git a/templates/Dialect/include/Dialect.h.jinja b/scripts/templates/Dialect/include/Dialect.h.jinja similarity index 100% rename from templates/Dialect/include/Dialect.h.jinja rename to scripts/templates/Dialect/include/Dialect.h.jinja diff --git a/templates/Dialect/include/Dialect.td.jinja b/scripts/templates/Dialect/include/Dialect.td.jinja similarity index 100% rename from templates/Dialect/include/Dialect.td.jinja rename to scripts/templates/Dialect/include/Dialect.td.jinja diff --git a/templates/Dialect/include/Ops.h.jinja b/scripts/templates/Dialect/include/Ops.h.jinja similarity index 100% rename from templates/Dialect/include/Ops.h.jinja rename to scripts/templates/Dialect/include/Ops.h.jinja diff --git a/templates/Dialect/include/Ops.td.jinja b/scripts/templates/Dialect/include/Ops.td.jinja similarity index 100% rename from templates/Dialect/include/Ops.td.jinja rename to scripts/templates/Dialect/include/Ops.td.jinja diff --git a/templates/Dialect/include/Types.h.jinja b/scripts/templates/Dialect/include/Types.h.jinja similarity index 100% rename from templates/Dialect/include/Types.h.jinja rename to scripts/templates/Dialect/include/Types.h.jinja diff --git a/templates/Dialect/include/Types.td.jinja b/scripts/templates/Dialect/include/Types.td.jinja similarity index 100% rename from templates/Dialect/include/Types.td.jinja rename to scripts/templates/Dialect/include/Types.td.jinja diff --git a/templates/Dialect/lib/Attributes.cpp.jinja b/scripts/templates/Dialect/lib/Attributes.cpp.jinja similarity index 100% rename from templates/Dialect/lib/Attributes.cpp.jinja rename to scripts/templates/Dialect/lib/Attributes.cpp.jinja diff --git a/templates/Dialect/lib/BUILD.jinja b/scripts/templates/Dialect/lib/BUILD.jinja similarity index 100% rename from templates/Dialect/lib/BUILD.jinja rename to scripts/templates/Dialect/lib/BUILD.jinja diff --git a/templates/Dialect/lib/Dialect.cpp.jinja b/scripts/templates/Dialect/lib/Dialect.cpp.jinja similarity index 100% rename from templates/Dialect/lib/Dialect.cpp.jinja rename to scripts/templates/Dialect/lib/Dialect.cpp.jinja diff --git a/templates/Dialect/lib/Ops.cpp.jinja b/scripts/templates/Dialect/lib/Ops.cpp.jinja similarity index 100% rename from templates/Dialect/lib/Ops.cpp.jinja rename to scripts/templates/Dialect/lib/Ops.cpp.jinja diff --git a/templates/Dialect/lib/Types.cpp.jinja b/scripts/templates/Dialect/lib/Types.cpp.jinja similarity index 100% rename from templates/Dialect/lib/Types.cpp.jinja rename to scripts/templates/Dialect/lib/Types.cpp.jinja diff --git a/templates/DialectTransforms/include/BUILD.jinja b/scripts/templates/DialectTransforms/include/BUILD.jinja similarity index 100% rename from templates/DialectTransforms/include/BUILD.jinja rename to scripts/templates/DialectTransforms/include/BUILD.jinja diff --git a/templates/DialectTransforms/include/Pass.h.jinja b/scripts/templates/DialectTransforms/include/Pass.h.jinja similarity index 100% rename from templates/DialectTransforms/include/Pass.h.jinja rename to scripts/templates/DialectTransforms/include/Pass.h.jinja diff --git a/templates/DialectTransforms/include/Passes.h.jinja b/scripts/templates/DialectTransforms/include/Passes.h.jinja similarity index 100% rename from templates/DialectTransforms/include/Passes.h.jinja rename to scripts/templates/DialectTransforms/include/Passes.h.jinja diff --git a/templates/DialectTransforms/include/Passes.td.jinja b/scripts/templates/DialectTransforms/include/Passes.td.jinja similarity index 100% rename from templates/DialectTransforms/include/Passes.td.jinja rename to scripts/templates/DialectTransforms/include/Passes.td.jinja diff --git a/templates/DialectTransforms/lib/BUILD.jinja b/scripts/templates/DialectTransforms/lib/BUILD.jinja similarity index 100% rename from templates/DialectTransforms/lib/BUILD.jinja rename to scripts/templates/DialectTransforms/lib/BUILD.jinja diff --git a/templates/DialectTransforms/lib/Pass.cpp.jinja b/scripts/templates/DialectTransforms/lib/Pass.cpp.jinja similarity index 100% rename from templates/DialectTransforms/lib/Pass.cpp.jinja rename to scripts/templates/DialectTransforms/lib/Pass.cpp.jinja diff --git a/templates/Transforms/include/BUILD.jinja b/scripts/templates/Transforms/include/BUILD.jinja similarity index 100% rename from templates/Transforms/include/BUILD.jinja rename to scripts/templates/Transforms/include/BUILD.jinja diff --git a/templates/Transforms/include/Pass.h.jinja b/scripts/templates/Transforms/include/Pass.h.jinja similarity index 100% rename from templates/Transforms/include/Pass.h.jinja rename to scripts/templates/Transforms/include/Pass.h.jinja diff --git a/templates/Transforms/include/Pass.td.jinja b/scripts/templates/Transforms/include/Pass.td.jinja similarity index 100% rename from templates/Transforms/include/Pass.td.jinja rename to scripts/templates/Transforms/include/Pass.td.jinja diff --git a/templates/Transforms/lib/BUILD.jinja b/scripts/templates/Transforms/lib/BUILD.jinja similarity index 100% rename from templates/Transforms/lib/BUILD.jinja rename to scripts/templates/Transforms/lib/BUILD.jinja diff --git a/templates/Transforms/lib/Pass.cpp.jinja b/scripts/templates/Transforms/lib/Pass.cpp.jinja similarity index 100% rename from templates/Transforms/lib/Pass.cpp.jinja rename to scripts/templates/Transforms/lib/Pass.cpp.jinja diff --git a/templates/templates.py b/scripts/templates/templates.py similarity index 99% rename from templates/templates.py rename to scripts/templates/templates.py index 597344dd7..c0000ecf4 100644 --- a/templates/templates.py +++ b/scripts/templates/templates.py @@ -51,7 +51,7 @@ class CLI: """ def __init__(self): - git_root = pathlib.Path(__file__).parent.parent + git_root = pathlib.Path(__file__).parent.parent.parent if not os.path.isdir(git_root / ".git"): raise RuntimeError(f"Could not find git root, looked at {git_root}") self.root = git_root diff --git a/scripts/test_lit_to_bazel.py b/scripts/test_lit_to_bazel.py new file mode 100644 index 000000000..c1bf82af5 --- /dev/null +++ b/scripts/test_lit_to_bazel.py @@ -0,0 +1,78 @@ +from scripts.lit_to_bazel import convert_to_run_commands, PIPE + + +def test_convert_to_run_commands_simple(): + run_lines = [ + "// RUN: heir-opt --canonicalize", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize", + ] + +def test_convert_to_run_commands_simple_with_filecheck(): + run_lines = [ + "// RUN: heir-opt --canonicalize | FileCheck %s", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize", + PIPE, + "FileCheck %s", + ] + +def test_convert_to_run_commands_simple_with_line_continuation(): + run_lines = [ + "// RUN: heir-opt \\", + "// RUN: --canonicalize | FileCheck %s", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize", + PIPE, + "FileCheck %s", + ] + +def test_convert_to_run_commands_simple_with_multiple_line_continuations(): + run_lines = [ + "// RUN: heir-opt \\", + "// RUN: --canonicalize \\", + "// RUN: --cse | FileCheck %s", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize --cse", + PIPE, + "FileCheck %s", + ] + +def test_convert_to_run_commands_simple_with_second_command(): + run_lines = [ + "// RUN: heir-opt --canonicalize > %t", + "// RUN: FileCheck %s < %t", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize > %t", + "FileCheck %s < %t", + ] + +def test_convert_to_run_commands_simple_with_non_run_garbage(): + run_lines = [ + "// RUN: heir-opt --canonicalize > %t", + "// wat", + "// RUN: FileCheck %s < %t", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize > %t", + "FileCheck %s < %t", + ] + +def test_convert_to_run_commands_with_multiple_pipes(): + run_lines = [ + "// RUN: heir-opt --canonicalize \\", + "// RUN: | heir-translate --emit-verilog \\", + "// RUN: | FileCheck %s", + ] + assert convert_to_run_commands(run_lines) == [ + "heir-opt --canonicalize", + PIPE, + "heir-translate --emit-verilog", + PIPE, + "FileCheck %s", + ]