add lit_to_bazel helper script

Also reorganize templates into scripts/ base directory

.github/workflows/scripts_test.yml

@@ -0,0 +1,31 @@
+name: Scripts tests
+permissions: read-all
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # pin@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # pin@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+      - name: Run tests
+        run: |
+          python -m pytest

.gitignore

@@ -21,3 +21,6 @@ tests/tfhe_rust_bool/end_to_end_fpga/tfhe-rs
 
 # vscode
 .vscode/**
+
+# for python files
+__pycache__

docs/content/en/docs/getting_started.md

@@ -40,7 +40,11 @@ Like above, run the following to skip tests that depend on Yosys:
 bazel test --define=HEIR_NO_YOSYS=1 --test_tag_filters=-yosys @heir//...
 ```
 
-## Optional: Run heir-opt on an mlir file
+## Run the hello-world example
+
+The `hello-world` example is a simple program that
+
+## Optional: Run a custom `heir-opt` pipeline
 
 HEIR comes with two central binaries, `heir-opt` for running optimization passes
 and dialect conversions, and `heir-translate` for backend code generation. To
@@ -63,6 +67,24 @@ bazel run //tools:heir-opt -- \
   $PWD/tests/comb_to_cggi/add_one.mlir
 ```
 
+To convert an existing lit test to a `bazel run` command for manual tweaking
+and introspection (e.g., adding `--debug` or `--mlir-print-ir-after-all` to see
+how he IR changes with each pass), use `python scripts/lit_to_bazel.py`.
+
+```bash
+# after pip installing requirements-dev.txt
+python scripts/lit_to_bazel.py tests/simd/box_blur_64x64.mlir
+```
+
+Which outputs
+
+```bash
+bazel run --noallow_analysis_cache_discard //tools:heir-opt -- \
+--secretize=entry-function=box_blur --wrap-generic --canonicalize --cse --full-loop-unroll \
+--insert-rotate --cse --canonicalize --collapse-insertion-chains \
+--canonicalize --cse /path/to/heir/tests/simd/box_blur_64x64.mlir
+```
+
 ## Developing in HEIR
 
 We use [pre-commit](https://pre-commit.com/) to manage a series of git
@@ -91,15 +113,18 @@ pre-commit run --all-files
 
 ## Creating a New Pass
 
-The `templates` folder contains Python scripts to create boilerplate for new
-conversion or (dialect-specific) transform passes.
+The `scripts/templates` folder contains Python scripts to create boilerplate
+for new conversion or (dialect-specific) transform passes. These should be used
+when the tablegen files containing existing pass definitions in the expected
+filepaths are not already present. Otherwise, you should modify the existing
+tablegen files directly.
 
 ### Conversion Pass
 
 To create a new conversion pass, run a command similar to the following:
 
 ```
-python templates/templates.py new_conversion_pass \
+python scripts/templates/templates.py new_conversion_pass \
 --source_dialect_name=CGGI \
 --source_dialect_namespace=cggi \
 --source_dialect_mnemonic=cggi \
@@ -117,7 +142,7 @@ To create a transform or rewrite pass that operates on a dialect, run a command
 similar to the following:
 
 ```
-python templates/templates.py new_dialect_transform \
+python scripts/templates/templates.py new_dialect_transform \
 --pass_name=ForgetSecrets \
 --pass_flag=forget-secrets \
 --dialect_name=Secret \
@@ -128,7 +153,7 @@ python templates/templates.py new_dialect_transform \
 If the transform does not operate from and to a specific dialect, use
 
 ```
-python templates/templates.py new_transform \
+python scripts/templates/templates.py new_transform \
 --pass_name=ForgetSecrets \
 --pass_flag=forget-secrets \
 --force=false

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.pytest.ini_options]
+norecursedirs = [
+  ".git",
+  "__pycache__",
+  "bazel",
+  "bazel-bin",
+  "bazel-heir",
+  "bazel-out",
+  "bazel-testlogs",
+  "docs",
+  "external",
+  "tests",
+  "tools",
+  "venv",
+]

requirements-dev.txt

@@ -1,3 +1,4 @@
 fire==0.5.0
 jinja2==3.1.3
 pre-commit==v3.3.3
+pytest==8.1.1

scripts/lit_to_bazel.py

@@ -0,0 +1,105 @@
+import os
+import pathlib
+from collections import deque
+
+import fire
+
+
+# a sentinel for a bash pipe
+PIPE = "|"
+OUT_REDIRECT = ">"
+IN_REDIRECT = "<"
+RUN_PREFIX = "// RUN:"
+
+def strip_run_prefix(line):
+    if RUN_PREFIX in line:
+        return line.split(RUN_PREFIX)[1]
+    return line
+
+
+def convert_to_run_commands(run_lines):
+    run_lines = deque(run_lines)
+    cmds = []
+    current_command = ""
+    while run_lines:
+        line = run_lines.popleft()
+        if RUN_PREFIX not in line:
+            continue
+
+        line = strip_run_prefix(line)
+
+        if '|' in line:
+            first, second = line.split('|', maxsplit=1)
+            current_command += " " + first.strip()
+            cmds.append(current_command.strip())
+            current_command = ""
+            cmds.append(PIPE)
+            run_lines.appendleft(RUN_PREFIX + " " + second.strip())
+            continue
+
+        # redirecting to a file implicitly ends the command on that line
+        if OUT_REDIRECT in line or IN_REDIRECT in line:
+            cmds.append(line.strip())
+            current_command = ""
+            continue
+
+        if line.strip().endswith('\\'):
+            current_command += " " + line.replace('\\', '').strip()
+            continue
+
+        current_command += line
+        cmds.append(current_command.strip())
+        current_command = ""
+
+    return cmds
+
+
+def lit_to_bazel(
+    lit_test_file: str,
+):
+    """A helper CLI that converts MLIR test files to bazel run commands.
+
+    Args:
+      lit_test_file: The lit test file that should be converted to a bazel run
+        command.
+    """
+
+    git_root = pathlib.Path(__file__).parent.parent
+    if not os.path.isdir(git_root / ".git"):
+        raise RuntimeError(f"Could not find git root, looked at {git_root}")
+
+    if not lit_test_file:
+        raise ValueError("lit_test_file must be provided")
+
+    if not os.path.isfile(lit_test_file):
+        raise ValueError("Unable to find lit_test_file '%s'" % lit_test_file)
+
+    run_lines = []
+    with open(lit_test_file, "r") as f:
+        for line in f:
+            if "// RUN:" in line:
+                run_lines.append(line)
+
+    commands = convert_to_run_commands(run_lines)
+    commands = [x for x in commands if 'FileCheck' not in x]
+    # remove consecutive and trailing pipes
+    if commands[-1] == PIPE:
+        commands.pop()
+    deduped_commands = []
+    for command in commands:
+        if command == PIPE and deduped_commands[-1] == PIPE:
+            continue
+        deduped_commands.append(command)
+
+    joined = " ".join(deduped_commands)
+    # I would consider using bazel-bin/tools/heir-opt, but the yosys
+    # requirement requires additional env vars to be set for the yosys and ABC
+    # paths, which is not yet worth doing for this script.
+    joined = joined.replace("heir-opt", "bazel run --noallow_analysis_cache_discard //tools:heir-opt --")
+    joined = joined.replace("heir-translate", f"{git_root}/bazel-bin/tools/heir-translate")
+    joined = joined.replace("%s", str(pathlib.Path(lit_test_file).absolute()))
+    print(joined)
+
+
+if __name__ == "__main__":
+    fire.Fire(lit_to_bazel)

templates/templates.py → scripts/templates/templates.py

@@ -51,7 +51,7 @@ class CLI:
   """
 
   def __init__(self):
-    git_root = pathlib.Path(__file__).parent.parent
+    git_root = pathlib.Path(__file__).parent.parent.parent
     if not os.path.isdir(git_root / ".git"):
       raise RuntimeError(f"Could not find git root, looked at {git_root}")
     self.root = git_root

scripts/test_lit_to_bazel.py

@@ -0,0 +1,78 @@
+from scripts.lit_to_bazel import convert_to_run_commands, PIPE
+
+
+def test_convert_to_run_commands_simple():
+    run_lines = [
+        "// RUN: heir-opt --canonicalize",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize",
+    ]
+
+def test_convert_to_run_commands_simple_with_filecheck():
+    run_lines = [
+        "// RUN: heir-opt --canonicalize | FileCheck %s",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize",
+        PIPE,
+        "FileCheck %s",
+    ]
+
+def test_convert_to_run_commands_simple_with_line_continuation():
+    run_lines = [
+        "// RUN: heir-opt \\",
+        "// RUN: --canonicalize | FileCheck %s",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize",
+        PIPE,
+        "FileCheck %s",
+    ]
+
+def test_convert_to_run_commands_simple_with_multiple_line_continuations():
+    run_lines = [
+        "// RUN: heir-opt \\",
+        "// RUN: --canonicalize \\",
+        "// RUN: --cse | FileCheck %s",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize --cse",
+        PIPE,
+        "FileCheck %s",
+    ]
+
+def test_convert_to_run_commands_simple_with_second_command():
+    run_lines = [
+        "// RUN: heir-opt --canonicalize > %t",
+        "// RUN: FileCheck %s < %t",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize > %t",
+        "FileCheck %s < %t",
+    ]
+
+def test_convert_to_run_commands_simple_with_non_run_garbage():
+    run_lines = [
+        "// RUN: heir-opt --canonicalize > %t",
+        "// wat",
+        "// RUN: FileCheck %s < %t",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize > %t",
+        "FileCheck %s < %t",
+    ]
+
+def test_convert_to_run_commands_with_multiple_pipes():
+    run_lines = [
+        "// RUN: heir-opt --canonicalize \\",
+        "// RUN: | heir-translate --emit-verilog \\",
+        "// RUN: | FileCheck %s",
+    ]
+    assert convert_to_run_commands(run_lines) == [
+        "heir-opt --canonicalize",
+        PIPE,
+        "heir-translate --emit-verilog",
+        PIPE,
+        "FileCheck %s",
+    ]