diff --git a/.gitea/workflows/Matrix-Timing.yaml b/.gitea/workflows/Matrix-Timing.yaml deleted file mode 100644 index 88fa5bd..0000000 --- a/.gitea/workflows/Matrix-Timing.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: Merge-Checker - -on: - pull_request: - branches: ["**"] - paths-ignore: - - 'unit-tests/timing-results/**' - -jobs: - build_and_test: - runs-on: ubuntu-latest - - steps: - - name: Checkout source code - uses: actions/checkout@v3 - with: - persist-credentials: true - fetch-depth: 0 - - - name: Install dependencies (CMake + Ninja + build tools) - run: | - sudo apt-get update - sudo apt-get install -y cmake ninja-build build-essential time git - - - name: Configure project with CMake - run: cmake -G Ninja -S . -B build/ - - - name: Build with Ninja - run: ninja -C build/ - - - name: Run matrix-timing-tests with per-test timing output and save results - run: | - mkdir -p unit-tests/timing-results - if [ -x build/unit-tests/matrix-timing-tests ]; then - echo "Running matrix-timing-tests with timing" - /usr/bin/time -v build/unit-tests/matrix-timing-tests -d yes &> unit-tests/timing-results/matrix-timing-tests.txt - cat unit-tests/timing-results/matrix-timing-tests.txt - else - echo "matrix-timing-tests executable not found or not executable" - exit 1 - fi - - name: Compare timing results - id: check_diff - run: | - # Ensure the previous version exists (from git) - git show origin/${{ github.event.pull_request.head.ref }}:unit-tests/timing-results/matrix-timing-tests.txt > old.txt || echo "" > old.txt - cp unit-tests/timing-results/matrix-timing-tests.txt new.txt - - echo "Comparing timing results..." - - CHANGED=0 - awk ' - BEGIN { change_threshold=0.1 } - /^[0-9]+\.[0-9]+ s: / { - new_time = $1 - label = substr($0, index($0, ":") + 2) - if (label_times[label]) { - old_time = label_times[label] - delta = new_time - old_time - if (delta < 0) delta = -delta - if (delta >= change_threshold) { - changed = 1 - } - } else { - changed = 1 - } - } - FILENAME=="old.txt" && /^[0-9]+\.[0-9]+ s: / { - label = substr($0, index($0, ":") + 2) - label_times[label] = $1 - } - END { - exit changed == 1 ? 0 : 1 - } - ' old.txt new.txt - - - name: Commit and push timing results - if: steps.check_diff.outcome == 'success' && github.event.pull_request.head.repo.full_name == github.repository - run: | - git config --global user.name "ci-bot" - git config --global user.email "ci-bot@local" - - BRANCH_NAME="${{ github.event.pull_request.head.ref }}" - git stash - echo "Checking out source branch $BRANCH_NAME" - git fetch origin "$BRANCH_NAME" - git checkout "$BRANCH_NAME" - git pull - - echo "Checking if last commit was a timing update" - LAST_COMMIT_MSG=$(git log -1 --pretty=%B) - - if echo "$LAST_COMMIT_MSG" | grep -q "Update matrix-timing-tests timings"; then - echo "Last commit was a timing update, skipping commit." - exit 0 - else - echo "Last commit name was: $LAST_COMMIT_MSG" - git stash pop - fi - - git add unit-tests/timing-results/matrix-timing-tests.txt - - if git diff --quiet --cached; then - echo "No changes to commit" - else - git commit -m "Update matrix-timing-tests timings [skip ci]" - git push origin "$BRANCH_NAME" - fi diff --git a/.gitea/workflows/Merge-Checker.yaml b/.gitea/workflows/Merge-Checker.yaml index e8d5237..03bab54 100644 --- a/.gitea/workflows/Merge-Checker.yaml +++ b/.gitea/workflows/Merge-Checker.yaml @@ -3,8 +3,129 @@ name: Merge-Checker on: pull_request: branches: ["**"] + paths-ignore: + - 'unit-tests/timing-results/**' jobs: + Benchmarking: + runs-on: ubuntu-latest + + steps: + - name: Checkout source code + uses: actions/checkout@v3 + with: + persist-credentials: true + fetch-depth: 0 + + - name: Install dependencies (CMake + Ninja + build tools) + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build build-essential time git + + - name: Configure project with CMake + run: cmake -G Ninja -S . -B build/ + + - name: Build with Ninja + run: ninja -C build/ + + - name: Run matrix-timing-tests + run: | + mkdir -p unit-tests/timing-results + if [ -x build/unit-tests/matrix-timing-tests ]; then + echo "Running matrix-timing-tests with timing" + /usr/bin/time -v build/unit-tests/matrix-timing-tests -d yes &> unit-tests/timing-results/matrix-timing-tests.txt + cat unit-tests/timing-results/matrix-timing-tests.txt + else + echo "matrix-timing-tests executable not found or not executable" + exit 1 + fi + + - name: Compare timing results + id: check_diff + run: | + git show origin/${{ github.event.pull_request.head.ref }}:unit-tests/timing-results/matrix-timing-tests.txt > old.txt || echo "" > old.txt + cp unit-tests/timing-results/matrix-timing-tests.txt new.txt + + echo "Comparing timing results for changes ≥ 0.1s (ignoring 'Timing Tests' lines)..." + + changed=0 + + awk -v changed_ref=/tmp/timings_changed.flag ' + BEGIN { + change_threshold = 0.1 + } + FILENAME == "old.txt" && /^[0-9]+\.[0-9]+ s: / { + label = substr($0, index($0, ":") + 2) + if (label != "Timing Tests") { + label_times[label] = $1 + } + } + FILENAME == "new.txt" && /^[0-9]+\.[0-9]+ s: / { + new_time = $1 + label = substr($0, index($0, ":") + 2) + if (label == "Timing Tests") next + + old_time = label_times[label] + delta = new_time - old_time + if (delta < 0) delta = -delta + + if (old_time != "" && delta >= change_threshold) { + printf "⚠️ %.3f s → %.3f s: %s (Δ=%.3f s)\n", old_time, new_time, label, delta + system("touch " changed_ref) + } else if (old_time == "") { + printf "🆕 New timing entry: %.3f s: %s\n", new_time, label + system("touch " changed_ref) + } + } + END { + if (!system("test -f " changed_ref)) { + exit 0 + } else { + print "✅ Timings haven’t changed significantly (Δ < 0.1s)." + exit 0 + } + } + ' old.txt new.txt + + if [ -f /tmp/timings_changed.flag ]; then + echo "timings_changed=true" >> $GITHUB_OUTPUT + else + echo "timings_changed=false" >> $GITHUB_OUTPUT + fi + + - name: Commit and push timing results + if: steps.check_diff.outputs.timings_changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository + run: | + git config --global user.name "ci-bot" + git config --global user.email "ci-bot@local" + + BRANCH_NAME="${{ github.event.pull_request.head.ref }}" + git stash + echo "Checking out source branch $BRANCH_NAME" + git fetch origin "$BRANCH_NAME" + git checkout "$BRANCH_NAME" + git pull + + echo "Checking if last commit was a timing update" + LAST_COMMIT_MSG=$(git log -1 --pretty=%B) + + if echo "$LAST_COMMIT_MSG" | grep -q "Update matrix-timing-tests timings"; then + echo "Last commit was a timing update, skipping commit." + exit 0 + else + echo "Last commit name was: $LAST_COMMIT_MSG" + git stash pop + fi + + git add unit-tests/timing-results/matrix-timing-tests.txt + + if git diff --quiet --cached; then + echo "No changes to commit" + else + git commit -m "Update matrix-timing-tests timings [skip ci]" + git push origin "$BRANCH_NAME" + fi + build_and_test: runs-on: ubuntu-latest diff --git a/unit-tests/timing-results/matrix-timing-tests.txt b/unit-tests/timing-results/matrix-timing-tests.txt index e457812..a09271e 100644 --- a/unit-tests/timing-results/matrix-timing-tests.txt +++ b/unit-tests/timing-results/matrix-timing-tests.txt @@ -3,7 +3,7 @@ Randomness seeded to: 2310772973 0.174 s: Timing Tests 0.169 s: Subtraction 0.169 s: Timing Tests -1.853 s: Multiplication +20 s: Multiplication 1.853 s: Timing Tests 0.121 s: Scalar Multiplication 0.121 s: Timing Tests