Added a check to see if the timing results have signifigantly changed
Update matrix-timing-tests timings [skip ci] Fixing timing test runner Update matrix-timing-tests timings Removed the seperate benchmark action
This commit is contained in:
@@ -1,72 +0,0 @@
|
||||
name: Merge-Checker
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
paths-ignore:
|
||||
- 'unit-tests/timing-results/**'
|
||||
|
||||
jobs:
|
||||
build_and_test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout source code
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
persist-credentials: true
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install dependencies (CMake + Ninja + build tools)
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake ninja-build build-essential time git
|
||||
|
||||
- name: Configure project with CMake
|
||||
run: cmake -G Ninja -S . -B build/
|
||||
|
||||
- name: Build with Ninja
|
||||
run: ninja -C build/
|
||||
|
||||
- name: Run matrix-timing-tests with per-test timing output and save results
|
||||
run: |
|
||||
mkdir -p unit-tests/timing-results
|
||||
if [ -x build/unit-tests/matrix-timing-tests ]; then
|
||||
echo "Running matrix-timing-tests with timing"
|
||||
/usr/bin/time -v build/unit-tests/matrix-timing-tests -d yes &> unit-tests/timing-results/matrix-timing-tests.txt
|
||||
else
|
||||
echo "matrix-timing-tests executable not found or not executable"
|
||||
exit 1
|
||||
fi
|
||||
- name: Commit and push timing results
|
||||
if: github.event.pull_request.head.repo.full_name == github.repository
|
||||
run: |
|
||||
git config --global user.name "ci-bot"
|
||||
git config --global user.email "ci-bot@local"
|
||||
|
||||
BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
|
||||
git stash
|
||||
echo "Checking out source branch $BRANCH_NAME"
|
||||
git fetch origin "$BRANCH_NAME"
|
||||
git checkout "$BRANCH_NAME"
|
||||
git pull
|
||||
|
||||
echo "Checking if last commit was a timing update"
|
||||
LAST_COMMIT_MSG=$(git log -1 --pretty=%B)
|
||||
|
||||
if echo "$LAST_COMMIT_MSG" | grep -q "Update matrix-timing-tests timings"; then
|
||||
echo "Last commit was a timing update, skipping commit."
|
||||
exit 0
|
||||
else
|
||||
echo "Last commit name was: $LAST_COMMIT_MSG"
|
||||
git stash pop
|
||||
fi
|
||||
|
||||
git add unit-tests/timing-results/matrix-timing-tests.txt
|
||||
|
||||
if git diff --quiet --cached; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
git commit -m "Update matrix-timing-tests timings [skip ci]"
|
||||
git push origin "$BRANCH_NAME"
|
||||
fi
|
||||
@@ -3,8 +3,6 @@ name: Merge-Checker
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["**"]
|
||||
paths-ignore:
|
||||
- 'unit-tests/timing-results/**'
|
||||
|
||||
jobs:
|
||||
build_and_test:
|
||||
@@ -38,3 +36,67 @@ jobs:
|
||||
echo "Warning: $test_exec not found or not executable"
|
||||
fi
|
||||
done
|
||||
- name: Run matrix-timing-tests
|
||||
run: |
|
||||
mkdir -p unit-tests/timing-results
|
||||
if [ -x build/unit-tests/matrix-timing-tests ]; then
|
||||
echo "Running matrix-timing-tests with timing"
|
||||
/usr/bin/time -v build/unit-tests/matrix-timing-tests -d yes &> unit-tests/timing-results/matrix-timing-tests.txt
|
||||
cat unit-tests/timing-results/matrix-timing-tests.txt
|
||||
else
|
||||
echo "matrix-timing-tests executable not found or not executable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Compare timing results
|
||||
id: check_diff
|
||||
run: |
|
||||
git show origin/${{ github.event.pull_request.head.ref }}:unit-tests/timing-results/matrix-timing-tests.txt > old.txt || echo "" > old.txt
|
||||
cp unit-tests/timing-results/matrix-timing-tests.txt new.txt
|
||||
|
||||
echo "Comparing timing results for changes ≥ 0.1s (ignoring 'Timing Tests' lines)..."
|
||||
|
||||
changed=0
|
||||
|
||||
awk -v changed_ref=/tmp/timings_changed.flag '
|
||||
BEGIN {
|
||||
change_threshold = 0.1
|
||||
}
|
||||
FILENAME == "old.txt" && /^[0-9]+\.[0-9]+ s: / {
|
||||
label = substr($0, index($0, ":") + 2)
|
||||
if (label != "Timing Tests") {
|
||||
label_times[label] = $1
|
||||
}
|
||||
}
|
||||
FILENAME == "new.txt" && /^[0-9]+\.[0-9]+ s: / {
|
||||
new_time = $1
|
||||
label = substr($0, index($0, ":") + 2)
|
||||
if (label == "Timing Tests") next
|
||||
|
||||
old_time = label_times[label]
|
||||
delta = new_time - old_time
|
||||
if (delta < 0) delta = -delta
|
||||
|
||||
if (old_time != "" && delta >= change_threshold) {
|
||||
printf "⚠️ %.3f s → %.3f s: %s (Δ=%.3f s)\n", old_time, new_time, label, delta
|
||||
system("touch " changed_ref)
|
||||
} else if (old_time == "") {
|
||||
printf "🆕 New timing entry: %.3f s: %s\n", new_time, label
|
||||
system("touch " changed_ref)
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (!system("test -f " changed_ref)) {
|
||||
exit 0
|
||||
} else {
|
||||
print "✅ Timings haven’t changed significantly (Δ < 0.1s)."
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
' old.txt new.txt
|
||||
|
||||
if [ -f /tmp/timings_changed.flag ]; then
|
||||
echo "timings_changed=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "timings_changed=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
@@ -1,32 +1,32 @@
|
||||
Randomness seeded to: 2680752211
|
||||
0.179 s: Addition
|
||||
0.179 s: Timing Tests
|
||||
0.172 s: Subtraction
|
||||
0.172 s: Timing Tests
|
||||
1.894 s: Multiplication
|
||||
1.894 s: Timing Tests
|
||||
0.128 s: Scalar Multiplication
|
||||
0.128 s: Timing Tests
|
||||
0.177 s: Element Multiply
|
||||
Randomness seeded to: 2444679151
|
||||
0.180 s: Addition
|
||||
0.180 s: Timing Tests
|
||||
0.177 s: Subtraction
|
||||
0.177 s: Timing Tests
|
||||
0.176 s: Element Divide
|
||||
0.176 s: Timing Tests
|
||||
0.150 s: Minor Matrix
|
||||
0.150 s: Timing Tests
|
||||
0.102 s: Determinant
|
||||
0.102 s: Timing Tests
|
||||
0.416 s: Matrix of Minors
|
||||
0.416 s: Timing Tests
|
||||
0.110 s: Invert
|
||||
0.110 s: Timing Tests
|
||||
0.124 s: Transpose
|
||||
0.124 s: Timing Tests
|
||||
0.189 s: Normalize
|
||||
0.189 s: Timing Tests
|
||||
1.868 s: Multiplication
|
||||
1.868 s: Timing Tests
|
||||
0.127 s: Scalar Multiplication
|
||||
0.127 s: Timing Tests
|
||||
0.173 s: Element Multiply
|
||||
0.173 s: Timing Tests
|
||||
0.178 s: Element Divide
|
||||
0.178 s: Timing Tests
|
||||
0.172 s: Minor Matrix
|
||||
0.172 s: Timing Tests
|
||||
0.103 s: Determinant
|
||||
0.103 s: Timing Tests
|
||||
0.411 s: Matrix of Minors
|
||||
0.411 s: Timing Tests
|
||||
0.109 s: Invert
|
||||
0.109 s: Timing Tests
|
||||
0.122 s: Transpose
|
||||
0.122 s: Timing Tests
|
||||
0.190 s: Normalize
|
||||
0.190 s: Timing Tests
|
||||
0.006 s: GET ROW
|
||||
0.006 s: Timing Tests
|
||||
0.234 s: GET COLUMN
|
||||
0.234 s: Timing Tests
|
||||
0.235 s: GET COLUMN
|
||||
0.235 s: Timing Tests
|
||||
===============================================================================
|
||||
test cases: 1 | 1 passed
|
||||
assertions: - none -
|
||||
@@ -34,18 +34,18 @@ assertions: - none -
|
||||
Command being timed: "build/unit-tests/matrix-timing-tests -d yes"
|
||||
User time (seconds): 4.05
|
||||
System time (seconds): 0.00
|
||||
Percent of CPU this job got: 99%
|
||||
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:04.06
|
||||
Percent of CPU this job got: 100%
|
||||
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:04.05
|
||||
Average shared text size (kbytes): 0
|
||||
Average unshared data size (kbytes): 0
|
||||
Average stack size (kbytes): 0
|
||||
Average total size (kbytes): 0
|
||||
Maximum resident set size (kbytes): 3200
|
||||
Average resident set size (kbytes): 0
|
||||
Major (requiring I/O) page faults: 186
|
||||
Minor (reclaiming a frame) page faults: 172
|
||||
Major (requiring I/O) page faults: 184
|
||||
Minor (reclaiming a frame) page faults: 171
|
||||
Voluntary context switches: 1
|
||||
Involuntary context switches: 40
|
||||
Involuntary context switches: 26
|
||||
Swaps: 0
|
||||
File system inputs: 12
|
||||
File system outputs: 1
|
||||
|
||||
Reference in New Issue
Block a user