Added a check to see if the timing results have signifigantly changed

Update matrix-timing-tests timings [skip ci]

Fixing timing test runner

Update matrix-timing-tests timings

Removed the seperate benchmark action

.gitea/workflows/Merge-Checker.yaml

@@ -5,22 +5,98 @@ on:
     branches: ["**"]
 
 jobs:
-  build:
+  build_and_test:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout source code
         uses: actions/checkout@v3
+        with:
+          persist-credentials: true
+          fetch-depth: 0
 
-      - name: Install dependencies (CMake + Ninja + Compiler)
+      - name: Install dependencies (CMake + Ninja + build tools)
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake ninja-build build-essential
+          sudo apt-get install -y cmake ninja-build build-essential time git
 
       - name: Configure project with CMake
-        run: |
+        run: cmake -G Ninja -S . -B build/
-          cmake -G Ninja -S . -B build/
 
       - name: Build with Ninja
+        run: ninja -C build/
+
+      - name: Run all unit tests except matrix-timing-tests
         run: |
-          ninja -C build/
+          for test_exec in build/unit-tests/matrix-tests build/unit-tests/quaternion-tests build/unit-tests/vector-3d-tests; do
+            if [ -x "$test_exec" ]; then
+              echo "Running $test_exec"
+              "$test_exec"
+            else
+              echo "Warning: $test_exec not found or not executable"
+            fi
+          done
+      - name: Run matrix-timing-tests
+        run: |
+          mkdir -p unit-tests/timing-results
+          if [ -x build/unit-tests/matrix-timing-tests ]; then
+            echo "Running matrix-timing-tests with timing"
+            /usr/bin/time -v build/unit-tests/matrix-timing-tests -d yes &> unit-tests/timing-results/matrix-timing-tests.txt
+            cat unit-tests/timing-results/matrix-timing-tests.txt
+          else
+            echo "matrix-timing-tests executable not found or not executable"
+            exit 1
+          fi
+
+      - name: Compare timing results
+        id: check_diff
+        run: |
+          git show origin/${{ github.event.pull_request.head.ref }}:unit-tests/timing-results/matrix-timing-tests.txt > old.txt || echo "" > old.txt
+          cp unit-tests/timing-results/matrix-timing-tests.txt new.txt
+
+          echo "Comparing timing results for changes ≥ 0.1s (ignoring 'Timing Tests' lines)..."
+
+          changed=0
+
+          awk -v changed_ref=/tmp/timings_changed.flag '
+            BEGIN {
+              change_threshold = 0.1
+            }
+            FILENAME == "old.txt" && /^[0-9]+\.[0-9]+ s: / {
+              label = substr($0, index($0, ":") + 2)
+              if (label != "Timing Tests") {
+                label_times[label] = $1
+              }
+            }
+            FILENAME == "new.txt" && /^[0-9]+\.[0-9]+ s: / {
+              new_time = $1
+              label = substr($0, index($0, ":") + 2)
+              if (label == "Timing Tests") next
+
+              old_time = label_times[label]
+              delta = new_time - old_time
+              if (delta < 0) delta = -delta
+
+              if (old_time != "" && delta >= change_threshold) {
+                printf "⚠️  %.3f s → %.3f s: %s (Δ=%.3f s)\n", old_time, new_time, label, delta
+                system("touch " changed_ref)
+              } else if (old_time == "") {
+                printf "🆕 New timing entry: %.3f s: %s\n", new_time, label
+                system("touch " changed_ref)
+              }
+            }
+            END {
+              if (!system("test -f " changed_ref)) {
+                exit 0
+              } else {
+                print "✅ Timings haven’t changed significantly (Δ < 0.1s)."
+                exit 0
+              }
+            }
+          ' old.txt new.txt
+
+          if [ -f /tmp/timings_changed.flag ]; then
+            echo "timings_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "timings_changed=false" >> $GITHUB_OUTPUT
+          fi

README.md

@@ -1 +1,5 @@
+# Introduction
 This matrix math library is focused on embedded development and avoids any heap memory allocation unless you explicitly ask for it.
+It uses templates to pre-allocate matrices on the stack.
+
+There are still several operations that are works in progress

unit-tests/CMakeLists.txt

@@ -16,6 +16,15 @@ target_link_libraries(matrix-tests
     Catch2::Catch2WithMain
 )
 
+# matrix timing tests
+add_executable(matrix-timing-tests matrix-timing-tests.cpp)
+ 
+target_link_libraries(matrix-timing-tests 
+    PRIVATE 
+    matrix
+    Catch2::Catch2WithMain
+)
+
 # Vector 3D Tests
 add_executable(vector-3d-tests vector-tests.cpp)
 

unit-tests/matrix-test-timings.txt

@@ -1,14 +0,0 @@
-Addition: 0.419 s
-Subtraction: 0.421 s
-Multiplication: 3.297 s
-Scalar Multiplication: 0.329 s
-Element Multiply: 0.306 s
-Element Divide: 0.302 s
-Minor Matrix: 0.331 s
-Determinant: 0.177 s
-Matrix of Minors: 0.766 s
-Invert: 0.183 s
-Transpose: 0.215 s
-Normalize: 0.315 s
-GET ROW: 0.008 s
-GET COLUMN: 0.43 s

unit-tests/matrix-tests.cpp

@@ -10,15 +10,13 @@
 #include <cmath>
 #include <iostream>
 
-TEST_CASE("Elementary Matrix Operations", "Matrix")
+TEST_CASE("Elementary Matrix Operations", "Matrix") {
-{
   std::array<float, 4> arr2{5, 6, 7, 8};
   Matrix<2, 2> mat1{1, 2, 3, 4};
   Matrix<2, 2> mat2{arr2};
   Matrix<2, 2> mat3{};
 
-  SECTION("Initialization")
+  SECTION("Initialization") {
-  {
     // array initialization
     REQUIRE(mat1.Get(0, 0) == 1);
     REQUIRE(mat1.Get(0, 1) == 2);
@@ -40,17 +38,14 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     // large matrix
     Matrix<255, 255> mat6{};
     mat6.Fill(4);
-    for (uint8_t row{0}; row < 255; row++)
+    for (uint8_t row{0}; row < 255; row++) {
-    {
+      for (uint8_t column{0}; column < 255; column++) {
-      for (uint8_t column{0}; column < 255; column++)
-      {
         REQUIRE(mat6.Get(row, column) == 4);
       }
     }
   }
 
-  SECTION("Fill")
+  SECTION("Fill") {
-  {
     mat1.Fill(0);
     REQUIRE(mat1.Get(0, 0) == 0);
     REQUIRE(mat1.Get(0, 1) == 0);
@@ -70,12 +65,10 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat3.Get(1, 1) == -20);
   }
 
-  SECTION("Addition")
+  SECTION("Addition") {
-  {
     std::string strBuf1 = "";
     mat1.ToString(strBuf1);
-    std::cout << "Matrix 1:\n"
+    std::cout << "Matrix 1:\n" << strBuf1 << std::endl;
-              << strBuf1 << std::endl;
 
     mat1.Add(mat2, mat3);
 
@@ -93,8 +86,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat3.Get(1, 1) == 12);
   }
 
-  SECTION("Subtraction")
+  SECTION("Subtraction") {
-  {
     mat1.Sub(mat2, mat3);
 
     REQUIRE(mat3.Get(0, 0) == -4);
@@ -111,8 +103,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat3.Get(1, 1) == -4);
   }
 
-  SECTION("Multiplication")
+  SECTION("Multiplication") {
-  {
     mat1.Mult(mat2, mat3);
 
     REQUIRE(mat3.Get(0, 0) == 19);
@@ -131,8 +122,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     // TODO: You need to add non-square multiplications to this.
   }
 
-  SECTION("Scalar Multiplication")
+  SECTION("Scalar Multiplication") {
-  {
     mat1.Mult(2, mat3);
 
     REQUIRE(mat3.Get(0, 0) == 2);
@@ -141,8 +131,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat3.Get(1, 1) == 8);
   }
 
-  SECTION("Element Multiply")
+  SECTION("Element Multiply") {
-  {
     mat1.ElementMultiply(mat2, mat3);
 
     REQUIRE(mat3.Get(0, 0) == 5);
@@ -151,8 +140,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat3.Get(1, 1) == 32);
   }
 
-  SECTION("Element Divide")
+  SECTION("Element Divide") {
-  {
     mat1.ElementDivide(mat2, mat3);
 
     REQUIRE_THAT(mat3.Get(0, 0), Catch::Matchers::WithinRel(0.2f, 1e-6f));
@@ -161,8 +149,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE_THAT(mat3.Get(1, 1), Catch::Matchers::WithinRel(0.5f, 1e-6f));
   }
 
-  SECTION("Minor Matrix")
+  SECTION("Minor Matrix") {
-  {
     // what about matrices of 0,0 or 1,1?
     // minor matrix for 2x2 matrix
     Matrix<1, 1> minorMat1{};
@@ -198,8 +185,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(minorMat4.Get(1, 1) == 5);
   }
 
-  SECTION("Determinant")
+  SECTION("Determinant") {
-  {
     float det1 = mat1.Det();
 
     REQUIRE_THAT(det1, Catch::Matchers::WithinRel(-2.0F, 1e-6f));
@@ -216,8 +202,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE_THAT(det5, Catch::Matchers::WithinRel(6.0F, 1e-6f));
   }
 
-  SECTION("Matrix of Minors")
+  SECTION("Matrix of Minors") {
-  {
     mat1.MatrixOfMinors(mat3);
     REQUIRE_THAT(mat3.Get(0, 0), Catch::Matchers::WithinRel(4.0F, 1e-6f));
     REQUIRE_THAT(mat3.Get(0, 1), Catch::Matchers::WithinRel(3.0F, 1e-6f));
@@ -238,8 +223,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE_THAT(mat5.Get(2, 2), Catch::Matchers::WithinRel(-3.0F, 1e-6f));
   }
 
-  SECTION("Invert")
+  SECTION("Invert") {
-  {
     mat3 = mat1.Invert();
     REQUIRE_THAT(mat3.Get(0, 0), Catch::Matchers::WithinRel(-2.0F, 1e-6f));
     REQUIRE_THAT(mat3.Get(0, 1), Catch::Matchers::WithinRel(1.0F, 1e-6f));
@@ -247,8 +231,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE_THAT(mat3.Get(1, 1), Catch::Matchers::WithinRel(-0.5F, 1e-6f));
   };
 
-  SECTION("Transpose")
+  SECTION("Transpose") {
-  {
     // transpose a square matrix
     mat3 = mat1.Transpose();
 
@@ -271,8 +254,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat5.Get(2, 1) == 6);
   }
 
-  SECTION("Normalize")
+  SECTION("Normalize") {
-  {
     mat1.Normalize(mat3);
 
     float sqrt_30{sqrt(30)};
@@ -292,8 +274,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
                  Catch::Matchers::WithinRel(0.957591346325f, 1e-6f));
   }
 
-  SECTION("GET ROW")
+  SECTION("GET ROW") {
-  {
     Matrix<1, 2> mat1Rows{};
     mat1.GetRow(0, mat1Rows);
     REQUIRE(mat1Rows.Get(0, 0) == 1);
@@ -304,8 +285,7 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat1Rows.Get(0, 1) == 4);
   }
 
-  SECTION("GET COLUMN")
+  SECTION("GET COLUMN") {
-  {
     Matrix<2, 1> mat1Columns{};
     mat1.GetColumn(0, mat1Columns);
     REQUIRE(mat1Columns.Get(0, 0) == 1);
@@ -316,12 +296,8 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat1Columns.Get(1, 0) == 4);
   }
 
-  SECTION("Get Sub-Matrices")
+  SECTION("Get Sub-Matrices") {
-  {
+    Matrix<3, 3> mat4{1, 2, 3, 4, 5, 6, 7, 8, 9};
-    Matrix<3, 3> mat4{
-        1, 2, 3,
-        4, 5, 6,
-        7, 8, 9};
 
     Matrix<2, 2> mat5 = mat4.SubMatrix<2, 2, 0, 0>();
 
@@ -347,12 +323,8 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat7.Get(0, 2) == 3);
   }
 
-  SECTION("Set Sub-Matrices")
+  SECTION("Set Sub-Matrices") {
-  {
+    Matrix<3, 3> startMatrix{1, 2, 3, 4, 5, 6, 7, 8, 9};
-    Matrix<3, 3> startMatrix{
-        1, 2, 3,
-        4, 5, 6,
-        7, 8, 9};
     Matrix<3, 3> mat4 = startMatrix;
 
     Matrix<2, 2> mat5{10, 11, 12, 13};
@@ -381,144 +353,4 @@ TEST_CASE("Elementary Matrix Operations", "Matrix")
     REQUIRE(mat4.Get(0, 1) == 11);
     REQUIRE(mat4.Get(0, 2) == 12);
   }
-}
-
-// basically re-run all of the previous tests with huge matrices and time the
-// results.
-TEST_CASE("Timing Tests", "Matrix")
-{
-  std::array<float, 50 * 50> arr1{};
-  for (uint16_t i{0}; i < 50 * 50; i++)
-  {
-    arr1[i] = i;
-  }
-  std::array<float, 50 * 50> arr2{5, 6, 7, 8};
-  for (uint16_t i{50 * 50}; i < 2 * 50 * 50; i++)
-  {
-    arr2[i] = i;
-  }
-  Matrix<50, 50> mat1{arr1};
-  Matrix<50, 50> mat2{arr2};
-  Matrix<50, 50> mat3{};
-
-  // A smaller matrix to use for really badly optimized operations
-  Matrix<4, 4> mat4{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
-  Matrix<4, 4> mat5{};
-
-  SECTION("Addition")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat3 = mat1 + mat2;
-    }
-  }
-
-  SECTION("Subtraction")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat3 = mat1 - mat2;
-    }
-  }
-
-  SECTION("Multiplication")
-  {
-    for (uint32_t i{0}; i < 1000; i++)
-    {
-      mat3 = mat1 * mat2;
-    }
-  }
-
-  SECTION("Scalar Multiplication")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat3 = mat1 * 3;
-    }
-  }
-
-  SECTION("Element Multiply")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat1.ElementMultiply(mat2, mat3);
-    }
-  }
-
-  SECTION("Element Divide")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat1.ElementDivide(mat2, mat3);
-    }
-  }
-
-  SECTION("Minor Matrix")
-  {
-    // what about matrices of 0,0 or 1,1?
-    // minor matrix for 2x2 matrix
-    Matrix<49, 49> minorMat1{};
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat1.MinorMatrix(minorMat1, 0, 0);
-    }
-  }
-
-  SECTION("Determinant")
-  {
-    for (uint32_t i{0}; i < 100000; i++)
-    {
-      float det1 = mat4.Det();
-    }
-  }
-
-  SECTION("Matrix of Minors")
-  {
-    for (uint32_t i{0}; i < 100000; i++)
-    {
-      mat4.MatrixOfMinors(mat5);
-    }
-  }
-
-  SECTION("Invert")
-  {
-    for (uint32_t i{0}; i < 100000; i++)
-    {
-      mat5 = mat4.Invert();
-    }
-  };
-
-  SECTION("Transpose")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat3 = mat1.Transpose();
-    }
-  }
-
-  SECTION("Normalize")
-  {
-    for (uint32_t i{0}; i < 10000; i++)
-    {
-      mat1.Normalize(mat3);
-    }
-  }
-
-  SECTION("GET ROW")
-  {
-    Matrix<1, 50> mat1Rows{};
-    for (uint32_t i{0}; i < 1000000; i++)
-    {
-      mat1.GetRow(0, mat1Rows);
-    }
-  }
-
-  SECTION("GET COLUMN")
-  {
-    Matrix<50, 1> mat1Columns{};
-    for (uint32_t i{0}; i < 1000000; i++)
-    {
-      mat1.GetColumn(0, mat1Columns);
-    }
-  }
 }

unit-tests/matrix-timing-tests.cpp

@@ -0,0 +1,119 @@
+// include the unit test framework first
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_floating_point.hpp>
+
+// include the module you're going to test next
+#include "Matrix.hpp"
+
+// any other libraries
+#include <array>
+#include <cmath>
+
+// basically re-run all of the matrix tests with huge matrices and time the
+// results.
+TEST_CASE("Timing Tests", "Matrix") {
+  std::array<float, 50 * 50> arr1{};
+  for (uint16_t i{0}; i < 50 * 50; i++) {
+    arr1[i] = i;
+  }
+  std::array<float, 50 * 50> arr2{5, 6, 7, 8};
+  for (uint16_t i{50 * 50}; i < 2 * 50 * 50; i++) {
+    arr2[i] = i;
+  }
+  Matrix<50, 50> mat1{arr1};
+  Matrix<50, 50> mat2{arr2};
+  Matrix<50, 50> mat3{};
+
+  // A smaller matrix to use for really badly optimized operations
+  Matrix<4, 4> mat4{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+  Matrix<4, 4> mat5{};
+
+  SECTION("Addition") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat3 = mat1 + mat2;
+    }
+  }
+
+  SECTION("Subtraction") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat3 = mat1 - mat2;
+    }
+  }
+
+  SECTION("Multiplication") {
+    for (uint32_t i{0}; i < 1000; i++) {
+      mat3 = mat1 * mat2;
+    }
+  }
+
+  SECTION("Scalar Multiplication") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat3 = mat1 * 3;
+    }
+  }
+
+  SECTION("Element Multiply") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat1.ElementMultiply(mat2, mat3);
+    }
+  }
+
+  SECTION("Element Divide") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat1.ElementDivide(mat2, mat3);
+    }
+  }
+
+  SECTION("Minor Matrix") {
+    // what about matrices of 0,0 or 1,1?
+    // minor matrix for 2x2 matrix
+    Matrix<49, 49> minorMat1{};
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat1.MinorMatrix(minorMat1, 0, 0);
+    }
+  }
+
+  SECTION("Determinant") {
+    for (uint32_t i{0}; i < 100000; i++) {
+      float det1 = mat4.Det();
+    }
+  }
+
+  SECTION("Matrix of Minors") {
+    for (uint32_t i{0}; i < 100000; i++) {
+      mat4.MatrixOfMinors(mat5);
+    }
+  }
+
+  SECTION("Invert") {
+    for (uint32_t i{0}; i < 100000; i++) {
+      mat5 = mat4.Invert();
+    }
+  };
+
+  SECTION("Transpose") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat3 = mat1.Transpose();
+    }
+  }
+
+  SECTION("Normalize") {
+    for (uint32_t i{0}; i < 10000; i++) {
+      mat1.Normalize(mat3);
+    }
+  }
+
+  SECTION("GET ROW") {
+    Matrix<1, 50> mat1Rows{};
+    for (uint32_t i{0}; i < 1000000; i++) {
+      mat1.GetRow(0, mat1Rows);
+    }
+  }
+
+  SECTION("GET COLUMN") {
+    Matrix<50, 1> mat1Columns{};
+    for (uint32_t i{0}; i < 1000000; i++) {
+      mat1.GetColumn(0, mat1Columns);
+    }
+  }
+}

unit-tests/run-matrix-timing-test.sh

@@ -1,7 +0,0 @@
-# be in the root folder of this project when you run this
-cd build/
-ninja matrix-tests
-echo "Running tests. This will take a while."
-./unit-tests/matrix-tests -n "Timing Tests" -d yes > ../unit-tests/matrix-test-timings-temp.txt
-cd ../unit-tests/
-python3 test-timing-post-process.py

unit-tests/timing-results/matrix-timing-tests.txt

@@ -0,0 +1,56 @@
+Randomness seeded to: 2444679151
+0.180 s: Addition
+0.180 s: Timing Tests
+0.177 s: Subtraction
+0.177 s: Timing Tests
+1.868 s: Multiplication
+1.868 s: Timing Tests
+0.127 s: Scalar Multiplication
+0.127 s: Timing Tests
+0.173 s: Element Multiply
+0.173 s: Timing Tests
+0.178 s: Element Divide
+0.178 s: Timing Tests
+0.172 s: Minor Matrix
+0.172 s: Timing Tests
+0.103 s: Determinant
+0.103 s: Timing Tests
+0.411 s: Matrix of Minors
+0.411 s: Timing Tests
+0.109 s: Invert
+0.109 s: Timing Tests
+0.122 s: Transpose
+0.122 s: Timing Tests
+0.190 s: Normalize
+0.190 s: Timing Tests
+0.006 s: GET ROW
+0.006 s: Timing Tests
+0.235 s: GET COLUMN
+0.235 s: Timing Tests
+===============================================================================
+test cases: 1 | 1 passed
+assertions: - none -
+
+	Command being timed: "build/unit-tests/matrix-timing-tests -d yes"
+	User time (seconds): 4.05
+	System time (seconds): 0.00
+	Percent of CPU this job got: 100%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:04.05
+	Average shared text size (kbytes): 0
+	Average unshared data size (kbytes): 0
+	Average stack size (kbytes): 0
+	Average total size (kbytes): 0
+	Maximum resident set size (kbytes): 3200
+	Average resident set size (kbytes): 0
+	Major (requiring I/O) page faults: 184
+	Minor (reclaiming a frame) page faults: 171
+	Voluntary context switches: 1
+	Involuntary context switches: 26
+	Swaps: 0
+	File system inputs: 12
+	File system outputs: 1
+	Socket messages sent: 0
+	Socket messages received: 0
+	Signals delivered: 0
+	Page size (bytes): 4096
+	Exit status: 0