Introduce scenario-based regression testing framework with expanded configuration coverage

.github/workflows/weekly-regression.yaml

@@ -10,9 +10,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  discover:
+    name: Discover regression systems
+    runs-on: ubuntu-24.04
+    outputs:
+      systems: ${{ steps.set.outputs.systems }}
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Generate system matrix
+        id: set
+        run: |
+          systems=$(ls tests/regression/configs | jq -R -s -c 'split("\n")[:-1]')
+          echo "systems=$systems" >> $GITHUB_OUTPUT
+
   regression:
-    name: Regression tests (including slow)
+    name: Regression (${{ matrix.system }})
+    needs: discover
     runs-on: ubuntu-24.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        system: ${{ fromJson(needs.discover.outputs.systems) }}
+
     steps:
       - name: Checkout repo
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
@@ -29,19 +52,20 @@ jobs:
           path: .testdata
           key: codeentropy-testdata-${{ runner.os }}-py314
 
-      - name: Install CodeEntropy and its testing dependencies
+      - name: Install CodeEntropy and dependencies
         run: |
           pip install --upgrade pip
           pip install -e .[testing]
 
-      - name: Run regression test suite
-        run: pytest tests/regression --run-slow
+      - name: Run regression tests for system
+        run: |
+          pytest tests/regression -k "${{ matrix.system }}" --run-slow -vv
 
       - name: Upload regression artifacts on failure
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: regression-failure-artifacts
+          name: regression-${{ matrix.system }}-artifacts
           path: |
             .testdata/**
             tests/regression/**/.pytest_cache/**

.gitignore

@@ -129,4 +129,4 @@ job*
 .testdata/
 
 !tests/regression/baselines/
-!tests/regression/baselines/*.json
+!tests/regression/baselines/*/*.json

tests/regression/baselines/benzaldehyde/axes_off.json

@@ -0,0 +1,45 @@
+{
+  "args": {
+    "top_traj_file": [
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/molecules.top",
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/trajectory.crd"
+    ],
+    "force_file": "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/forces.frc",
+    "file_format": "MDCRD",
+    "kcal_force_units": true,
+    "selection_string": "resid 1:10",
+    "start": 0,
+    "end": 1,
+    "step": 1,
+    "bin_width": 30,
+    "temperature": 298.0,
+    "verbose": false,
+    "output_file": "/tmp/pytest-of-harry-swift/pytest-53/test_regression_matches_baseli0/job001/output_file.json",
+    "force_partitioning": 0.5,
+    "water_entropy": true,
+    "grouping": "molecules",
+    "combined_forcetorque": true,
+    "customised_axes": false,
+    "search_type": "grid"
+  },
+  "provenance": {
+    "python": "3.14.3",
+    "platform": "Linux-6.17.0-19-generic-x86_64-with-glibc2.39",
+    "codeentropy_version": "2.1.0",
+    "git_sha": "be46d826f4ae38e3c6e62a7d5bcddeca85e31590"
+  },
+  "groups": {
+    "0": {
+      "components": {
+        "united_atom:Transvibrational": 0.08982962903796131,
+        "united_atom:Rovibrational": 32.16018134884085,
+        "residue:FTmat-Transvibrational": 88.7671666695003,
+        "residue:FTmat-Rovibrational": 61.61036267672132,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 7.791711490122748
+      },
+      "total": 190.41925181422317
+    }
+  }
+}

tests/regression/baselines/benzaldehyde/combined_forcetorque_false.json

@@ -0,0 +1,45 @@
+{
+  "args": {
+    "top_traj_file": [
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/molecules.top",
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/trajectory.crd"
+    ],
+    "force_file": "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/forces.frc",
+    "file_format": "MDCRD",
+    "kcal_force_units": true,
+    "selection_string": "resid 1:10",
+    "start": 0,
+    "end": 1,
+    "step": 1,
+    "bin_width": 30,
+    "temperature": 298.0,
+    "verbose": false,
+    "output_file": "/tmp/pytest-of-harry-swift/pytest-53/test_regression_matches_baseli1/job001/output_file.json",
+    "force_partitioning": 0.5,
+    "water_entropy": true,
+    "grouping": "molecules",
+    "combined_forcetorque": false,
+    "customised_axes": true,
+    "search_type": "grid"
+  },
+  "provenance": {
+    "python": "3.14.3",
+    "platform": "Linux-6.17.0-19-generic-x86_64-with-glibc2.39",
+    "codeentropy_version": "2.1.0",
+    "git_sha": "be46d826f4ae38e3c6e62a7d5bcddeca85e31590"
+  },
+  "groups": {
+    "0": {
+      "components": {
+        "united_atom:Transvibrational": 0.07119323721997475,
+        "united_atom:Rovibrational": 49.68669738152346,
+        "residue:Transvibrational": 69.48692941204929,
+        "residue:Rovibrational": 68.46147102540942,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 7.791711490122748
+      },
+      "total": 195.49800254632487
+    }
+  }
+}

tests/regression/baselines/benzaldehyde.json → tests/regression/baselines/benzaldehyde/default.json

@@ -1,10 +1,10 @@
 {
   "args": {
     "top_traj_file": [
-      "/home/ogo12949/CodeEntropy/.testdata/benzaldehyde/molecules.top",
-      "/home/ogo12949/CodeEntropy/.testdata/benzaldehyde/trajectory.crd"
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/molecules.top",
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/trajectory.crd"
     ],
-    "force_file": "/home/ogo12949/CodeEntropy/.testdata/benzaldehyde/forces.frc",
+    "force_file": "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/forces.frc",
     "file_format": "MDCRD",
     "kcal_force_units": true,
     "selection_string": "all",
@@ -14,7 +14,7 @@
     "bin_width": 30,
     "temperature": 298.0,
     "verbose": false,
-    "output_file": "/tmp/pytest-of-ogo12949/pytest-5/test_regression_matches_baseli1/job001/output_file.json",
+    "output_file": "/tmp/pytest-of-harry-swift/pytest-53/test_regression_matches_baseli2/job001/output_file.json",
     "force_partitioning": 0.5,
     "water_entropy": true,
     "grouping": "molecules",
@@ -23,10 +23,10 @@
     "search_type": "grid"
   },
   "provenance": {
-    "python": "3.13.5",
-    "platform": "Linux-6.17.0-1012-oem-x86_64-with-glibc2.39",
-    "codeentropy_version": "2.0.0",
-    "git_sha": "697cc5da0af766f2f69b0fa31254f2ce7b2b1141"
+    "python": "3.14.3",
+    "platform": "Linux-6.17.0-19-generic-x86_64-with-glibc2.39",
+    "codeentropy_version": "2.1.0",
+    "git_sha": "be46d826f4ae38e3c6e62a7d5bcddeca85e31590"
   },
   "groups": {
     "0": {

tests/regression/baselines/benzaldehyde/frame_window.json

@@ -0,0 +1,45 @@
+{
+  "args": {
+    "top_traj_file": [
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/molecules.top",
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/trajectory.crd"
+    ],
+    "force_file": "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/forces.frc",
+    "file_format": "MDCRD",
+    "kcal_force_units": true,
+    "selection_string": "resid 1:10",
+    "start": 0,
+    "end": 5,
+    "step": 2,
+    "bin_width": 30,
+    "temperature": 298.0,
+    "verbose": false,
+    "output_file": "/tmp/pytest-of-harry-swift/pytest-53/test_regression_matches_baseli3/job001/output_file.json",
+    "force_partitioning": 0.5,
+    "water_entropy": true,
+    "grouping": "molecules",
+    "combined_forcetorque": true,
+    "customised_axes": true,
+    "search_type": "grid"
+  },
+  "provenance": {
+    "python": "3.14.3",
+    "platform": "Linux-6.17.0-19-generic-x86_64-with-glibc2.39",
+    "codeentropy_version": "2.1.0",
+    "git_sha": "be46d826f4ae38e3c6e62a7d5bcddeca85e31590"
+  },
+  "groups": {
+    "0": {
+      "components": {
+        "united_atom:Transvibrational": 40.30267601961045,
+        "united_atom:Rovibrational": 38.21906858443615,
+        "residue:FTmat-Transvibrational": 73.41098578352612,
+        "residue:FTmat-Rovibrational": 57.881504393660364,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 7.791711490122748
+      },
+      "total": 217.60594627135583
+    }
+  }
+}

tests/regression/baselines/benzaldehyde/grouping_each.json

@@ -0,0 +1,153 @@
+{
+  "args": {
+    "top_traj_file": [
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/molecules.top",
+      "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/trajectory.crd"
+    ],
+    "force_file": "/home/harry-swift/BioSim/software/CodeEntropy/.testdata/benzaldehyde/forces.frc",
+    "file_format": "MDCRD",
+    "kcal_force_units": true,
+    "selection_string": "resid 1:10",
+    "start": 0,
+    "end": 1,
+    "step": 1,
+    "bin_width": 30,
+    "temperature": 298.0,
+    "verbose": false,
+    "output_file": "/tmp/pytest-of-harry-swift/pytest-53/test_regression_matches_baseli4/job001/output_file.json",
+    "force_partitioning": 0.5,
+    "water_entropy": true,
+    "grouping": "each",
+    "combined_forcetorque": true,
+    "customised_axes": true,
+    "search_type": "grid"
+  },
+  "provenance": {
+    "python": "3.14.3",
+    "platform": "Linux-6.17.0-19-generic-x86_64-with-glibc2.39",
+    "codeentropy_version": "2.1.0",
+    "git_sha": "be46d826f4ae38e3c6e62a7d5bcddeca85e31590"
+  },
+  "groups": {
+    "0": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.062499436482478445,
+        "residue:FTmat-Transvibrational": 16.83857036769833,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 9.587915463990273
+      },
+      "total": 26.488985268171085
+    },
+    "1": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.04214874235319735,
+        "residue:FTmat-Transvibrational": 9.439228173427328,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 0.0
+      },
+      "total": 9.481376915780524
+    },
+    "2": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.0022147884278620462,
+        "residue:FTmat-Transvibrational": 17.449372254125862,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 13.250255095331166
+      },
+      "total": 30.70184213788489
+    },
+    "3": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 138.65701144028458,
+        "residue:FTmat-Transvibrational": 15.803976930845373,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 0.0
+      },
+      "total": 154.46098837112996
+    },
+    "4": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.003874883925216773,
+        "residue:FTmat-Transvibrational": 23.899916844406114,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 0.0
+      },
+      "total": 23.90379172833133
+    },
+    "5": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.010824819910336835,
+        "residue:FTmat-Transvibrational": 11.05479779559611,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 23.16437388578126
+      },
+      "total": 34.22999650128771
+    },
+    "6": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.00018982532508837594,
+        "residue:FTmat-Transvibrational": 19.723925127490933,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 18.044518549147842
+      },
+      "total": 37.76863350196386
+    },
+    "7": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.004033941970593011,
+        "residue:FTmat-Transvibrational": 18.90676882142135,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 0.0
+      },
+      "total": 18.910802763391942
+    },
+    "8": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 0.020362757918507005,
+        "residue:FTmat-Transvibrational": 16.13204209116967,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 10.453350878685521
+      },
+      "total": 26.605755727773698
+    },
+    "9": {
+      "components": {
+        "united_atom:Transvibrational": 0.0,
+        "united_atom:Rovibrational": 138.65525900218074,
+        "residue:FTmat-Transvibrational": 21.42548529236268,
+        "residue:FTmat-Rovibrational": 0.0,
+        "united_atom:Conformational": 0.0,
+        "residue:Conformational": 0.0,
+        "residue:Orientational": 0.0
+      },
+      "total": 160.08074429454342
+    }
+  }
+}