Fix testcase order

problemtools/checks/testdata.py

@@ -287,6 +287,11 @@ def _check_group(
         elif path.suffix == '.ans' and not files.is_file(path.with_suffix('.in')):
             diag.error(f"No matching input file for answer '{path.relative_to(files.root)}'")
 
+    subgroup_names = {subgroup.path.name for subgroup in group.get_subgroups()}
+    for testcase in group.get_testcases():
+        if testcase.path.name in subgroup_names:
+            diag.error(f"Test case '{testcase.path}' has the same name as a test data group")
+
     if not group.get_subgroups() and not group.get_testcases():
         group_dir = group.datadir.relative_to(files.root)
         if group.datadir.name != 'sample':

problemtools/model/testdata.py

@@ -158,8 +158,9 @@ def _load_group(
     for key, default in DEFAULT_CONFIG.items():
         merged_config.setdefault(key, default)
 
+    # Order by base name (not file name, which would put foo-bar.in before foo.in)
     items: list[TestCase | TestDataGroup] = []
-    for entry in files.listdir(datadir):
+    for entry in sorted(files.listdir(datadir), key=lambda entry: entry.name if files.is_dir(entry) else entry.stem):
         if files.is_dir(entry):
             items.append(_load_group(files, entry, merged_config, metadata, is_root=False))
         elif entry.suffix == '.ans' and files.is_file(entry.with_suffix('.in')):

problemtools/statement_util.py

@@ -181,7 +181,7 @@ def format_samples(problem_root: Path) -> list[str]:
 
     Returns:
         List[str]: All samples, converted to a format appropriate to be pasted into
-        a markdown file. Ordered lexicographically by file names
+        a markdown file. Ordered lexicographically by sample name (file name without extension)
     """
 
     sample_path = os.path.join(str(problem_root), 'data', 'sample')
@@ -193,7 +193,7 @@ def format_samples(problem_root: Path) -> list[str]:
     is_multi_pass = problem_metadata.is_multi_pass()
     is_interactive = problem_metadata.is_interactive()
     all_cases = os.listdir(sample_path)
-    for sample in sorted(os.listdir(sample_path)):
+    for sample in sorted(all_cases, key=lambda name: Path(name).stem):
         if sample.endswith('.interaction'):
             samples.append(format_interactive_sample(sample_path, sample, casenum, is_interactive, is_multi_pass))
             casenum += 1

tests/test_load_testdata.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+
+from problemtools.metadata import load_metadata
+from problemtools.model import load_files, load_testdata
+from problemtools.model.testdata import TestCase, TestDataGroup
+
+# Not test classes -- just named that way by the model. Tell pytest not to collect them.
+TestCase.__test__ = False  # type: ignore[attr-defined]
+TestDataGroup.__test__ = False  # type: ignore[attr-defined]
+
+
+def test_items_are_ordered_by_base_name(tmp_path: Path) -> None:
+    (tmp_path / 'problem.yaml').write_text('name: Test\n')
+    secret = tmp_path / 'data' / 'secret'
+    for name in ['foo', 'foo-bar', 'foo-baz/1', 'foo.a/1']:
+        (secret / name).parent.mkdir(parents=True, exist_ok=True)
+        (secret / f'{name}.in').write_text('')
+        (secret / f'{name}.ans').write_text('')
+
+    testdata = load_testdata(load_files(tmp_path), load_metadata(tmp_path))
+
+    secret_group = testdata.items[0]
+    assert isinstance(secret_group, TestDataGroup)
+    assert [str(item.path) for item in secret_group.items] == [
+        'secret/foo',
+        'secret/foo-bar',
+        'secret/foo-baz',
+        'secret/foo.a',
+    ]
+    assert [str(case.path) for case in testdata.get_all_testcases()] == [
+        'secret/foo',
+        'secret/foo-bar',
+        'secret/foo-baz/1',
+        'secret/foo.a/1',
+    ]