Handle default units in early schema versions (#1468)

spec_tests/test_hed_cache.py

@@ -131,26 +131,29 @@ def test_get_hed_version_path_no_auto_refresh_for_custom_directory(self):
     def test_get_hed_version_path_auto_refresh_downloads_missing_version(self):
         """get_hed_version_path downloads only the single requested version from GitHub.
 
-        Uses score 1.2.0, a released library version that hedtools does not bundle in
-        ``hed/schema/schema_data/`` (the bundle has score 1.0.0, 1.1.0, 2.0.0 and 2.1.0), so seeding the
-        fresh cache with the bundle cannot satisfy the lookup and the file must come from GitHub. score
-        lives in a normal ``hedxml/`` folder, so this exercises the real on-demand download path (manifest
-        fast path and REST API fallback alike). Only the listing for the relevant library and the one XML
-        file are fetched - not the entire catalog.
+        Every released lang and score version is bundled in ``hed/schema/schema_data/``, so the test
+        stands in a bundle of its own: a copy of the real one without ``HED_score_1.2.0.xml``. Seeding
+        the fresh cache from that bundle cannot satisfy the lookup and the file must come from GitHub.
+        score lives in a normal ``hedxml/`` folder, so this exercises the real on-demand download path
+        (manifest fast path and REST API fallback alike). Only the listing for the relevant library and
+        the one XML file are fetched - not the entire catalog.
         """
-        # Guard the premise: if this version is ever bundled, the test stops exercising the download.
-        self.assertFalse(
-            os.path.isfile(os.path.join(hed_cache.INSTALLED_CACHE_LOCATION, "HED_score_1.2.0.xml")),
-            "score 1.2.0 is now bundled; pick an unbundled released version for this test",
-        )
+        # A bundle without the requested version, so the lookup cannot be satisfied locally.
+        fake_bundle = os.path.join(os.path.dirname(self.hed_cache_dir), "schema_bundle_auto_refresh/")
+        if os.path.exists(fake_bundle):
+            shutil.rmtree(fake_bundle)
+        shutil.copytree(hed_cache.INSTALLED_CACHE_LOCATION, fake_bundle)
+        os.remove(os.path.join(fake_bundle, "HED_score_1.2.0.xml"))
         # Use a fresh cache directory so the version is definitely not present
         fresh_cache = os.path.join(os.path.dirname(self.hed_cache_dir), "schema_cache_auto_refresh/")
         if os.path.exists(fresh_cache):
             shutil.rmtree(fresh_cache)
         os.makedirs(fresh_cache)
         saved = hed_cache.HED_CACHE_DIRECTORY
+        saved_bundle = hed_cache.INSTALLED_CACHE_LOCATION
         try:
             hed_cache.HED_CACHE_DIRECTORY = fresh_cache
+            hed_cache.INSTALLED_CACHE_LOCATION = fake_bundle
             result = hed_cache.get_hed_version_path("1.2.0", library_name="score")
             self.assertIsNotNone(result, "get_hed_version_path should download score 1.2.0 from GitHub on demand")
             self.assertTrue(os.path.exists(result))
@@ -170,7 +173,9 @@ def test_get_hed_version_path_auto_refresh_downloads_missing_version(self):
             )
         finally:
             hed_cache.HED_CACHE_DIRECTORY = saved
+            hed_cache.INSTALLED_CACHE_LOCATION = saved_bundle
             shutil.rmtree(fresh_cache, ignore_errors=True)
+            shutil.rmtree(fake_bundle, ignore_errors=True)
 
 
 class TestLocal(unittest.TestCase):

tests/validator/test_reserved_checker.py

@@ -5,7 +5,7 @@
 import os
 import unittest
 
-from hed import HedString, Sidecar, TabularInput, load_schema_version
+from hed import HedString, Sidecar, TabularInput, load_schema_version, schema
 from hed.errors import ErrorSeverity, ValidationErrors
 from hed.models import DefinitionDict
 from hed.validator import HedValidator
@@ -275,6 +275,46 @@ def test_value_as_default_unit(self):
         later = load_schema_version("8.1.0")
         self.assertEqual(HedString("Duration/2 minute", later).get_all_tags()[0].value_as_default_unit(), 120.0)
 
+    @staticmethod
+    def converted(hed_schema, texts):
+        """Return value_as_default_unit() of the single tag of each text."""
+        return {text: HedString(text, hed_schema).get_all_tags()[0].value_as_default_unit() for text in texts}
+
+    def test_weight_units_si_default(self):
+        """weightUnits (default g, SI): g, gram and grams convert as 1; pound and kg do not."""
+        values = self.converted(
+            self.schema, ("Weight/3 g", "Weight/3 gram", "Weight/3 grams", "Weight/3 pound", "Weight/3 kg")
+        )
+        self.assertEqual(
+            values,
+            {
+                "Weight/3 g": 3.0,
+                "Weight/3 gram": 3.0,
+                "Weight/3 grams": 3.0,
+                "Weight/3 pound": None,
+                "Weight/3 kg": None,
+            },
+        )
+
+    def test_speed_units_compound_default(self):
+        """speedUnits (default m-per-s, compound): m-per-s and no unit convert as 1; km-per-s does not."""
+        values = self.converted(self.schema, ("Speed/3 m-per-s", "Speed/3", "Speed/3 km-per-s"))
+        self.assertEqual(values, {"Speed/3 m-per-s": 3.0, "Speed/3": 3.0, "Speed/3 km-per-s": None})
+
+    def test_intensity_units_non_si_default(self):
+        """intensityUnits (default dB, not SI): dB converts as 1; the SI units candela and cd do not."""
+        # This case exists for the SI guard of HedTag._is_default_unit_without_factors: the Weight cases of
+        # test_weight_units_si_default cannot detect the guard being removed, because g is itself SI. No 8.0.0
+        # tag uses intensityUnits, so Weight's placeholder is given that class in a copy of the bundled schema.
+        path = os.path.join(os.path.dirname(schema.__file__), "schema_data", "HED8.0.0.xml")
+        with open(path, encoding="utf-8") as file:
+            text = file.read()
+        self.assertEqual(text.count("<value>weightUnits</value>"), 1)
+        text = text.replace("<value>weightUnits</value>", "<value>intensityUnits</value>")
+        intensity_schema = schema.from_string(text, schema_format=".xml")
+        values = self.converted(intensity_schema, ("Weight/3 dB", "Weight/3 candela", "Weight/3 cd"))
+        self.assertEqual(values, {"Weight/3 dB": 3.0, "Weight/3 candela": None, "Weight/3 cd": None})
+
 
 class TestTimelineBan(ReservedTestBase):
     """A requiresTimeline tag may not appear in a table without an onset column (rule 4)."""