MRG: Merge pull request #483 from octue/enhancement/add-metadata-hash-property

Add metadata hash property

Author: cortadocodes

octue/mixins/metadata.py

@@ -2,23 +2,48 @@
 
 import pkg_resources
 
+from octue.mixins.hashable import Hashable
+
 
 class Metadata:
     _METADATA_ATTRIBUTES = tuple()
 
-    def metadata(self, include_sdk_version=True):
-        """Get the instance's metadata in primitive form.
+    @property
+    def metadata_hash_value(self):
+        """Get the hash of the instance's metadata, not including its ID.
+
+        :return str:
+        """
+        return self._metadata_hash_value()
+
+    def metadata(self, include_id=True, include_sdk_version=True, **kwargs):
+        """Get the instance's metadata in primitive form. The metadata is the set of attributes included in the class
+        variable `self._METADATA_ATTRIBUTES`.
 
-        :param bool include_sdk_version: if `True`, include the `octue` version that instantiated the instance in the metadata
+        :param bool include_id: if `True`, include the ID of the instance if it is included in `self._METADATA_ATTRIBUTES`
+        :param bool include_sdk_version: if `True`, include the `octue` version that instantiated the instance
+        :param kwargs: any kwargs to use in an overridden `self.metadata` method
         :return dict:
         """
         metadata = {name: getattr(self, name) for name in self._METADATA_ATTRIBUTES}
 
         if include_sdk_version:
             metadata["sdk_version"] = pkg_resources.get_distribution("octue").version
 
+        if not include_id and "id" in metadata:
+            del metadata["id"]
+
         return metadata
 
+    def _metadata_hash_value(self, **kwargs):
+        """Get the hash of the instance's metadata, not including its ID. Override this method to change what kwargs
+        `self.metadata` gets.
+
+        :param kwargs: any kwargs to use in an overridden `self.metadata` method when calculating the metadata hash value
+        :return str:
+        """
+        return Hashable.hash_non_class_object(self.metadata(include_id=False, include_sdk_version=False, **kwargs))
+
     @abstractmethod
     def _set_metadata(self, metadata):
         """Set the instance's metadata.

octue/resources/datafile.py

@@ -427,15 +427,16 @@ def download(self, local_path=None):
         self.reset_hash()
         return self._local_path
 
-    def metadata(self, include_sdk_version=True, use_octue_namespace=True):
+    def metadata(self, include_id=True, include_sdk_version=True, use_octue_namespace=True):
         """Get the datafile's metadata in a serialised form (i.e. the attributes `id`, `timestamp`, `labels`, `tags`,
         and `sdk_version`).
 
+        :param bool include_id: if `True`, include the ID of the datafile
         :param bool include_sdk_version: if `True`, include the `octue` version that instantiated the datafile in the metadata
         :param bool use_octue_namespace: if `True`, prefix metadata names with "octue__"
         :return dict:
         """
-        metadata = super().metadata(include_sdk_version=include_sdk_version)
+        metadata = super().metadata(include_sdk_version=include_sdk_version, include_id=include_id)
 
         if not use_octue_namespace:
             return metadata
@@ -593,6 +594,13 @@ def _set_metadata(self, metadata):
 
             setattr(self, attribute, metadata[attribute])
 
+    def _metadata_hash_value(self):
+        """Get the hash of the datafile's metadata, not including its ID.
+
+        :return str:
+        """
+        return super()._metadata_hash_value(use_octue_namespace=False)
+
     def _calculate_hash(self):
         """Get the hash of the datafile according to the first of the following methods that is applicable:
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "octue"
-version = "0.28.1"
+version = "0.28.2"
 description = "A package providing template applications for data services, and a python SDK to the Octue API."
 readme = "README.md"
 authors = ["Marcus Lugg <[email protected]>", "Thomas Clark <[email protected]>"]

tests/resources/test_datafile.py

@@ -172,6 +172,29 @@ def test_hash_of_cloud_datafile_avoids_downloading_file(self):
         # Check that the reloaded datafile hasn't been downloaded.
         self.assertIsNone(datafile_reloaded_from_cloud._local_path)
 
+    def test_metadata_hash_is_same_for_different_files_with_the_same_metadata(self):
+        """Test that the metadata hash is the same for datafiles with different files but the same metadata."""
+        with tempfile.NamedTemporaryFile(delete=False) as temporary_file:
+            first_file = Datafile(path=temporary_file.name, labels={"a", "b", "c"})
+
+            with first_file.open("w") as f:
+                f.write("hello")
+
+            with tempfile.NamedTemporaryFile(delete=False) as temporary_file:
+                second_file = Datafile(path=temporary_file.name, labels={"a", "b", "c"})
+
+                with second_file.open("w") as f:
+                    f.write("goodbye")
+
+                self.assertEqual(first_file.metadata_hash_value, second_file.metadata_hash_value)
+
+    def test_metadata_hash_is_different_for_same_file_but_different_metadata(self):
+        """Test that the metadata hash is different for datafiles with the same files but different metadata."""
+        first_file = Datafile(path=self.path, labels={"a", "b", "c"})
+        second_file = copy.deepcopy(first_file)
+        second_file.labels = {"d", "e", "f"}
+        self.assertNotEqual(first_file.metadata_hash_value, second_file.metadata_hash_value)
+
     def test_exists_in_cloud(self):
         """Test whether it can be determined that a datafile exists in the cloud or not."""
         self.assertFalse(self.create_valid_datafile().exists_in_cloud)

tests/resources/test_dataset.py

@@ -368,6 +368,19 @@ def test_hashes_for_the_same_dataset_are_the_same(self):
         second_dataset = copy.deepcopy(first_dataset)
         self.assertEqual(first_dataset.hash_value, second_dataset.hash_value)
 
+    def test_metadata_hash_is_same_for_different_datasets_with_the_same_metadata(self):
+        """Test that the metadata hash is the same for datasets with different files but the same metadata."""
+        first_dataset = Dataset(labels={"a", "b", "c"})
+        second_dataset = Dataset(files={Datafile(path="blah", hypothetical=True)}, labels={"a", "b", "c"})
+        self.assertEqual(first_dataset.metadata_hash_value, second_dataset.metadata_hash_value)
+
+    def test_metadata_hash_is_different_for_same_dataset_but_different_metadata(self):
+        """Test that the metadata hash is different for datasets with the same files but different metadata."""
+        first_dataset = self.create_valid_dataset()
+        second_dataset = copy.deepcopy(first_dataset)
+        second_dataset.labels = {"d", "e", "f"}
+        self.assertNotEqual(first_dataset.metadata_hash_value, second_dataset.metadata_hash_value)
+
     def test_serialisation_and_deserialisation(self):
         """Test that a dataset can be serialised and deserialised."""
         dataset_id = "e376fb31-8f66-414d-b99f-b43395cebbf1"