made 'hash_fieldname' default for random seed generation for better value spread (#58)

* made 'hash_fieldname' default for random seed generation for better spread of values

* made 'hash_fieldname' default for random seed generation for better spread of values

Author: ronanstokes-db

dbldatagen/data_generator.py

@@ -19,8 +19,6 @@
 _OLD_MIN_OPTION = 'min'
 _OLD_MAX_OPTION = 'max'
 
-NO_SEED_SUPPLIED = -2
-
 
 class DataGenerator:
     """ Main Class for test data set generation
@@ -53,7 +51,7 @@ class DataGenerator:
     logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.NOTSET)
 
     def __init__(self, sparkSession=None, name=None, randomSeedMethod=None,
-                 rows=1000000, startingId=0, randomSeed=NO_SEED_SUPPLIED, partitions=None, verbose=False,
+                 rows=1000000, startingId=0, randomSeed=None, partitions=None, verbose=False,
                  batchSize=None, debug=False, **kwargs):
         """ Constructor for data generator object """
 
@@ -93,11 +91,20 @@ def __init__(self, sparkSession=None, name=None, randomSeedMethod=None,
             self.logger.warning("option 'generateWithSelects' switch is deprecated - selects will always be used")
 
         self._seedMethod = randomSeedMethod
-        self._instanceRandomSeed = randomSeed if randomSeed != NO_SEED_SUPPLIED else self._randomSeed
 
-        # if a valid random seed was supplied but no seed method was applied, make the seed method "fixed"
-        if (randomSeed is not None and randomSeed != RANDOM_SEED_RANDOM) and randomSeedMethod is None:
-            self._seedMethod = "fixed"
+        if randomSeed is None:
+            self._instanceRandomSeed =  self._randomSeed
+
+            if randomSeedMethod is None:
+                self._seedMethod = RANDOM_SEED_HASH_FIELD_NAME
+            else:
+                self._seedMethod = randomSeedMethod
+        else:
+            self._instanceRandomSeed = randomSeed
+
+            # if a valid random seed was supplied but no seed method was applied, make the seed method "fixed"
+            if randomSeedMethod is None:
+                self._seedMethod = "fixed"
 
         if self._seedMethod not in [None, RANDOM_SEED_FIXED, RANDOM_SEED_HASH_FIELD_NAME]:
             msg = f"""seedMethod should be None, '{RANDOM_SEED_FIXED}' or '{RANDOM_SEED_HASH_FIELD_NAME}' """
@@ -700,6 +707,12 @@ def _generateColumnDefinition(self, colName, colType=None, baseColumn=None,
             effective_random_seed = new_props["randomSeed"]
             new_props.pop("randomSeed")
             new_props["random"] = True
+
+            # if random seed has override but randomSeedMethod does not
+            # set it to fixed
+            if "randomSeedMethod" not in new_props:
+                new_props["randomSeedMethod"] = RANDOM_SEED_FIXED
+
         elif "random" in new_props and new_props["random"]:
             effective_random_seed = self._instanceRandomSeed
         else:

docs/source/repeatable_data_generation.rst

@@ -69,7 +69,8 @@ column specification is marked as ``random``.
 If a random seed is provided, either as an argument to the DataGenerator instance specification,
 or as option on the column specification, the random seed will be applied to fields when random data generation is used.
 
-By default, a default random seed is used unless a specific random seed is provided.
+By default, a default random seed is used and the randomSeedMethod is set to 'hash_fieldname' unless a specific
+random seed is provided.
 
 Use of Hashed field names
 ^^^^^^^^^^^^^^^^^^^^^^^^^

tests/test_repeatable_data.py

@@ -296,21 +296,21 @@ def test_random_seed_flow(self):
         self.assertEqual(pluginDataspec.randomSeed, dg.DEFAULT_RANDOM_SEED)
 
         code1Spec = pluginDataspec.getColumnSpec("code1")
-        self.assertEqual(code1Spec.randomSeed, dg.DEFAULT_RANDOM_SEED)
+        self.assertIsNotNone(code1Spec.randomSeed)
         code2Spec = pluginDataspec.getColumnSpec("code2")
-        self.assertEqual(code2Spec.randomSeed, 2021)
+        self.assertEqual(code2Spec.randomSeed, 2021, "code2")
 
         textSpec = pluginDataspec.getColumnSpec("text")
-        self.assertEqual(textSpec.randomSeed, dg.DEFAULT_RANDOM_SEED)
+        self.assertIsNotNone(textSpec.randomSeed)
 
         textSpec2 = pluginDataspec.getColumnSpec("text2")
-        self.assertEqual(textSpec2.textGenerator.randomSeed, dg.DEFAULT_RANDOM_SEED)
+        self.assertIsNotNone(textSpec2.textGenerator.randomSeed)
 
         ilTextSpec = pluginDataspec.getColumnSpec("paras")
-        self.assertEqual(ilTextSpec.randomSeed, dg.DEFAULT_RANDOM_SEED)
+        self.assertIsNotNone(ilTextSpec.randomSeed)
 
         ilTextSpec2 = pluginDataspec.getColumnSpec("paras2")
-        self.assertEqual(ilTextSpec2.textGenerator.randomSeed, dg.DEFAULT_RANDOM_SEED)
+        self.assertIsNotNone(ilTextSpec2.textGenerator.randomSeed)
 
         self.assertEqual(textSpec.randomSeed, textSpec.textGenerator.randomSeed)
         self.assertEqual(textSpec2.randomSeed, textSpec2.textGenerator.randomSeed)
@@ -335,16 +335,16 @@ def test_random_seed_flow2(self):
         self.assertEqual(pluginDataspec.randomSeed, effective_random_seed)
 
         code1Spec = pluginDataspec.getColumnSpec("code1")
-        self.assertEqual(code1Spec.randomSeed, effective_random_seed, "code1")
+        self.assertIsNotNone(code1Spec.randomSeed, "code1")
 
         textSpec = pluginDataspec.getColumnSpec("text")
-        self.assertEqual(textSpec.randomSeed, effective_random_seed, "text")
+        self.assertIsNotNone(textSpec.randomSeed, "text")
 
         code2Spec = pluginDataspec.getColumnSpec("code2")
-        self.assertEqual(code1Spec.randomSeed, effective_random_seed, "code2")
+        self.assertIsNotNone(code1Spec.randomSeed, "code2")
 
         text2Spec = pluginDataspec.getColumnSpec("text2")
-        self.assertEqual(text2Spec.randomSeed, effective_random_seed, "text2")
+        self.assertIsNotNone(text2Spec.randomSeed, "text2")
 
         self.assertEqual(textSpec.randomSeed, textSpec.textGenerator.randomSeed)
         self.assertEqual(text2Spec.randomSeed, text2Spec.textGenerator.randomSeed)