skip adding param group when no param (#1797)

* skip adding param group when no param

* fix test case

Author: hnyu

alf/optimizers/optimizers.py

@@ -490,6 +490,10 @@ def add_param_group(self, param_group):
             if (not self._ignore_param_not_requiring_grad or p.requires_grad)
         ]
 
+        if len(param_group['params']) == 0:
+            # If no params are in this group, ignore adding
+            return
+
         lr_scheduler = param_group.get('lr_scheduler', None)
         if isinstance(lr_scheduler, Callable):
             self._lr_schedulers.append(lr_scheduler)

alf/utils/checkpoint_utils_test.py

@@ -366,68 +366,38 @@ def test_with_cycle(self):
                     '_sub_alg2._optimizers.0',
                     {
                         'state': {},
-                        'param_groups': [
-                            {
-                                'lr': 0.2,
-                                'betas': (0.9, 0.999),
-                                'capturable': False,
-                                'differentiable': False,
-                                'eps': 1e-08,
-                                'foreach': None,
-                                'fused': None,
-                                'weight_decay': 0,
-                                'amsgrad': False,
-                                'maximize': False,
-                                'params': []
-                            },
-                            {
-                                'lr': 0.2,
-                                'betas': (0.9, 0.999),
-                                'capturable': False,
-                                'differentiable': False,
-                                'eps': 1e-08,
-                                'foreach': None,
-                                'fused': None,
-                                'weight_decay': 0,
-                                'amsgrad': False,
-                                'maximize': False,
-                                'params': [0]  # order index instead of id
-                            }
-                        ]
+                        'param_groups': [{
+                            'lr': 0.2,
+                            'betas': (0.9, 0.999),
+                            'capturable': False,
+                            'differentiable': False,
+                            'eps': 1e-08,
+                            'foreach': None,
+                            'fused': None,
+                            'weight_decay': 0,
+                            'amsgrad': False,
+                            'maximize': False,
+                            'params': [0]  # order index instead of id
+                        }]
                     }),
                 ('_param_list.0', torch.tensor([0.])),
                 (
                     '_optimizers.0',
                     {
                         'state': {},
-                        'param_groups': [
-                            {
-                                'lr': 0.1,
-                                'betas': (0.9, 0.999),
-                                'capturable': False,
-                                'differentiable': False,
-                                'eps': 1e-08,
-                                'foreach': None,
-                                'fused': None,
-                                'weight_decay': 0,
-                                'amsgrad': False,
-                                'maximize': False,
-                                'params': []
-                            },
-                            {
-                                'lr': 0.1,
-                                'betas': (0.9, 0.999),
-                                'capturable': False,
-                                'differentiable': False,
-                                'eps': 1e-08,
-                                'foreach': None,
-                                'fused': None,
-                                'weight_decay': 0,
-                                'amsgrad': False,
-                                'maximize': False,
-                                'params': [0, 1]  # order indices instead of id
-                            }
-                        ]
+                        'param_groups': [{
+                            'lr': 0.1,
+                            'betas': (0.9, 0.999),
+                            'capturable': False,
+                            'differentiable': False,
+                            'eps': 1e-08,
+                            'foreach': None,
+                            'fused': None,
+                            'weight_decay': 0,
+                            'amsgrad': False,
+                            'maximize': False,
+                            'params': [0, 1]  # order indices instead of id
+                        }]
                     })
             ])