Improved keeper logging, now maintains updated current daily statistics until day rollover, along with network capacities.

Author: mattjhawken

tensorlink/nodes/contract_manager.py

@@ -12,8 +12,8 @@ class ContractManager:
     """
     Manages blockchain contract interactions for validator proposals and job management.
 
-    This class handles the creation, submission, and execution of proposals for
-    validator removal and job completion on the blockchain.
+    This class handles the creation, submission, voting, and execution of proposals for
+    validator removal, job completion, and reward distribution on the blockchain.
     """
 
     def __init__(self, node, multi_sig_contract, chain, public_key: str):

tensorlink/nodes/job_monitor.py

@@ -125,6 +125,7 @@ def monitor_job(self, job_id: str):
             f"Job monitor beginning for job: {job_id}",
             colour="blue",
             level=logging.INFO,
+            tag="Job Monitor",
         )
 
         job_data = self._get_job_data(job_id)
@@ -171,6 +172,7 @@ def monitor_job(self, job_id: str):
                         self.node.debug_print(
                             f"Validator -> Job inspection complete for job: {job_id}",
                             colour="blue",
+                            tag="Job Monitor",
                         )
                         job_data["last_seen"] = time.time()
                         self.node.dht.routing_table[job_id] = job_data
@@ -180,6 +182,7 @@ def monitor_job(self, job_id: str):
                         f"Error in health check cycle: {str(e)}",
                         colour="bright_red",
                         level=logging.ERROR,
+                        tag="Job Monitor",
                     )
                     break
 
@@ -240,6 +243,7 @@ def _check_single_worker(self, worker: str, module_id: str) -> bool:
                 f"Worker health check failed for {worker}: {str(e)}",
                 colour="yellow",
                 level=logging.WARNING,
+                tag="Job Monitor",
             )
             return False
 
@@ -287,6 +291,7 @@ def _verify_worker_proof(self, worker: str, proof: Dict, metrics: Dict) -> bool:
                 f"Error verifying proof of work: {str(e)}",
                 colour="bright_red",
                 level=logging.ERROR,
+                tag="Job Monitor",
             )
             return False
 
@@ -322,6 +327,7 @@ def _handle_invalid_proof(self, worker: str, module_id: str):
             f"Invalid proof of work from worker {worker} for module {module_id}",
             colour="bright_red",
             level=logging.WARNING,
+            tag="Job Monitor",
         )
 
         # Record violation
@@ -349,6 +355,7 @@ def _penalize_worker(self, worker: str, module_id: str):
                 f"Error applying worker penalty: {str(e)}",
                 colour="bright_red",
                 level=logging.ERROR,
+                tag="Job Monitor",
             )
 
     def _get_job_data(self, job_id: str) -> Optional[Dict]:
@@ -360,6 +367,7 @@ def _get_job_data(self, job_id: str) -> Optional[Dict]:
                 f"Failed to retrieve job data: {str(e)}",
                 colour="bright_red",
                 level=logging.ERROR,
+                tag="Job Monitor",
             )
             return None
 
@@ -388,6 +396,7 @@ def _check_user_status(self, job_data: Dict) -> bool:
                 f"Error checking user status: {str(e)}",
                 colour="yellow",
                 level=logging.WARNING,
+                tag="Job Monitor",
             )
             return False
 
@@ -411,6 +420,7 @@ def _handle_worker_failure(self, worker: str, module_id: str):
             f"Worker {worker} failed for module {module_id}. Initiating recovery...",
             colour="yellow",
             level=logging.WARNING,
+            tag="Job Monitor",
         )
         # Implement worker recovery strategy
         # TODO: Add worker replacement logic
@@ -437,6 +447,7 @@ def _cleanup_job(self, job_data: Dict, final_status: JobStatus):
                 f"Job {job_data['id']} cleaned up successfully with status: {final_status.value}",
                 colour="green",
                 level=logging.INFO,
+                tag="Job Monitor",
             )
 
             # Pass over job to contract manager
@@ -448,6 +459,7 @@ def _cleanup_job(self, job_data: Dict, final_status: JobStatus):
                 f"Error during job cleanup: {str(e)}",
                 colour="bright_red",
                 level=logging.ERROR,
+                tag="Job Monitor",
             )
 
     def _cleanup_workers(self, job_data: Dict):
@@ -467,6 +479,7 @@ def _cleanup_workers(self, job_data: Dict):
                             f"Error shutting down worker {worker}: {str(e)}",
                             colour="yellow",
                             level=logging.WARNING,
+                            tag="Job Monitor",
                         )
 
     def _should_terminate_job(self, job_data: Dict, current_status: JobStatus) -> bool:
@@ -525,6 +538,7 @@ def _handle_job_failure(self, job_id: str, reason: str):
                     f"Could not retrieve data for failed job {job_id}",
                     colour="bright_red",
                     level=logging.ERROR,
+                    tag="Job Monitor",
                 )
                 return
 
@@ -562,11 +576,13 @@ def _handle_job_failure(self, job_id: str, reason: str):
                 f"Job {job_id} failed: {reason}. Cleaning Up",
                 colour="bright_red",
                 level=logging.ERROR,
+                tag="Job Monitor",
             )
 
         except Exception as e:
             self.node.debug_print(
                 f"Error handling job failure: {str(e)}",
                 colour="bright_red",
                 level=logging.ERROR,
+                tag="Job Monitor",
             )