fix: visualiza successfully all logs from a python project

Author: tph-kds

.gitignore

@@ -166,3 +166,4 @@ datasets/*
 anod_env/*
 grafana/note.md
 # /data
+note.md

Makefile

@@ -18,7 +18,32 @@ define anod_env
 conda create -p anod_env python==3.9 -y
 endef
 
+## # MONITORING # ##
 
+.PHONY: up
+up: 
+	docker compose -f docker-compose.yaml up  -d --build
+
+.PHONY: down
+down: 
+	docker compose down
+
+# logs_name ?= grafana
+.PHONY: logs
+logs: 
+	docker compose logs otel-collector loki grafana prometheus alertmanager
+# ifeq (${logs_name},true) 
+# 	echo "Getting logs... Please wait..."
+# 	git commit -m "${message}" 
+# 	git push origin ${branch}
+
+# else ifeq (${status}, false)
+# 	echo "Withdrawing commit message..."
+# 	git reset
+		
+
+
+## # TESTING # ##
 
 .PHONY: test
 test: 

docker-compose.yaml

@@ -1,5 +1,20 @@
-version: '3'
+version: '3.5'
 services:
+  # anod_demo:
+  #   image: anod_demo:latest
+  #   labels:
+  #     logging: "promtail"
+  #     logging_jobname: "containerlogs"
+  #   ports:
+  #     - 8080:80
+  #   volumes:
+  #     - ./Dockerfile:/Dockerfile
+  #   networks:
+  #     - loki
+  #   depends_on:
+  #     - loki
+      
+
   otel-collector:
     image: otel/opentelemetry-collector:latest
     volumes:
@@ -12,16 +27,16 @@ services:
     depends_on:
       - loki
     networks:
-      - loki
+      - loki_network
   loki:
-    image: grafana/loki:main
+    image: grafana/loki:latest
     ports:
       - "3100:3100"
     volumes:
       - ./grafana/loki_config.yaml:/etc/loki/local-config.yaml
     command: -config.file=/etc/loki/local-config.yaml
     networks:
-      - loki
+      - loki_network
   grafana:
     image: grafana/grafana:latest
     environment:
@@ -31,28 +46,48 @@ services:
       - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall
       - GF_INSTALL_PLUGINS=https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app
     ports:
-      - 3000:3000/tcp
+      - "3000:3000/tcp"
     volumes:
       - ./grafana:/etc/grafana/provisioning
     networks:
-      - loki
+      - loki_network
 
   prometheus:
     image: prom/prometheus
     volumes:
-      - ./grafana/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml # Mount the prometheus.yml file
+      - ./grafana/prometheus/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml # Mount the prometheus.yml file
     ports:
       - "9090:9090" # Expose port 9090 for Prometheus
+    networks:
+      - loki_network
 
   alertmanager:
     image: prom/alertmanager
     volumes:
-      - ./grafana/prometheus/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml # Mount the alertmanager.yml file
+      - ./grafana/prometheus/alertmanager/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml # Mount the alertmanager.yml file
     ports:
       - "9093:9093" # Expose port 9093 for Alertmanager
+    networks:
+      - loki_network
+  promtail: 
+    image:  grafana/promtail:latest
+    container_name: promtail
+    volumes:
+      - ./grafana//promtail.yaml:/etc/promtail/docker-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./src/artifacts/logs/running_logs.log:/var/log/running_logs.log
+    command: -config.file=/etc/promtail/docker-config.yaml
+    depends_on:
+      - loki
+    networks:
+      - loki_network
 
 networks:
-    loki:
+  loki_network:
+    driver: bridge
+    external: true
+      
 
 
 

grafana/datasources/default.yaml

@@ -4,3 +4,6 @@ datasources:
   type: loki
   access: proxy
   url: http://loki:3100
+  version: 1
+  editable: false
+  isDefault: true

grafana/loki_config.yaml

@@ -37,4 +37,8 @@ storage_config:
     directory: /tmp/loki/chunks
 
 pattern_ingester:
-  enabled: true
+  enabled: true
+
+ruler:
+  alertmanager_url: http://localhost:9093
+

grafana/otel_config.yaml

@@ -4,9 +4,16 @@ receivers:
     protocols:
       grpc:
         endpoint: 0.0.0.0:4317
-        insecure: true
+        # insecure: true
       http:
         endpoint: 0.0.0.0:4318
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: otel-collector
+          scrape_interval: 5s
+          static_configs:
+            - targets: [localhost:8889]
 
 
 # Processors
@@ -17,25 +24,43 @@ processors:
 
 # Exporters
 exporters:
+  prometheus/metrics:
+    endpoint: "0.0.0.0:8889"
+    # tls:
+    #   insecure: true
+      # labels:
+      #   job: "prometheus" 
+
   otlphttp/logs:
     endpoint: "http://loki:3100/otlp"
     tls:
       insecure: true
 
+
+  
+
+
 
 # Pipelines
-services:
-  otlp:
-    pipelines:
-      traces:
-        receivers: [otlp]
-        processors: [batch]
-        exporters: [otlphttp/logs]
-      metrics:
-        receivers: [otlp]
-        processors: [batch]
-        exporters: [otlphttp/logs]
-      logs:
-        receivers: [otlp]
-        processors: [batch]
-        exporters: [otlphttp/logs]
+service:
+  pipelines:
+    # traces:
+    #   receivers: [otlp]
+    #   processors: [batch]
+    #   exporters: [otlphttp]
+    # metrics:
+    #   receivers: [otlp]
+    #   processors: [batch]
+    #   exporters: [otlphttp]
+    # logs:
+    #   receivers: [otlp]
+    #   processors: [batch]
+    #   exporters: [otlphttp/logs]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus/metrics]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlphttp/logs]

grafana/prometheus/alert.rules.yaml renamed to grafana/prometheus/alertmanager/alert.rules.yaml

@@ -0,0 +1,11 @@
+ global:
+   scrape_interval: 15s # Scrape metrics every 15 seconds
+
+ scrape_configs:
+   - job_name: 'otel-collector' # Name of the job for scraping our app
+     scrape_interval: 5s # Scrape metrics every 15 seconds
+     static_configs:
+       - targets: ['localhost:8889'] # Target to scrape (our app)
+      #  - targets: ['host.docker.internal:4000'] # Target to scrape (our app)
+ rule_files:
+   - './alertmanager/alert.rules.yml'

grafana/prometheus/alertmanager.yaml renamed to grafana/prometheus/alertmanager/alertmanager.yaml

@@ -0,0 +1,45 @@
+# server:
+#   http_listen_port: 9080
+#   grpc_listen_port: 0
+
+# positions:
+#   filename: /tmp/positions.yaml
+
+# clients:
+#   - url: http://loki:3100/loki/api/v1/push
+
+# scrape_configs:
+#   - job_name: flog_scrape
+#     docker_sd_configs:
+#       - host: unix:///var/run/docker.sock
+#         refresh_interval: 5s
+#         filters:
+#           - name: label
+#             values: ["logging=promtail"]
+#     relabel_configs:
+#       - source_labels: ['__meta_docker_container_name']
+#         regex: '/(.*)'
+#         target_label: 'container'
+#       - source_labels: ['__meta_docker_container_log_stream']
+#         target_label: 'logstream'
+#       - source_labels: ['__meta_docker_container_label_logging_jobname']
+#         target_label: 'job'
+
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: python-logs
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: "python-logs"
+      __path__: /var/log/running_logs.log

grafana/prometheus/prometheus.yaml

@@ -32,6 +32,7 @@ mkdocs-material==9.5.42
 ## logs and alerting
 opentelemetry-distro==0.48b0
 opentelemetry-exporter-otlp==1.27.0
+python-logging-loki
 
 
 ## dev 

grafana/prometheus/prometheus/prometheus.yaml

@@ -23,7 +23,7 @@ class GrafanaLogsConfig:
     GrafanaLogsConfig: sets up logging using OpenTelemetry with a specified service name and instance ID.
     """
 
-    def __init__(self, service_name, instance_id, endpoint="otel-collector:4317"):
+    def __init__(self, service_name, instance_id, endpoint="http://loki:3100/loki/metrics"):
         """
         Initialize the GrafanaLogsConfig: with a service name and instance ID.
 
@@ -52,30 +52,30 @@ def setup_logging(self, notset):
         Returns:
             LoggingHandler instance configured with the logger provider.
         """
-        try:
-            # Set the created LoggerProvider as the global logger provider.
-            set_logger_provider(self.logger_provider)
-
-            # Create an instance of OTLPLogExporter with insecure connection.
-            exporter = OTLPLogExporter(endpoint="otel-collector:4317", insecure=True)
-
-            # Add a BatchLogRecordProcessor to the logger provider with the exporter.
-            self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
-
-            print("Connected to OTLP exporter at", self.endpoint)
-
-        except RpcError as e:
-            # Check for UNAVAILABLE status and set up console fallback
-            if e.code() == StatusCode.UNAVAILABLE:
-                print(f"Warning: OTLP endpoint '{self.endpoint}' unavailable. Using console logging as fallback.")
-                self.logger_provider = LoggerProvider(resource=Resource.create(
-                    {
-                        "service.name": self.service_name,
-                        "service.instance.id": self.instance_id,
-                    }
-                ))
-                set_logger_provider(self.logger_provider)
-                # logging.getLogger().addHandler(logging.StreamHandler())
+        # try:
+        # Set the created LoggerProvider as the global logger provider.
+        set_logger_provider(self.logger_provider)
+
+        # Create an instance of OTLPLogExporter with insecure connection.
+        exporter = OTLPLogExporter(endpoint="otel-collector:4317", insecure=True)
+
+        # Add a BatchLogRecordProcessor to the logger provider with the exporter.
+        self.logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+
+        print("Connected to OTLP exporter at", self.endpoint)
+
+        # except RpcError as e:
+        #     # Check for UNAVAILABLE status and set up console fallback
+        #     if e.code() == StatusCode.UNAVAILABLE:
+        #         print(f"Warning: OTLP endpoint '{self.endpoint}' unavailable. Using console logging as fallback.")
+        #         self.logger_provider = LoggerProvider(resource=Resource.create(
+        #             {
+        #                 "service.name": self.service_name,
+        #                 "service.instance.id": self.instance_id,
+        #             }
+        #         ))
+        #         set_logger_provider(self.logger_provider)
+        #         # logging.getLogger().addHandler(logging.StreamHandler())
 
         # Create a LoggingHandler with the specified logger provider and log level set to NOTSET.
         handler = LoggingHandler(level=notset, logger_provider=self.logger_provider)