Add OpenMetrics endpoint

This patch add an OpenMetrics endpoint, exposing the current state of
cameras and reporting occurring errors.

This fixes #14
This fixes #28

README.md

@@ -8,3 +8,30 @@ Control PTZ camera to move to certain presets when starting a scheduled recordin
 ❯ pip install -r requirements.txt
 ❯ python -m occameracontrol
 ```
+
+## Metrics
+
+You can enable an OpenMetrics endpoint in the configuration. This can give you
+insight into current camera positions and will report occurring errors.
+
+The resulting metrics data will look like this:
+
+```
+# HELP request_errors_total Number of errors related to HTTP requests
+# TYPE request_errors_total counter
+request_errors_total{ressource="http://camera-3-panasonic.example.com",type="ConnectionError"} 77.0
+request_errors_total{ressource="http://camera-panasonic.example.com",type="ReadTimeout"} 12.0
+# HELP request_errors_created Number of errors related to HTTP requests
+# TYPE request_errors_created gauge
+request_errors_created{ressource="http://camera-3-panasonic.example.com",type="ConnectionError"} 1.707571882114209e+09
+request_errors_created{ressource="http://camera-panasonic.example.com",type="ReadTimeout"} 1.7075718871156712e+09
+# HELP agent_calendar_update_total Nuber of calendar update
+# TYPE agent_calendar_update_total gauge
+agent_calendar_update_total{agent="test_agent"} 4.0
+# HELP agent_calendar_update_time Time of the last calendar update
+# TYPE agent_calendar_update_time gauge
+agent_calendar_update_time{agent="test_agent"} 1.707571943100096e+09
+# HELP camera_position Last position (preset number) a camera moved to
+# TYPE camera_position gauge
+camera_position{camera="http://camera-2-panasonic.example.com"} 10.0
+```

camera-control.yml

@@ -45,4 +45,19 @@ camera:
     - url: http://camera-sony-15-217.virtuos.uni-osnabrueck.de
       type: sony
 
+metrics:
+  # Default: False
+  enabled: True
+  # Default: 8000
+  port: 8000
+  # Default: 0.0.0.0
+  addr: '0.0.0.0'
+
+  # Default: null
+  certfile: null
+
+  # Default: null
+  keyfile: null
+
+# Configure the log level
 loglevel: debug

occameracontrol/__main__.py

@@ -16,36 +16,41 @@
 
 import argparse
 import logging
-import requests
 import time
 
 from confygure import setup, config
 from threading import Thread
 
 from occameracontrol.agent import Agent
 from occameracontrol.camera import Camera
+from occameracontrol.metrics import start_metrics_exporter, RequestErrorHandler
 
 
 logger = logging.getLogger(__name__)
 
 
 def update_agents(agents: list[Agent]):
+    error_handlers = {
+        agent.agent_id: RequestErrorHandler(
+                agent.agent_id,
+                f'Failed to update calendar of agent {agent.agent_id}')
+        for agent in agents}
+
+    # Continuously update agent calendars
     while True:
         for agent in agents:
-            try:
+            with error_handlers[agent.agent_id]:
                 agent.update_calendar()
-            except requests.exceptions.HTTPError as e:
-                logger.error('Failed to update calendar of agent %s: %s',
-                             agent.agent_id, e)
         time.sleep(config('calendar', 'update_frequency'))
 
 
 def control_camera(camera: Camera):
+    error_handler = RequestErrorHandler(
+            camera.url,
+            f'Failed to communicate with camera {camera}')
     while True:
-        try:
+        with error_handler:
             camera.update_position()
-        except requests.exceptions.HTTPError as e:
-            logger.error('Failed to communicate with camera %s: %s', camera, e)
         time.sleep(1)
 
 
@@ -89,6 +94,9 @@ def main():
         threads.append(control_truead)
         control_truead.start()
 
+    # Start delivering metrics
+    start_metrics_exporter()
+
     try:
         for thread in threads:
             thread.join()

occameracontrol/agent.py

@@ -21,6 +21,8 @@
 from confygure import config
 from dateutil.parser import parse
 
+from occameracontrol.metrics import register_calendar_update
+
 
 logger = logging.getLogger(__name__)
 
@@ -90,6 +92,7 @@ def update_calendar(self):
         logger.debug('Calendar data: %s', calendar)
 
         self.events = self.parse_calendar(calendar)
+        register_calendar_update(self.agent_id)
 
     def active_events(self):
         '''Return a list of active events

occameracontrol/camera.py

@@ -22,6 +22,7 @@
 from requests.auth import HTTPDigestAuth
 
 from occameracontrol.agent import Agent
+from occameracontrol.metrics import register_camera_move
 
 
 logger = logging.getLogger(__name__)
@@ -59,8 +60,7 @@ def __init__(self,
         self.preset_inactive = preset_inactive
 
     def __str__(self):
-        return f"'{self.agent.agent_id}' @ '{self.url}' " \
-                f"(type: '{self.type.value}', position: {self.position})"
+        return f"'{self.agent.agent_id}' @ '{self.url}'"
 
     def move_to_preset(self, preset):
         if self.type == CameraType.panasonic:
@@ -85,6 +85,7 @@ def move_to_preset(self, preset):
             response.raise_for_status()
 
         self.position = preset
+        register_camera_move(self.url, preset)
 
     def update_position(self):
         agent_id = self.agent.agent_id

occameracontrol/metrics.py

@@ -0,0 +1,74 @@
+# Opencast Camera Control
+# Copyright 2024 Osnabrück University, virtUOS
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+import time
+
+from confygure import config
+from prometheus_client import Counter, Gauge
+from prometheus_client import start_http_server
+
+
+logger = logging.getLogger(__name__)
+
+request_errors = Counter('request_errors',
+                         'Number of errors related to HTTP requests',
+                         ('ressource', 'type'))
+agent_calendar_update_total = Gauge('agent_calendar_update_total',
+                                    'Nuber of calendar update',
+                                    ('agent',))
+agent_calendar_update_time = Gauge('agent_calendar_update_time',
+                                   'Time of the last calendar update',
+                                   ('agent',))
+camera_position = Gauge('camera_position',
+                        'Last position (preset number) a camera moved to',
+                        ('camera',))
+
+
+class RequestErrorHandler():
+    def __init__(self, ressource, message):
+        self.ressource = ressource
+        self.message = message
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type:
+            logger.exception(self.message)
+            request_errors.labels(self.ressource, exc_type.__name__).inc()
+        # Silence Exception types
+        return exc_type is None or issubclass(exc_type, Exception)
+
+
+def register_calendar_update(agent_id: str):
+    agent_calendar_update_total.labels(agent_id).inc()
+    agent_calendar_update_time.labels(agent_id).set(time.time())
+
+
+def register_camera_move(camera: str, position: int):
+    camera_position.labels(camera).set(position)
+
+
+def start_metrics_exporter():
+    if not config('metrics', 'enabled'):
+        return
+
+    start_http_server(
+        port=config('metrics', 'port') or 8000,
+        addr=config('metrics', 'addr') or '0.0.0.0',
+        certfile=config('metrics', 'certfile'),
+        keyfile=config('metrics', 'keyfile'))

requirements.txt

@@ -1,2 +1,3 @@
 confygure >= 0.0.3
+prometheus-client
 requests