From c199640c88f6229e7d396a2aa4599bac5a91b761 Mon Sep 17 00:00:00 2001 From: Lars Kiesow Date: Sat, 10 Feb 2024 15:00:46 +0100 Subject: [PATCH] Add OpenMetrics endpoint This patch add an OpenMetrics endpoint, exposing the current state of cameras and reporting occurring errors. This fixes #14 This fixes #28 --- README.md | 27 ++++++++++++++ camera-control.yml | 15 ++++++++ occameracontrol/__main__.py | 24 ++++++++---- occameracontrol/agent.py | 3 ++ occameracontrol/camera.py | 5 ++- occameracontrol/metrics.py | 74 +++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 7 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 occameracontrol/metrics.py diff --git a/README.md b/README.md index 03b9aff..fc37877 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,30 @@ Control PTZ camera to move to certain presets when starting a scheduled recordin ❯ pip install -r requirements.txt ❯ python -m occameracontrol ``` + +## Metrics + +You can enable an OpenMetrics endpoint in the configuration. This can give you +insight into current camera positions and will report occurring errors. + +The resulting metrics data will look like this: + +``` +# HELP request_errors_total Number of errors related to HTTP requests +# TYPE request_errors_total counter +request_errors_total{ressource="http://camera-3-panasonic.example.com",type="ConnectionError"} 77.0 +request_errors_total{ressource="http://camera-panasonic.example.com",type="ReadTimeout"} 12.0 +# HELP request_errors_created Number of errors related to HTTP requests +# TYPE request_errors_created gauge +request_errors_created{ressource="http://camera-3-panasonic.example.com",type="ConnectionError"} 1.707571882114209e+09 +request_errors_created{ressource="http://camera-panasonic.example.com",type="ReadTimeout"} 1.7075718871156712e+09 +# HELP agent_calendar_update_total Nuber of calendar update +# TYPE agent_calendar_update_total gauge +agent_calendar_update_total{agent="test_agent"} 4.0 +# HELP agent_calendar_update_time Time of the last calendar update +# TYPE agent_calendar_update_time gauge +agent_calendar_update_time{agent="test_agent"} 1.707571943100096e+09 +# HELP camera_position Last position (preset number) a camera moved to +# TYPE camera_position gauge +camera_position{camera="http://camera-2-panasonic.example.com"} 10.0 +``` diff --git a/camera-control.yml b/camera-control.yml index 30b7b37..02117b3 100644 --- a/camera-control.yml +++ b/camera-control.yml @@ -45,4 +45,19 @@ camera: - url: http://camera-sony-15-217.virtuos.uni-osnabrueck.de type: sony +metrics: + # Default: False + enabled: True + # Default: 8000 + port: 8000 + # Default: 0.0.0.0 + addr: '0.0.0.0' + + # Default: null + certfile: null + + # Default: null + keyfile: null + +# Configure the log level loglevel: debug diff --git a/occameracontrol/__main__.py b/occameracontrol/__main__.py index 320f444..19cfb4a 100644 --- a/occameracontrol/__main__.py +++ b/occameracontrol/__main__.py @@ -16,7 +16,6 @@ import argparse import logging -import requests import time from confygure import setup, config @@ -24,28 +23,34 @@ from occameracontrol.agent import Agent from occameracontrol.camera import Camera +from occameracontrol.metrics import start_metrics_exporter, RequestErrorHandler logger = logging.getLogger(__name__) def update_agents(agents: list[Agent]): + error_handlers = { + agent.agent_id: RequestErrorHandler( + agent.agent_id, + f'Failed to update calendar of agent {agent.agent_id}') + for agent in agents} + + # Continuously update agent calendars while True: for agent in agents: - try: + with error_handlers[agent.agent_id]: agent.update_calendar() - except requests.exceptions.HTTPError as e: - logger.error('Failed to update calendar of agent %s: %s', - agent.agent_id, e) time.sleep(config('calendar', 'update_frequency')) def control_camera(camera: Camera): + error_handler = RequestErrorHandler( + camera.url, + f'Failed to communicate with camera {camera}') while True: - try: + with error_handler: camera.update_position() - except requests.exceptions.HTTPError as e: - logger.error('Failed to communicate with camera %s: %s', camera, e) time.sleep(1) @@ -89,6 +94,9 @@ def main(): threads.append(control_truead) control_truead.start() + # Start delivering metrics + start_metrics_exporter() + try: for thread in threads: thread.join() diff --git a/occameracontrol/agent.py b/occameracontrol/agent.py index 211f4cd..51dc07a 100644 --- a/occameracontrol/agent.py +++ b/occameracontrol/agent.py @@ -21,6 +21,8 @@ from confygure import config from dateutil.parser import parse +from occameracontrol.metrics import register_calendar_update + logger = logging.getLogger(__name__) @@ -90,6 +92,7 @@ def update_calendar(self): logger.debug('Calendar data: %s', calendar) self.events = self.parse_calendar(calendar) + register_calendar_update(self.agent_id) def active_events(self): '''Return a list of active events diff --git a/occameracontrol/camera.py b/occameracontrol/camera.py index c109257..c4022eb 100644 --- a/occameracontrol/camera.py +++ b/occameracontrol/camera.py @@ -22,6 +22,7 @@ from requests.auth import HTTPDigestAuth from occameracontrol.agent import Agent +from occameracontrol.metrics import register_camera_move logger = logging.getLogger(__name__) @@ -59,8 +60,7 @@ def __init__(self, self.preset_inactive = preset_inactive def __str__(self): - return f"'{self.agent.agent_id}' @ '{self.url}' " \ - f"(type: '{self.type.value}', position: {self.position})" + return f"'{self.agent.agent_id}' @ '{self.url}'" def move_to_preset(self, preset): if self.type == CameraType.panasonic: @@ -85,6 +85,7 @@ def move_to_preset(self, preset): response.raise_for_status() self.position = preset + register_camera_move(self.url, preset) def update_position(self): agent_id = self.agent.agent_id diff --git a/occameracontrol/metrics.py b/occameracontrol/metrics.py new file mode 100644 index 0000000..498f76c --- /dev/null +++ b/occameracontrol/metrics.py @@ -0,0 +1,74 @@ +# Opencast Camera Control +# Copyright 2024 Osnabrück University, virtUOS +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import time + +from confygure import config +from prometheus_client import Counter, Gauge +from prometheus_client import start_http_server + + +logger = logging.getLogger(__name__) + +request_errors = Counter('request_errors', + 'Number of errors related to HTTP requests', + ('ressource', 'type')) +agent_calendar_update_total = Gauge('agent_calendar_update_total', + 'Nuber of calendar update', + ('agent',)) +agent_calendar_update_time = Gauge('agent_calendar_update_time', + 'Time of the last calendar update', + ('agent',)) +camera_position = Gauge('camera_position', + 'Last position (preset number) a camera moved to', + ('camera',)) + + +class RequestErrorHandler(): + def __init__(self, ressource, message): + self.ressource = ressource + self.message = message + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type: + logger.exception(self.message) + request_errors.labels(self.ressource, exc_type.__name__).inc() + # Silence Exception types + return exc_type is None or issubclass(exc_type, Exception) + + +def register_calendar_update(agent_id: str): + agent_calendar_update_total.labels(agent_id).inc() + agent_calendar_update_time.labels(agent_id).set(time.time()) + + +def register_camera_move(camera: str, position: int): + camera_position.labels(camera).set(position) + + +def start_metrics_exporter(): + if not config('metrics', 'enabled'): + return + + start_http_server( + port=config('metrics', 'port') or 8000, + addr=config('metrics', 'addr') or '0.0.0.0', + certfile=config('metrics', 'certfile'), + keyfile=config('metrics', 'keyfile')) diff --git a/requirements.txt b/requirements.txt index 4f47a5b..8155850 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ confygure >= 0.0.3 +prometheus-client requests