From a08d36ad18153607369162dca1d1608f40bfda51 Mon Sep 17 00:00:00 2001 From: David Robb Date: Mon, 4 Jan 2021 17:08:56 +1300 Subject: [PATCH 01/11] Add prometheus client support, and expose first metric (frames counted) --- setup.py | 1 + umt/umt_main.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a6af936..f956fc8 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ 'matplotlib', 'numpy', 'Pillow', + 'prometheus_client', 'scipy', 'scikit-image' ], diff --git a/umt/umt_main.py b/umt/umt_main.py index bf98add..abd77f4 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -14,6 +14,8 @@ from umt.umt_utils import persist_image_output from umt.umt_utils import plot_colors +from prometheus_client import start_http_server, Summary, Counter + #--- CONSTANTS ----------------------------------------------------------------+ LABEL_PATH = "models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29/labelmap.txt" @@ -43,6 +45,8 @@ def main(): if args.label_map_path: assert os.path.exists(args.label_map_path)==True, "can't find the specified label map..." if args.video_path: assert os.path.exists(args.video_path)==True, "can't find the specified video file..." + # initialize counters for metrics + frames = Counter('frame_counter', 'Number of frames processed') print('> INITIALIZING UMT...') print(' > THRESHOLD:',args.threshold) @@ -72,6 +76,7 @@ def main(): f_time = int(time.time()) print('> FRAME:', i) + frames.inc() # get detections new_dets, classes, scores = generate_detections(pil_img, interpreter, args.threshold) @@ -104,6 +109,7 @@ def main(): #--- MAIN ---------------------------------------------------------------------+ if __name__ == '__main__': + start_http_server(8000) main() - + #--- END ----------------------------------------------------------------------+ From f45c87e8b9f395f672d1d3e59689ccd290b7caa8 Mon Sep 17 00:00:00 2001 From: David Robb Date: Mon, 4 Jan 2021 19:02:50 +1300 Subject: [PATCH 02/11] Add -nolog parameter, to disable logging to object_paths.txt (e.g. if using metrics) Initialise metric counter for each label in use Move frame counter metric initialisation call next to label metic counter initialisation --- umt/umt_main.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index abd77f4..9214033 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -36,6 +36,7 @@ def main(): parser.add_argument('-tpu', dest='tpu', required=False, default=False, action='store_true', help='add this when using a coral usb accelerator') parser.add_argument('-nframes', dest='nframes', type=int, required=False, default=10, help='specify nunber of frames to process') parser.add_argument('-display', dest='display', required=False, default=False, action='store_true', help='add this flag to output images from tracker. note, that this will greatly slow down the fps rate.') + parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.txt. note, file is still created, just not written to.') args = parser.parse_args() @@ -45,9 +46,6 @@ def main(): if args.label_map_path: assert os.path.exists(args.label_map_path)==True, "can't find the specified label map..." if args.video_path: assert os.path.exists(args.video_path)==True, "can't find the specified video file..." - # initialize counters for metrics - frames = Counter('frame_counter', 'Number of frames processed') - print('> INITIALIZING UMT...') print(' > THRESHOLD:',args.threshold) @@ -57,6 +55,13 @@ def main(): # parse label map labels = parse_label_map(args, DEFAULT_LABEL_MAP_PATH) + # initialize counters for metrics + frames = Counter('umt_frame_counter', 'Number of frames processed') + + label_counter = Counter ('umt_label_counter', 'Number of each label counted', ['type']) + for label in labels.values(): + label_counter.labels(type=label) + # create output directory if not os.path.exists('output'): os.makedirs('output') @@ -97,7 +102,8 @@ def main(): # save object locations for d, tracker_label, tracker_score in zip(trackers, tracker_labels, tracker_scores): - print(f'{i},{f_time},{d[4]},{d[0]},{d[1]},{d[2]-d[0]},{d[3]-d[1]},{tracker_label},{tracker_score}', file=out_file) + if not(args.nolog): + print(f'{i},{f_time},{d[4]},{d[0]},{d[1]},{d[2]-d[0]},{d[3]-d[1]},{tracker_label},{tracker_score}', file=out_file) except: print(' > TRACKER FAILED...') From e246942992b000e52b2aa7d73301fa991f85f2db Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 13:15:19 +1300 Subject: [PATCH 03/11] Add in deepsort patch --- umt/deep_sort/generate_detections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/umt/deep_sort/generate_detections.py b/umt/deep_sort/generate_detections.py index 2d68f11..9ce1e9a 100644 --- a/umt/deep_sort/generate_detections.py +++ b/umt/deep_sort/generate_detections.py @@ -78,9 +78,9 @@ def __init__(self, checkpoint_filename, input_name="images", graph_def.ParseFromString(file_handle.read()) tf.import_graph_def(graph_def, name="net") self.input_var = tf.get_default_graph().get_tensor_by_name( - "%s:0" % input_name) + "net/%s:0" % input_name) self.output_var = tf.get_default_graph().get_tensor_by_name( - "%s:0" % output_name) + "net/%s:0" % output_name) assert len(self.output_var.get_shape()) == 2 assert len(self.input_var.get_shape()) == 4 From 366d11b7072230085ca65d457b3d7177638b769f Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 15:55:37 +1300 Subject: [PATCH 04/11] Add metrics for detected/no-detection/errors Fix no-log argument for new deepsort module --- umt/umt_main.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index 7573b15..12e23c7 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -46,6 +46,7 @@ def main(): parser.add_argument('-nframes', dest='nframes', type=int, required=False, default=10, help='specify nunber of frames to process') parser.add_argument('-display', dest='live_view', required=False, default=False, action='store_true', help='add this flag to view a live display. note, that this will greatly slow down the fps rate.') parser.add_argument('-save', dest='save_frames', required=False, default=False, action='store_true', help='add this flag if you want to persist the image output. note, that this will greatly slow down the fps rate.') + parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='enable prometheus metrics on port 8000') parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.txt. note, file is still created, just not written to.') args = parser.parse_args() @@ -64,12 +65,16 @@ def main(): # initialize detector interpreter = initialize_detector(args) - # initialize counters for metrics - frames = Counter('umt_frame_counter', 'Number of frames processed') + if args.metrics: + # initialize counters for metrics + frames = Counter('umt_frame_counter', 'Number of frames processed', ['result']) + frames.labels(result='no_detection') + frames.labels(result='detection') + frames.labels(result='error') - label_counter = Counter ('umt_label_counter', 'Number of each label counted', ['type']) - for label in labels.values(): - label_counter.labels(type=label) + label_counter = Counter ('umt_label_counter', 'Number of each label counted', ['type']) + for label in labels.values(): + label_counter.labels(type=label) # create output directory if not os.path.exists('output') and args.save_frames: os.makedirs('output') @@ -103,15 +108,19 @@ def main(): detections = generate_detections(pil_img, interpreter, args.threshold) # proceed to updating state - if len(detections) == 0: print(' > no detections...') + if len(detections) == 0: + print(' > no detections...') + if args.metrics: frames.labels(result='no_detection').inc() else: - + # update metric + if args.metrics: frames.labels(result='detection').inc() + # update tracker tracker.predict() tracker.update(detections) # save object locations - if len(tracker.tracks) > 0: + if len(tracker.tracks) > 0 and not args.nolog: for track in tracker.tracks: bbox = track.to_tlbr() class_name = labels[track.get_class()] From a21e39c99cfcbec31f2391f44ad88be5f441c0f8 Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 17:09:29 +1300 Subject: [PATCH 05/11] Add track_count metric for total number of objects tracked Increment metrics for newly seen objects --- umt/umt_main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index 12e23c7..c3a753d 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -18,7 +18,7 @@ from umt.umt_utils import initialize_img_source from umt.umt_utils import generate_detections -from prometheus_client import start_http_server, Summary, Counter +from prometheus_client import start_http_server, Summary, Counter, Gauge #--- CONSTANTS ----------------------------------------------------------------+ @@ -76,6 +76,9 @@ def main(): for label in labels.values(): label_counter.labels(type=label) + track_count_hwm = 0 # Track id high water mark + track_count = Gauge('umt_tracked_objects', 'Number of objects that have been tracked') + # create output directory if not os.path.exists('output') and args.save_frames: os.makedirs('output') @@ -120,7 +123,7 @@ def main(): tracker.update(detections) # save object locations - if len(tracker.tracks) > 0 and not args.nolog: + if len(tracker.tracks) > 0: for track in tracker.tracks: bbox = track.to_tlbr() class_name = labels[track.get_class()] @@ -129,7 +132,12 @@ def main(): f'{int(track.time_since_update)},{str(track.hits)},' f'{int(bbox[0])},{int(bbox[1])},' f'{int(bbox[2])},{int(bbox[3])}') - print(row, file=out_file) + if not args.nolog: print(row, file=out_file) + if args.metrics: # update the metrics + if track.track_id > track_count_hwm: # new thing being tracked + track_count_hwm = track.track_id + track_count.set(track.track_id) + label_counter.labels(type=class_name).inc() # only for live display if args.live_view or args.save_frames: From 56885357bd948d1715846f79bcc7247902622fa5 Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 18:01:10 +1300 Subject: [PATCH 06/11] Make metric port configurable Move http server (for metric) start inside main() so it can reference arguments --- umt/umt_main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index c3a753d..a80ab16 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -46,7 +46,8 @@ def main(): parser.add_argument('-nframes', dest='nframes', type=int, required=False, default=10, help='specify nunber of frames to process') parser.add_argument('-display', dest='live_view', required=False, default=False, action='store_true', help='add this flag to view a live display. note, that this will greatly slow down the fps rate.') parser.add_argument('-save', dest='save_frames', required=False, default=False, action='store_true', help='add this flag if you want to persist the image output. note, that this will greatly slow down the fps rate.') - parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='enable prometheus metrics on port 8000') + parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='enable prometheus metrics') + parser.add_argument('-metricport', dest='metric_port', type=int, required=False, default=8000, help='prometheus metrics port (default 8000)') parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.txt. note, file is still created, just not written to.') args = parser.parse_args() @@ -79,6 +80,10 @@ def main(): track_count_hwm = 0 # Track id high water mark track_count = Gauge('umt_tracked_objects', 'Number of objects that have been tracked') + print(' > METRIC PORT',args.metric_port) + print(' > STARTING METRIC SERVER') + start_http_server(args.metric_port) + # create output directory if not os.path.exists('output') and args.save_frames: os.makedirs('output') @@ -173,7 +178,6 @@ def main(): #--- MAIN ---------------------------------------------------------------------+ if __name__ == '__main__': - start_http_server(8000) main() #--- END ----------------------------------------------------------------------+ From 89227a36805305a3d15304ee8b9fc5a902eb8366 Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 18:10:46 +1300 Subject: [PATCH 07/11] Add new argument -initlabelcounters to always return a value (e.g. 0) for all possible model labels, otherwise label counter metrics are added as they are seen --- umt/umt_main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index a80ab16..7f5566f 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -48,6 +48,7 @@ def main(): parser.add_argument('-save', dest='save_frames', required=False, default=False, action='store_true', help='add this flag if you want to persist the image output. note, that this will greatly slow down the fps rate.') parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='enable prometheus metrics') parser.add_argument('-metricport', dest='metric_port', type=int, required=False, default=8000, help='prometheus metrics port (default 8000)') + parser.add_argument('-initlabelcounters', dest='init_label_counters', required=False, default=False, action='store_true', help='metrics return 0 for all possible label counters available in the model') parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.txt. note, file is still created, just not written to.') args = parser.parse_args() @@ -74,8 +75,9 @@ def main(): frames.labels(result='error') label_counter = Counter ('umt_label_counter', 'Number of each label counted', ['type']) - for label in labels.values(): - label_counter.labels(type=label) + if args.init_label_counters: + for label in labels.values(): + label_counter.labels(type=label) track_count_hwm = 0 # Track id high water mark track_count = Gauge('umt_tracked_objects', 'Number of objects that have been tracked') From 16da8b7284afa750e4cd09e9ba5fcecfca9df1fc Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 18:12:27 +1300 Subject: [PATCH 08/11] Update -nolog parameter with new description --- umt/umt_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index 7f5566f..80b208f 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -49,7 +49,7 @@ def main(): parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='enable prometheus metrics') parser.add_argument('-metricport', dest='metric_port', type=int, required=False, default=8000, help='prometheus metrics port (default 8000)') parser.add_argument('-initlabelcounters', dest='init_label_counters', required=False, default=False, action='store_true', help='metrics return 0 for all possible label counters available in the model') - parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.txt. note, file is still created, just not written to.') + parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.csv. note, file is still created, just not written to.') args = parser.parse_args() # basic checks From bda9f40c8de86c3dd7c99daf7d55e21bb81e8a68 Mon Sep 17 00:00:00 2001 From: David Robb Date: Thu, 7 Jan 2021 18:14:02 +1300 Subject: [PATCH 09/11] Update new argument descriptions to be consistent with existing ones --- umt/umt_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index 80b208f..49f2b0a 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -46,9 +46,9 @@ def main(): parser.add_argument('-nframes', dest='nframes', type=int, required=False, default=10, help='specify nunber of frames to process') parser.add_argument('-display', dest='live_view', required=False, default=False, action='store_true', help='add this flag to view a live display. note, that this will greatly slow down the fps rate.') parser.add_argument('-save', dest='save_frames', required=False, default=False, action='store_true', help='add this flag if you want to persist the image output. note, that this will greatly slow down the fps rate.') - parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='enable prometheus metrics') - parser.add_argument('-metricport', dest='metric_port', type=int, required=False, default=8000, help='prometheus metrics port (default 8000)') - parser.add_argument('-initlabelcounters', dest='init_label_counters', required=False, default=False, action='store_true', help='metrics return 0 for all possible label counters available in the model') + parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='add this flag to enable prometheus metrics') + parser.add_argument('-metricport', dest='metric_port', type=int, required=False, default=8000, help='specify the prometheus metrics port (default 8000)') + parser.add_argument('-initlabelcounters', dest='init_label_counters', required=False, default=False, action='store_true', help='add this flag to return 0 for all possible label counter metrics available in the model') parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.csv. note, file is still created, just not written to.') args = parser.parse_args() From 0999b738dfecaac30cd7097e0c1f1ea0f1b9e63e Mon Sep 17 00:00:00 2001 From: David Robb Date: Mon, 11 Jan 2021 11:49:58 +1300 Subject: [PATCH 10/11] Revert change in setup.py for Pillow vs pillow (to be consistent with origin) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab65dc9..a987379 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ 'filterpy', 'imutils', 'numpy', - 'Pillow', + 'pillow', 'opencv-python', 'prometheus_client', 'scipy', From 5a76e4e5e1ee7197f8f9113b9d86516854f2aebc Mon Sep 17 00:00:00 2001 From: David Robb Date: Mon, 11 Jan 2021 11:58:55 +1300 Subject: [PATCH 11/11] Rename label_counter to object_counter to make language consistent --- umt/umt_main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/umt/umt_main.py b/umt/umt_main.py index 49f2b0a..f70e826 100644 --- a/umt/umt_main.py +++ b/umt/umt_main.py @@ -48,7 +48,7 @@ def main(): parser.add_argument('-save', dest='save_frames', required=False, default=False, action='store_true', help='add this flag if you want to persist the image output. note, that this will greatly slow down the fps rate.') parser.add_argument('-metrics', dest='metrics', required=False, default=False, action='store_true', help='add this flag to enable prometheus metrics') parser.add_argument('-metricport', dest='metric_port', type=int, required=False, default=8000, help='specify the prometheus metrics port (default 8000)') - parser.add_argument('-initlabelcounters', dest='init_label_counters', required=False, default=False, action='store_true', help='add this flag to return 0 for all possible label counter metrics available in the model') + parser.add_argument('-initlabelcounters', dest='init_object_counters', required=False, default=False, action='store_true', help='add this flag to return 0 for all possible object counter metrics available in the model') parser.add_argument('-nolog', dest='nolog', required=False, default=False, action='store_true', help='add this flag to disable logging to object_paths.csv. note, file is still created, just not written to.') args = parser.parse_args() @@ -74,10 +74,10 @@ def main(): frames.labels(result='detection') frames.labels(result='error') - label_counter = Counter ('umt_label_counter', 'Number of each label counted', ['type']) - if args.init_label_counters: + object_counter = Counter ('umt_object_counter', 'Number of each object counted', ['type']) + if args.init_object_counters: for label in labels.values(): - label_counter.labels(type=label) + object_counter.labels(type=label) track_count_hwm = 0 # Track id high water mark track_count = Gauge('umt_tracked_objects', 'Number of objects that have been tracked') @@ -144,7 +144,7 @@ def main(): if track.track_id > track_count_hwm: # new thing being tracked track_count_hwm = track.track_id track_count.set(track.track_id) - label_counter.labels(type=class_name).inc() + object_counter.labels(type=class_name).inc() # only for live display if args.live_view or args.save_frames: