From 1989d1801ff52e5431b5ed3a3162a9070d146ab6 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Mon, 21 Oct 2024 15:36:16 -0400 Subject: [PATCH 01/11] change coco saving to use the annotation --- .../tcn_hpl/predict.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/angel_system/activity_classification/tcn_hpl/predict.py b/angel_system/activity_classification/tcn_hpl/predict.py index 4715b3f8f..0b5cfe0e9 100644 --- a/angel_system/activity_classification/tcn_hpl/predict.py +++ b/angel_system/activity_classification/tcn_hpl/predict.py @@ -464,19 +464,30 @@ def collect( raise RuntimeError( "No video set before results collection. See `set_video` method." ) - packet = dict( + # get the global id for the image from the frame number + + # add the image + img = dict( video_id=self._vid, frame_index=frame_index, - activity_pred=activity_pred, - activity_conf=list(activity_conf_vec), ) if name is not None: - packet["name"] = name + img["name"] = name if file_name is not None: - packet["file_name"] = file_name + img["file_name"] = file_name if activity_gt is not None: - packet["activity_gt"] = activity_gt - self._dset.add_image(**packet) + img["activity_gt"] = activity_gt + # save the gid from the image to link to the annot + gid = self._dset.add_image(**img) + + # additional items to save + add_items = dict( + prob=list(activity_conf_vec), + ) + # add the annotation + self._dset.add_annotation( + image_id=gid, category_id=activity_pred, **add_items + ) def write_file(self): """ From ccf803230568e10dbd6947ea7abf03825a8274f1 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 09:05:48 -0400 Subject: [PATCH 02/11] add a debug option to the TCN node in order to see the inputs it has when it decides not to create a classification --- .../activity_classifier_tcn.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py index 1117fc074..d6adac23f 100644 --- a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py +++ b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py @@ -102,6 +102,10 @@ # activity prediction for the "live" image will not occur until object # detections are predicted for that frame. PARAM_WINDOW_LEADS_WITH_OBJECTS = "window_leads_with_objects" +# Debug file saved out to the filesystem for understanding the node's +# inputs when it decides not to create an activity classification. +# the format will be csv with a list of the object detections and the pose +PARAM_DEBUG_FILE = "debug_file" class NoActivityClassification(Exception): @@ -148,6 +152,7 @@ def __init__(self): (PARAM_TOPIC, "medical"), (PARAM_POSE_REPEAT_RATE, 0), (PARAM_WINDOW_LEADS_WITH_OBJECTS, False), + (PARAM_DEBUG_FILE, ""), ], ) self._img_ts_topic = param_values[PARAM_IMG_TS_TOPIC] @@ -166,6 +171,12 @@ def __init__(self): self._window_lead_with_objects = param_values[PARAM_WINDOW_LEADS_WITH_OBJECTS] + self._debug_file = param_values[PARAM_DEBUG_FILE] + # clear the file if it exists (since we are appending to it) + if self._debug_file != "": + with open(self._debug_file, "w") as f: + f.write("") + self.topic = param_values[PARAM_TOPIC] # Load in TCN classification model and weights with SimpleTimer("Loading inference module", log.info): @@ -655,6 +666,12 @@ def rt_loop(self): "not yield an activity classification for " "publishing." ) + if self._debug_file != "": + # save the info for why this window was not processed + repr = window.__repr__() + with open(self._debug_file, "a") as f: + f.write(f"timestamp: {self.get_clock().now().to_msg()}\n") + f.write(f"{repr}\n") # This window has completed processing - record its leading # timestamp now. @@ -888,5 +905,34 @@ def destroy_node(self): main = make_default_main(ActivityClassifierTCN, multithreaded_executor=4) +if __name__ == "__main__": + main() + """ + Save results if we have been initialized to do that. + + This method does nothing if this node has not been initialized to + collect results. + """ + rc = self._results_collector + if rc is not None: + self.get_logger().info( + f"Writing classification results to: {self._output_kwcoco_path}" + ) + self._results_collector.write_file() + + def destroy_node(self): + log = self.get_logger() + log.info("Stopping node runtime") + self.rt_stop() + with SimpleTimer("Shutting down runtime thread...", log.info): + self._rt_active.clear() # make RT active flag "False" + self._rt_thread.join() + self._save_results() + super().destroy_node() + + +main = make_default_main(ActivityClassifierTCN, multithreaded_executor=4) + + if __name__ == "__main__": main() From 24b3080c0871ba86c189c688d1e12167d3766039 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 09:07:20 -0400 Subject: [PATCH 03/11] adjust saving of coco output and add score --- angel_system/activity_classification/tcn_hpl/predict.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/angel_system/activity_classification/tcn_hpl/predict.py b/angel_system/activity_classification/tcn_hpl/predict.py index 0b5cfe0e9..94129e01a 100644 --- a/angel_system/activity_classification/tcn_hpl/predict.py +++ b/angel_system/activity_classification/tcn_hpl/predict.py @@ -480,13 +480,12 @@ def collect( # save the gid from the image to link to the annot gid = self._dset.add_image(**img) - # additional items to save - add_items = dict( - prob=list(activity_conf_vec), - ) # add the annotation self._dset.add_annotation( - image_id=gid, category_id=activity_pred, **add_items + image_id=gid, + category_id=activity_pred, + score=activity_conf_vec[activity_pred], + prob=list(activity_conf_vec), ) def write_file(self): From a1e3bef8d3a23225f3f886d82b6814aec771cf28 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 09:08:16 -0400 Subject: [PATCH 04/11] add collection of no activity classification --- .../activity_classification/activity_classifier_tcn.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py index d6adac23f..c52f68640 100644 --- a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py +++ b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py @@ -660,6 +660,15 @@ def rt_loop(self): self._activity_publisher.publish(act_msg) except NoActivityClassification: + # collect the results if we are saving to coco file + if self._results_collector: + # Prepare output message + activity_msg = ActivityDetection() + # set the only needed items for collection + activity_msg.source_stamp_end_frame = window.frames[-1][0] + activity_msg.conf_vec = [0.0 for x in self._model.classes] + self._collect_results(activity_msg) + # No ramifications, but don't publish activity message. log.warn( "Runtime loop window processing function did " From 6ef0d7122d29f062ed6f224c1cfa883007f836e9 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 09:09:22 -0400 Subject: [PATCH 05/11] add a note about usage to the video/image to bag conversion script --- ros/angel_utils/scripts/convert_video_to_ros_bag.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ros/angel_utils/scripts/convert_video_to_ros_bag.py b/ros/angel_utils/scripts/convert_video_to_ros_bag.py index 90b11d16f..af2a4151d 100755 --- a/ros/angel_utils/scripts/convert_video_to_ros_bag.py +++ b/ros/angel_utils/scripts/convert_video_to_ros_bag.py @@ -1,4 +1,12 @@ #!/usr/bin/env python3 +""" +Convert a video (mp4) or a series of images into a ROS bag. + +Example running (inside ROS environment): +ros2 run angel_utils convert_video_to_ros_bag.py \ + --video-fn video.mp4 \ + --output-bag-folder ros_bags/new_bag +""" import argparse from glob import glob from pathlib import Path From f242ea2bcf8d6c4c2d811b60ab45c60379e48930 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 09:45:27 -0400 Subject: [PATCH 06/11] black formatting a clean copy error --- .../activity_classifier_tcn.py | 33 ++----------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py index c52f68640..afe5268c0 100644 --- a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py +++ b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py @@ -679,7 +679,9 @@ def rt_loop(self): # save the info for why this window was not processed repr = window.__repr__() with open(self._debug_file, "a") as f: - f.write(f"timestamp: {self.get_clock().now().to_msg()}\n") + f.write( + f"timestamp: {self.get_clock().now().to_msg()}\n" + ) f.write(f"{repr}\n") # This window has completed processing - record its leading @@ -914,34 +916,5 @@ def destroy_node(self): main = make_default_main(ActivityClassifierTCN, multithreaded_executor=4) -if __name__ == "__main__": - main() - """ - Save results if we have been initialized to do that. - - This method does nothing if this node has not been initialized to - collect results. - """ - rc = self._results_collector - if rc is not None: - self.get_logger().info( - f"Writing classification results to: {self._output_kwcoco_path}" - ) - self._results_collector.write_file() - - def destroy_node(self): - log = self.get_logger() - log.info("Stopping node runtime") - self.rt_stop() - with SimpleTimer("Shutting down runtime thread...", log.info): - self._rt_active.clear() # make RT active flag "False" - self._rt_thread.join() - self._save_results() - super().destroy_node() - - -main = make_default_main(ActivityClassifierTCN, multithreaded_executor=4) - - if __name__ == "__main__": main() From 54e81041c45dc625044787b201b3e7f4b1507d73 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 11:18:23 -0400 Subject: [PATCH 07/11] check the frame number first before ignoring a frame (for playing back bags) --- .../python/angel_utils/activity_classification.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ros/angel_utils/python/angel_utils/activity_classification.py b/ros/angel_utils/python/angel_utils/activity_classification.py index 9b5ea102b..e582bb5b3 100644 --- a/ros/angel_utils/python/angel_utils/activity_classification.py +++ b/ros/angel_utils/python/angel_utils/activity_classification.py @@ -192,13 +192,17 @@ def queue_image( # self.get_logger_fn().info(f"self.frames[-1][0] header stamp: {self.frames[-1][0]}") with self.__state_lock: # before the current lead frame? - if self.frames and time_to_int(img_header_stamp) <= time_to_int( - self.frames[-1][0] + if ( + self.frames + and self.frames[-1][2] == image_frame_number + and time_to_int(img_header_stamp) <= time_to_int(self.frames[-1][0]) ): self.get_logger_fn().warn( f"Input image frame was NOT after the previous latest: " f"(prev) {time_to_int(self.frames[-1][0])} " - f"!< {time_to_int(img_header_stamp)} (new)" + f"!< {time_to_int(img_header_stamp)} (new)\n" + f"frame number: {image_frame_number}\n" + f"prev frame number: {self.frames[-1][2]}" ) return False self.frames.append((img_header_stamp, img_mat, image_frame_number)) From 3650376d43c357ac2707405b33672b0d0c38ebcb Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 12:39:42 -0400 Subject: [PATCH 08/11] drop unneeded argument --- angel_system/activity_classification/tcn_hpl/predict.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/angel_system/activity_classification/tcn_hpl/predict.py b/angel_system/activity_classification/tcn_hpl/predict.py index 94129e01a..1d8e22d4d 100644 --- a/angel_system/activity_classification/tcn_hpl/predict.py +++ b/angel_system/activity_classification/tcn_hpl/predict.py @@ -454,7 +454,6 @@ def collect( activity_conf_vec: Sequence[float], name: Optional[str] = None, file_name: Optional[str] = None, - activity_gt: Optional[int] = None, ) -> None: """ See `CocoDataset.add_image` for more details. @@ -475,8 +474,6 @@ def collect( img["name"] = name if file_name is not None: img["file_name"] = file_name - if activity_gt is not None: - img["activity_gt"] = activity_gt # save the gid from the image to link to the annot gid = self._dset.add_image(**img) From afec82da16027f4e498b224412846b534639e363 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 13:39:09 -0400 Subject: [PATCH 09/11] change coco saving to add an image regardless of whether the activity classification was created --- .../tcn_hpl/predict.py | 54 +++++++++++++++-- .../activity_classifier_tcn.py | 59 +++++++++++++++---- 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/angel_system/activity_classification/tcn_hpl/predict.py b/angel_system/activity_classification/tcn_hpl/predict.py index 1d8e22d4d..b37e87362 100644 --- a/angel_system/activity_classification/tcn_hpl/predict.py +++ b/angel_system/activity_classification/tcn_hpl/predict.py @@ -447,24 +447,45 @@ def set_video(self, video_name: str) -> None: else: self._vid = self._dset.add_video(name=video_name) - def collect( + def check_for_existing_image(self, name, file_name) -> bool: + """ + Check if an image already exists in the dataset. + """ + already_exists = None + if name is not None: + try: + already_exists = self._dset.images().lookup(name) + except KeyError: + pass + if file_name is not None: + try: + already_exists = self._dset.images().lookup(file_name) + except KeyError: + pass + if already_exists: + return True + return False + + def add_image( self, frame_index: int, - activity_pred: int, - activity_conf_vec: Sequence[float], name: Optional[str] = None, file_name: Optional[str] = None, - ) -> None: + ) -> int: """ - See `CocoDataset.add_image` for more details. + Add an image to the dataset. Returns the global image id. + If the image was already added (by name or file name), returns -1. """ with self._lock: if self._vid is None: raise RuntimeError( "No video set before results collection. See `set_video` method." ) - # get the global id for the image from the frame number + # confirm we haven't already added this image + if self.check_for_existing_image(name, file_name): + return -1 + # get the global id for the image from the frame number # add the image img = dict( video_id=self._vid, @@ -477,6 +498,27 @@ def collect( # save the gid from the image to link to the annot gid = self._dset.add_image(**img) + return gid + + def collect( + self, + gid: int, + activity_pred: int, + activity_conf_vec: Sequence[float], + ) -> None: + """ + See `CocoDataset.add_image` for more details. + + :param gid: Global image id. + :param activity_pred: Predicted activity class index. + :param activity_conf_vec: Confidence vector for all activity classes. + """ + with self._lock: + if self._vid is None: + raise RuntimeError( + "No video set before results collection. See `set_video` method." + ) + # add the annotation self._dset.add_annotation( image_id=gid, diff --git a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py index afe5268c0..6a9468f29 100644 --- a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py +++ b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py @@ -543,6 +543,21 @@ def _rt_keep_looping(self) -> bool: # TODO: add has-finished-processing-file-input check. return rt_active + def _save_image_to_coco(self, window: InputWindow) -> int: + """ + This will add an image to the output coco file + if you are not saving to a coco file, this will return -1 + """ + if self._results_collector: + # Prepare output message + activity_msg = ActivityDetection() + # set the only needed items for collection + activity_msg.source_stamp_end_frame = window.frames[-1][0] + activity_msg.conf_vec = [0.0 for x in self._model.classes] + gid = self._collect_image(activity_msg) + return gid + return -1 + def _window_criterion_correct_size(self, window: InputBuffer) -> bool: window_ok = len(window) == self._window_size if not window_ok: @@ -550,6 +565,8 @@ def _window_criterion_correct_size(self, window: InputBuffer) -> bool: f"Window is not the appropriate size " f"(actual:{len(window)} != {self._window_size}:expected)" ) + self._save_image_to_coco(window) + return window_ok def _window_criterion_new_leading_frame(self, window: InputWindow) -> bool: @@ -643,6 +660,7 @@ def rt_loop(self): # out older data at and before the first item in the window. self._buffer.clear_before(time_to_int(window.frames[1][0])) + image_gid = None # set this to None to signal if we saved the image or not try: if enable_time_trace_logging: log.info( @@ -653,22 +671,17 @@ def rt_loop(self): act_msg = self._process_window(window) # log.info(f"activity message: {act_msg}") - self._collect_results(act_msg) + image_gid = self._collect_image(act_msg) + self._collect_results(act_msg, image_gid) # set the header right before publishing so that the time is after processing act_msg.header.frame_id = "Activity Classification" act_msg.header.stamp = self.get_clock().now().to_msg() self._activity_publisher.publish(act_msg) except NoActivityClassification: - # collect the results if we are saving to coco file - if self._results_collector: - # Prepare output message - activity_msg = ActivityDetection() - # set the only needed items for collection - activity_msg.source_stamp_end_frame = window.frames[-1][0] - activity_msg.conf_vec = [0.0 for x in self._model.classes] - self._collect_results(activity_msg) - + # collect the image if we are saving to coco file + if self._results_collector and image_gid is None: + self._save_image_to_coco(window) # No ramifications, but don't publish activity message. log.warn( "Runtime loop window processing function did " @@ -864,7 +877,7 @@ def _process_window(self, window: InputWindow) -> ActivityDetection: return activity_msg - def _collect_results(self, msg: ActivityDetection): + def _collect_image(self, msg: ActivityDetection) -> int: """ Collect into our ResultsCollector instance from the produced activity classification message if we were initialized to do that. @@ -880,10 +893,30 @@ def _collect_results(self, msg: ActivityDetection): # When reading from an input COCO file, this aligns with the input # `image` `frame_index` attributes. frame_index = time_to_int(msg.source_stamp_end_frame) - pred_cls_idx = int(np.argmax(msg.conf_vec)) - rc.collect( + gid = rc.add_image( frame_index=frame_index, name=f"ros-frame-nsec-{frame_index}", + ) + return gid + return -1 + + def _collect_results(self, msg: ActivityDetection, gid: int) -> None: + """ + Collect into our ResultsCollector instance from the produced activity + classification message if we were initialized to do that. + + This method does nothing if this node has not been initialized to + collect results. + + :param msg: ROS2 activity classification message that would be output. + :param gid: Global ID of the image associated with the activity + """ + rc = self._results_collector + if rc is not None: + # use the gid that was created when the image was added + pred_cls_idx = int(np.argmax(msg.conf_vec)) + rc.collect( + gid=gid, activity_pred=pred_cls_idx, activity_conf_vec=list(msg.conf_vec), ) From cc257d84567dce7ee31880372adaa97767797db2 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 16:00:56 -0400 Subject: [PATCH 10/11] fix for beginning frame issue --- .../activity_classification/activity_classifier_tcn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py index 6a9468f29..4f953c4ed 100644 --- a/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py +++ b/ros/angel_system_nodes/angel_system_nodes/activity_classification/activity_classifier_tcn.py @@ -552,7 +552,10 @@ def _save_image_to_coco(self, window: InputWindow) -> int: # Prepare output message activity_msg = ActivityDetection() # set the only needed items for collection - activity_msg.source_stamp_end_frame = window.frames[-1][0] + if len(window.frames) > 0: + activity_msg.source_stamp_end_frame = window.frames[-1][0] + else: + self.get_logger().warn(f"window.frames: {window.frames}") activity_msg.conf_vec = [0.0 for x in self._model.classes] gid = self._collect_image(activity_msg) return gid From aec3eb1d9ac70f35625644de3a4383548adb72b1 Mon Sep 17 00:00:00 2001 From: Joseph VanPelt Date: Fri, 25 Oct 2024 16:02:00 -0400 Subject: [PATCH 11/11] simplify handling images that were already added --- .../tcn_hpl/predict.py | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/angel_system/activity_classification/tcn_hpl/predict.py b/angel_system/activity_classification/tcn_hpl/predict.py index b37e87362..97a2bff14 100644 --- a/angel_system/activity_classification/tcn_hpl/predict.py +++ b/angel_system/activity_classification/tcn_hpl/predict.py @@ -447,25 +447,6 @@ def set_video(self, video_name: str) -> None: else: self._vid = self._dset.add_video(name=video_name) - def check_for_existing_image(self, name, file_name) -> bool: - """ - Check if an image already exists in the dataset. - """ - already_exists = None - if name is not None: - try: - already_exists = self._dset.images().lookup(name) - except KeyError: - pass - if file_name is not None: - try: - already_exists = self._dset.images().lookup(file_name) - except KeyError: - pass - if already_exists: - return True - return False - def add_image( self, frame_index: int, @@ -481,9 +462,6 @@ def add_image( raise RuntimeError( "No video set before results collection. See `set_video` method." ) - # confirm we haven't already added this image - if self.check_for_existing_image(name, file_name): - return -1 # get the global id for the image from the frame number # add the image @@ -496,7 +474,10 @@ def add_image( if file_name is not None: img["file_name"] = file_name # save the gid from the image to link to the annot - gid = self._dset.add_image(**img) + try: + gid = self._dset.add_image(**img) + except Exception: + return -1 # image already exists return gid