/* * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual * property and proprietary rights in and to this material, related * documentation and any modifications thereto. Any use, reproduction, * disclosure or distribution of this material and related documentation * without an express license agreement from NVIDIA CORPORATION or * its affiliates is strictly prohibited. */ #include #include #include #include #include #include #include "nvds_yml_parser.h" #include "gstnvdsmeta.h" #define PGIE_CONFIG_FILE "dstest2_pgie_nvinferserver_config.txt" #define SGIE1_CONFIG_FILE "dstest2_sgie1_nvinferserver_config.txt" #define SGIE2_CONFIG_FILE "dstest2_sgie2_nvinferserver_config.txt" #define MAX_DISPLAY_LEN 64 #define TRACKER_CONFIG_FILE "dstest2_tracker_config.txt" #define MAX_TRACKING_ID_LEN 16 #define PGIE_CLASS_ID_VEHICLE 0 #define PGIE_CLASS_ID_PERSON 2 /* The muxer output resolution must be set if the input streams will be of * different resolution. The muxer will scale all the input frames to this * resolution. */ #define MUXER_OUTPUT_WIDTH 1920 #define MUXER_OUTPUT_HEIGHT 1080 /* Muxer batch formation timeout, for e.g. 40 millisec. Should ideally be set * based on the fastest source's framerate. */ #define MUXER_BATCH_TIMEOUT_USEC 40000 /* Create an inference element instance of the specified type. */ #define CREATE_GIE_INSTANCE(element, type, name) \ if ((type) == NVDS_GIE_PLUGIN_INFER_SERVER) { \ element = gst_element_factory_make ("nvinferserver", name); \ } else { \ element = gst_element_factory_make ("nvinfer", name); \ } /* Check for parsing error. */ #define RETURN_ON_PARSER_ERROR(parse_expr) \ if (NVDS_YAML_PARSER_SUCCESS != parse_expr) { \ g_printerr("Error in parsing configuration file.\n"); \ return -1; \ } gint frame_number = 0; /* These are the strings of the labels for the respective models */ gchar sgie1_classes_str[20][32] = { "Acura", "Audi", "BMW", "Chevrolet", "Chrysler", "Dodge", "Ford", "GMC", "Honda", "Hyundai", "Infiniti", "Jeep", "Kia", "Lexus", "Mazda", "Mercedes", "Nissan", "Subaru", "Toyota", "Volkswagen" }; gchar sgie2_classes_str[6][32] = { "coupe", "largevehicle", "sedan", "suv", "truck", "van" }; gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person", "RoadSign" }; /* gie_unique_id is one of the properties in the above dstest2_sgiex_config.txt * files. These should be unique and known when we want to parse the Metadata * respective to the sgie labels. Ideally these should be read from the config * files but for brevity we ensure they are same. */ guint sgie1_unique_id = 2; guint sgie2_unique_id = 3; /* This is the buffer probe function that we have registered on the sink pad * of the OSD element. All the infer elements in the pipeline shall attach * their metadata to the GstBuffer, here we will iterate & process the metadata * forex: class ids to strings, counting of class_id objects etc. */ static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data) { GstBuffer *buf = (GstBuffer *) info->data; guint num_rects = 0; NvDsObjectMeta *obj_meta = NULL; guint vehicle_count = 0; guint person_count = 0; NvDsMetaList * l_frame = NULL; NvDsMetaList * l_obj = NULL; NvDsDisplayMeta *display_meta = NULL; NvDsBatchMeta *batch_meta = gst_buffer_get_nvds_batch_meta (buf); for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next) { NvDsFrameMeta *frame_meta = (NvDsFrameMeta *) (l_frame->data); int offset = 0; for (l_obj = frame_meta->obj_meta_list; l_obj != NULL; l_obj = l_obj->next) { obj_meta = (NvDsObjectMeta *) (l_obj->data); if (obj_meta->class_id == PGIE_CLASS_ID_VEHICLE) { vehicle_count++; num_rects++; } if (obj_meta->class_id == PGIE_CLASS_ID_PERSON) { person_count++; num_rects++; } // printf("labels is %s\n", obj_meta->obj_label); } display_meta = nvds_acquire_display_meta_from_pool(batch_meta); NvOSD_TextParams *txt_params = &display_meta->text_params[0]; display_meta->num_labels = 1; txt_params->display_text = g_malloc0 (MAX_DISPLAY_LEN); offset = snprintf(txt_params->display_text, MAX_DISPLAY_LEN, "Person = %d ", person_count); offset = snprintf(txt_params->display_text + offset , MAX_DISPLAY_LEN, "Vehicle = %d ", vehicle_count); /* Now set the offsets where the string should appear */ txt_params->x_offset = 10; txt_params->y_offset = 12; /* Font , font-color and font-size */ txt_params->font_params.font_name = "Serif"; txt_params->font_params.font_size = 10; txt_params->font_params.font_color.red = 1.0; txt_params->font_params.font_color.green = 1.0; txt_params->font_params.font_color.blue = 1.0; txt_params->font_params.font_color.alpha = 1.0; /* Text background color */ txt_params->set_bg_clr = 1; txt_params->text_bg_clr.red = 0.0; txt_params->text_bg_clr.green = 0.0; txt_params->text_bg_clr.blue = 0.0; txt_params->text_bg_clr.alpha = 1.0; nvds_add_display_meta_to_frame(frame_meta, display_meta); } g_print ("Frame Number = %d Number of objects = %d " "Vehicle Count = %d Person Count = %d\n", frame_number, num_rects, vehicle_count, person_count); frame_number++; return GST_PAD_PROBE_OK; } static gboolean bus_call (GstBus * bus, GstMessage * msg, gpointer data) { GMainLoop *loop = (GMainLoop *) data; switch (GST_MESSAGE_TYPE (msg)) { case GST_MESSAGE_EOS: g_print ("End of stream\n"); g_main_loop_quit (loop); break; case GST_MESSAGE_ERROR:{ gchar *debug = NULL; GError *error = NULL; gst_message_parse_error (msg, &error, &debug); g_printerr ("ERROR from element %s: %s\n", GST_OBJECT_NAME (msg->src), error->message); if (debug) g_printerr ("Error details: %s\n", debug); g_free (debug); g_error_free (error); g_main_loop_quit (loop); break; } default: break; } return TRUE; } /* Tracker config parsing */ #define CHECK_ERROR(error) \ if (error) { \ g_printerr ("Error while parsing config file: %s\n", error->message); \ goto done; \ } #define CONFIG_GROUP_TRACKER "tracker" #define CONFIG_GROUP_TRACKER_WIDTH "tracker-width" #define CONFIG_GROUP_TRACKER_HEIGHT "tracker-height" #define CONFIG_GROUP_TRACKER_LL_CONFIG_FILE "ll-config-file" #define CONFIG_GROUP_TRACKER_LL_LIB_FILE "ll-lib-file" #define CONFIG_GPU_ID "gpu-id" static gchar * get_absolute_file_path (gchar *cfg_file_path, gchar *file_path) { gchar abs_cfg_path[PATH_MAX + 1]; gchar *abs_file_path; gchar *delim; if (file_path && file_path[0] == '/') { return file_path; } if (!realpath (cfg_file_path, abs_cfg_path)) { g_free (file_path); return NULL; } // Return absolute path of config file if file_path is NULL. if (!file_path) { abs_file_path = g_strdup (abs_cfg_path); return abs_file_path; } delim = g_strrstr (abs_cfg_path, "/"); *(delim + 1) = '\0'; abs_file_path = g_strconcat (abs_cfg_path, file_path, NULL); g_free (file_path); return abs_file_path; } static gboolean set_tracker_properties (GstElement *nvtracker) { gboolean ret = FALSE; GError *error = NULL; gchar **keys = NULL; gchar **key = NULL; GKeyFile *key_file = g_key_file_new (); if (!g_key_file_load_from_file (key_file, TRACKER_CONFIG_FILE, G_KEY_FILE_NONE, &error)) { if (error) { g_printerr ("Failed to load config file: %s\n", error->message); g_error_free (error); } else { g_printerr ("Failed to load config file.\n"); } return FALSE; } keys = g_key_file_get_keys (key_file, CONFIG_GROUP_TRACKER, NULL, &error); CHECK_ERROR (error); for (key = keys; *key; key++) { if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_WIDTH)) { gint width = g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, CONFIG_GROUP_TRACKER_WIDTH, &error); CHECK_ERROR (error); g_object_set (G_OBJECT (nvtracker), "tracker-width", width, NULL); } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_HEIGHT)) { gint height = g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, CONFIG_GROUP_TRACKER_HEIGHT, &error); CHECK_ERROR (error); g_object_set (G_OBJECT (nvtracker), "tracker-height", height, NULL); } else if (!g_strcmp0 (*key, CONFIG_GPU_ID)) { guint gpu_id = g_key_file_get_integer (key_file, CONFIG_GROUP_TRACKER, CONFIG_GPU_ID, &error); CHECK_ERROR (error); g_object_set (G_OBJECT (nvtracker), "gpu_id", gpu_id, NULL); } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE)) { char* ll_config_file = get_absolute_file_path (TRACKER_CONFIG_FILE, g_key_file_get_string (key_file, CONFIG_GROUP_TRACKER, CONFIG_GROUP_TRACKER_LL_CONFIG_FILE, &error)); CHECK_ERROR (error); g_object_set (G_OBJECT (nvtracker), "ll-config-file", ll_config_file, NULL); g_free(ll_config_file); } else if (!g_strcmp0 (*key, CONFIG_GROUP_TRACKER_LL_LIB_FILE)) { char* ll_lib_file = get_absolute_file_path (TRACKER_CONFIG_FILE, g_key_file_get_string (key_file, CONFIG_GROUP_TRACKER, CONFIG_GROUP_TRACKER_LL_LIB_FILE, &error)); CHECK_ERROR (error); g_object_set (G_OBJECT (nvtracker), "ll-lib-file", ll_lib_file, NULL); g_free(ll_lib_file); } else { g_printerr ("Unknown key '%s' for group [%s]", *key, CONFIG_GROUP_TRACKER); } } ret = TRUE; done: if (error) { g_error_free (error); } if (keys) { g_strfreev (keys); } if (!ret) { g_printerr ("%s failed", __func__); } return ret; } int main (int argc, char *argv[]) { GMainLoop *loop = NULL; GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL, *decoder = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL, *nvosd = NULL, *sgie1 = NULL, *sgie2 = NULL, *nvtracker = NULL, *encoder = NULL, *h264parser1 = NULL, *mp4mux = NULL; g_print ("With tracker\n"); GstBus *bus = NULL; guint bus_watch_id = 0; GstPad *osd_sink_pad = NULL; gboolean yaml_config = FALSE; NvDsGieType pgie_type = NVDS_GIE_PLUGIN_INFER_SERVER; NvDsGieType sgie1_type = NVDS_GIE_PLUGIN_INFER_SERVER; NvDsGieType sgie2_type = NVDS_GIE_PLUGIN_INFER_SERVER; int current_device = -1; cudaGetDevice(¤t_device); struct cudaDeviceProp prop; cudaGetDeviceProperties(&prop, current_device); /* Check input arguments */ if (argc != 2 && argc != 3) { g_printerr ("Usage: %s \n", argv[0]); g_printerr ("OR: %s \n", argv[0]); return -1; } /* Standard GStreamer initialization */ gst_init (&argc, &argv); loop = g_main_loop_new (NULL, FALSE); /* Parse inference plugin type */ yaml_config = (g_str_has_suffix (argv[1], ".yml") || g_str_has_suffix (argv[1], ".yaml")); if (yaml_config) { RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&pgie_type, argv[1], "primary-gie")); RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&sgie1_type, argv[1], "secondary-gie1")); RETURN_ON_PARSER_ERROR(nvds_parse_gie_type(&sgie2_type, argv[1], "secondary-gie2")); } /* Create gstreamer elements */ /* Create Pipeline element that will be a container of other elements */ pipeline = gst_pipeline_new ("dstest2-pipeline"); /* Source element for reading from the file */ source = gst_element_factory_make ("filesrc", "file-source"); /* Since the data format in the input file is elementary h264 stream, * we need a h264parser */ h264parser = gst_element_factory_make ("h264parse", "h264-parser"); /* Use nvdec_h264 for hardware accelerated decode on GPU */ decoder = gst_element_factory_make ("mach264dec", "mach264-decoder"); /* Create nvstreammux instance to form batches from one or more sources. */ streammux = gst_element_factory_make ("nvstreammux", "stream-muxer"); if (!pipeline || !streammux) { g_printerr ("One element could not be created. Exiting1.\n"); return -1; } /* Use nvinfer or nvinferserver to run inferencing on decoder's output, * behaviour of inferencing is set through config file */ CREATE_GIE_INSTANCE(pgie, pgie_type, "primary-nvinference-engine"); /* We need to have a tracker to track the identified objects */ nvtracker = gst_element_factory_make ("nvtracker", "tracker"); /* We need three secondary gies so lets create 3 more instances of nvinfer or nvinferserver */ CREATE_GIE_INSTANCE(sgie1, sgie1_type, "secondary1-nvinference-engine"); CREATE_GIE_INSTANCE(sgie2, sgie2_type, "secondary2-nvinference-engine"); /* Create OSD to draw on the converted RGBA buffer */ nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay"); encoder = gst_element_factory_make ("maccudah264enc", "mac-cudah264enc"); h264parser1 = gst_element_factory_make ("h264parse", "h264-parser1"); mp4mux = gst_element_factory_make ("qtmux", "qt-mux"); sink = gst_element_factory_make ("filesink", "file-sink"); if (!source || !h264parser || !decoder || !pgie || !nvtracker || !sgie1 || !sgie2 || !nvosd || !encoder || !h264parser1 || !mp4mux || !sink) { g_printerr ("One element could not be created. Exiting2.\n"); return -1; } if (g_str_has_suffix (argv[1], ".h264")) { /* Set the input filename to the source element */ g_object_set (G_OBJECT (source), "location", argv[1], NULL); g_object_set (G_OBJECT (streammux), "batch-size", 1, NULL); g_object_set (G_OBJECT (streammux), "gpu-id", 3, NULL); g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height", MUXER_OUTPUT_HEIGHT, "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL); /* Set all the necessary properties of the nvinfer element, * the necessary ones are : */ g_object_set (G_OBJECT (pgie), "config-file-path", PGIE_CONFIG_FILE, NULL); g_object_set (G_OBJECT (sgie1), "config-file-path", SGIE1_CONFIG_FILE, NULL); g_object_set (G_OBJECT (sgie2), "config-file-path", SGIE2_CONFIG_FILE, NULL); /* Set necessary properties of the tracker element. */ if (!set_tracker_properties(nvtracker)) { g_printerr ("Failed to set tracker properties. Exiting.\n"); return -1; } g_object_set (G_OBJECT (nvosd), "display-bbox", 1, NULL); g_object_set (G_OBJECT (sink), "location", argv[2], NULL); } if (yaml_config) { RETURN_ON_PARSER_ERROR(nvds_parse_file_source(source, argv[1], "source")); RETURN_ON_PARSER_ERROR(nvds_parse_streammux(streammux, argv[1], "streammux")); RETURN_ON_PARSER_ERROR(nvds_parse_gie(pgie, argv[1], "primary-gie")); RETURN_ON_PARSER_ERROR(nvds_parse_gie(sgie1, argv[1], "secondary-gie1")); RETURN_ON_PARSER_ERROR(nvds_parse_gie(sgie2, argv[1], "secondary-gie2")); RETURN_ON_PARSER_ERROR(nvds_parse_tracker(nvtracker, argv[1], "tracker")); RETURN_ON_PARSER_ERROR(nvds_parse_osd(nvosd, argv[1], "nvdsosd")); RETURN_ON_PARSER_ERROR(nvds_parse_file_sink(sink, argv[1], "filesink")); } /* we add a message handler */ bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); bus_watch_id = gst_bus_add_watch (bus, bus_call, loop); gst_object_unref (bus); /* Set up the pipeline */ /* we add all elements into the pipeline */ /* decoder | pgie1 | nvtracker | sgie1 | sgie2 | etc.. */ gst_bin_add_many (GST_BIN (pipeline), source, h264parser, decoder, streammux, pgie, nvtracker, sgie1, sgie2, encoder, h264parser1, mp4mux, nvosd, sink, NULL); GstPad *sinkpad, *srcpad; gchar pad_name_sink[16] = "sink_0"; gchar pad_name_src[16] = "src"; sinkpad = gst_element_request_pad_simple (streammux, pad_name_sink); if (!sinkpad) { g_printerr ("Streammux request sink pad failed. Exiting.\n"); return -1; } srcpad = gst_element_get_static_pad (decoder, pad_name_src); if (!srcpad) { g_printerr ("Decoder request src pad failed. Exiting.\n"); return -1; } if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) { g_printerr ("Failed to link decoder to stream muxer. Exiting.\n"); return -1; } gst_object_unref (sinkpad); gst_object_unref (srcpad); /* Link the elements together */ if (!gst_element_link_many (source, h264parser, decoder, NULL)) { g_printerr ("Elements could not be linked: 1. Exiting.\n"); return -1; } if (!gst_element_link_many (streammux, pgie, nvtracker, sgie1, sgie2, nvosd, encoder, h264parser1, mp4mux, sink, NULL)) { g_printerr ("Elements could not be linked. Exiting.\n"); return -1; } /* Lets add probe to get informed of the meta data generated, we add probe to * the sink pad of the osd element, since by that time, the buffer would have * had got all the metadata. */ osd_sink_pad = gst_element_get_static_pad (nvosd, "sink"); if (!osd_sink_pad) g_print ("Unable to get sink pad\n"); else gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER, osd_sink_pad_buffer_probe, NULL, NULL); gst_object_unref (osd_sink_pad); /* Set the pipeline to "playing" state */ g_print ("Using file: %s\n", argv[1]); gst_element_set_state (pipeline, GST_STATE_PLAYING); /* Iterate */ g_print ("Running...\n"); g_main_loop_run (loop); /* Out of the main loop, clean up nicely */ g_print ("Returned, stopping playback\n"); gst_element_set_state (pipeline, GST_STATE_NULL); g_print ("Deleting pipeline\n"); gst_object_unref (GST_OBJECT (pipeline)); g_source_remove (bus_watch_id); g_main_loop_unref (loop); return 0; }