x-linux-ai: recipes-samples: fix USB camera support on X-LINUX-AI v6.2.0
Restore the V4L2SRC based pipeline for USB webcams that was dropped in newer X-LINUX-AI releases, fixing USB camera support on the v6.2.0 baseline. https://onedigi.atlassian.net/browse/DEL-10040 Signed-off-by: Arturo Buzarra <arturo.buzarra@digi.com>
This commit is contained in:
Arturo Buzarra
parent
599fa04a01
commit
cb70c15394
|
|
@ -0,0 +1,233 @@
|
||||||
|
From: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
|
Date: Tue, 12 May 2026 11:23:56 +0200
|
||||||
|
Subject: [PATCH] pose-estimation: restore V4L2SRC pipeline for USB Web cameras
|
||||||
|
|
||||||
|
Backport the previous V4L2SRC-based capture path so MP2x platforms can use USB
|
||||||
|
webcams again.
|
||||||
|
|
||||||
|
https://onedigi.atlassian.net/browse/DEL-10040
|
||||||
|
|
||||||
|
Signed-off-by: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
|
---
|
||||||
|
stai-mpu/stai_mpu_pose_estimation.py | 157 ++++++++++++++++++++++++++-
|
||||||
|
1 file changed, 154 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/stai-mpu/stai_mpu_pose_estimation.py b/stai-mpu/stai_mpu_pose_estimation.py
|
||||||
|
index fbbcfa7..94dde53 100644
|
||||||
|
--- a/stai-mpu/stai_mpu_pose_estimation.py
|
||||||
|
+++ b/stai-mpu/stai_mpu_pose_estimation.py
|
||||||
|
@@ -57,12 +57,125 @@ class GstWidget(Gtk.Box):
|
||||||
|
|
||||||
|
def _on_realize(self, widget):
|
||||||
|
if(args.camera_src == "LIBCAMERA"):
|
||||||
|
self.camera_dual_pipeline_creation()
|
||||||
|
self.pipeline_preview.set_state(Gst.State.PLAYING)
|
||||||
|
+ elif(args.camera_src == "V4L2SRC"):
|
||||||
|
+ self.camera_pipeline_creation()
|
||||||
|
else:
|
||||||
|
- print("Camera source used not supported use LIBCAMERA \n")
|
||||||
|
+ print("Camera source used not supported use LIBCAMERA or V4L2SRC\n")
|
||||||
|
+
|
||||||
|
+ def camera_pipeline_creation(self):
|
||||||
|
+ """
|
||||||
|
+ Creation of the gstreamer pipeline when gstwidget is created dedicated to handle
|
||||||
|
+ camera stream and NN inference (mode single pipeline)
|
||||||
|
+ """
|
||||||
|
+ # gstreamer pipeline creation
|
||||||
|
+ self.pipeline_preview = Gst.Pipeline()
|
||||||
|
+
|
||||||
|
+ # creation of the source v4l2src
|
||||||
|
+ self.v4lsrc1 = Gst.ElementFactory.make("v4l2src", "source")
|
||||||
|
+ video_device = "/dev/" + str(self.app.video_device_prev)
|
||||||
|
+ self.v4lsrc1.set_property("device", video_device)
|
||||||
|
+
|
||||||
|
+ #creation of the v4l2src caps
|
||||||
|
+ caps = "video/x-raw,width=" + str(self.app.frame_width) + ",height=" + str(self.app.frame_height) + ", framerate=" + str(args.framerate)+ "/1"
|
||||||
|
+ print("Camera pipeline configuration : ",caps)
|
||||||
|
+ camera1caps = Gst.Caps.from_string(caps)
|
||||||
|
+ self.camerafilter1 = Gst.ElementFactory.make("capsfilter", "filter1")
|
||||||
|
+ self.camerafilter1.set_property("caps", camera1caps)
|
||||||
|
+
|
||||||
|
+ # creation of capsfilter to force RGB16 on preview branch
|
||||||
|
+ self.rgb16_filter = Gst.ElementFactory.make("capsfilter", "rgb16-filter")
|
||||||
|
+ rgb16_caps = Gst.Caps.from_string("video/x-raw,format=RGB16")
|
||||||
|
+ self.rgb16_filter.set_property("caps", rgb16_caps)
|
||||||
|
+
|
||||||
|
+ # creation of the videoconvert elements
|
||||||
|
+ self.videoformatconverter1 = Gst.ElementFactory.make("videoconvert", "video_convert1")
|
||||||
|
+ self.videoformatconverter2 = Gst.ElementFactory.make("videoconvert", "video_convert2")
|
||||||
|
+
|
||||||
|
+ self.tee = Gst.ElementFactory.make("tee", "tee")
|
||||||
|
+
|
||||||
|
+ # creation and configuration of the queue elements
|
||||||
|
+ self.queue1 = Gst.ElementFactory.make("queue", "queue-1")
|
||||||
|
+ self.queue2 = Gst.ElementFactory.make("queue", "queue-2")
|
||||||
|
+ self.queue1.set_property("max-size-buffers", 1)
|
||||||
|
+ self.queue1.set_property("leaky", 2)
|
||||||
|
+ self.queue2.set_property("max-size-buffers", 1)
|
||||||
|
+ self.queue2.set_property("leaky", 2)
|
||||||
|
+
|
||||||
|
+ # creation and configuration of the appsink element
|
||||||
|
+ self.appsink = Gst.ElementFactory.make("appsink", "appsink")
|
||||||
|
+ nn_caps = "video/x-raw,width=" + str(self.app.nn_input_width) + ",height=" + str(self.app.nn_input_height) + ",format=RGB, framerate=" + str(args.framerate)+ "/1"
|
||||||
|
+ print("Aux pipe configuration: ", nn_caps)
|
||||||
|
+ nncaps = Gst.Caps.from_string(nn_caps)
|
||||||
|
+ self.appsink.set_property("caps", nncaps)
|
||||||
|
+ self.appsink.set_property("emit-signals", True)
|
||||||
|
+ self.appsink.set_property("sync", False)
|
||||||
|
+ self.appsink.set_property("max-buffers", 1)
|
||||||
|
+ self.appsink.set_property("drop", True)
|
||||||
|
+ self.appsink.connect("new-sample", self.new_sample)
|
||||||
|
+
|
||||||
|
+ # creation of the gtkwaylandsink element to handle the gestreamer video stream
|
||||||
|
+ self.gtkwaylandsink = Gst.ElementFactory.make("gtkwaylandsink")
|
||||||
|
+ self.pack_start(self.gtkwaylandsink.props.widget, True, True, 0)
|
||||||
|
+ self.gtkwaylandsink.props.widget.show()
|
||||||
|
+
|
||||||
|
+ # creation and configuration of the fpsdisplaysink element to measure display fps
|
||||||
|
+ self.fps_disp_sink = Gst.ElementFactory.make("fpsdisplaysink", "fpsmeasure1")
|
||||||
|
+ self.fps_disp_sink.set_property("signal-fps-measurements", True)
|
||||||
|
+ self.fps_disp_sink.set_property("fps-update-interval", 2000)
|
||||||
|
+ self.fps_disp_sink.set_property("text-overlay", False)
|
||||||
|
+ self.fps_disp_sink.set_property("video-sink", self.gtkwaylandsink)
|
||||||
|
+ self.fps_disp_sink.connect("fps-measurements",self.get_fps_display)
|
||||||
|
+
|
||||||
|
+ # creation of the video rate and video scale elements
|
||||||
|
+ self.video_rate = Gst.ElementFactory.make("videorate", "video-rate")
|
||||||
|
+ self.video_scale = Gst.ElementFactory.make("videoscale", "video-scale")
|
||||||
|
+
|
||||||
|
+ # Add all elements to the pipeline
|
||||||
|
+ self.pipeline_preview.add(self.v4lsrc1)
|
||||||
|
+ self.pipeline_preview.add(self.camerafilter1)
|
||||||
|
+ self.pipeline_preview.add(self.videoformatconverter1)
|
||||||
|
+ self.pipeline_preview.add(self.videoformatconverter2)
|
||||||
|
+ self.pipeline_preview.add(self.rgb16_filter)
|
||||||
|
+ self.pipeline_preview.add(self.tee)
|
||||||
|
+ self.pipeline_preview.add(self.queue1)
|
||||||
|
+ self.pipeline_preview.add(self.queue2)
|
||||||
|
+ self.pipeline_preview.add(self.appsink)
|
||||||
|
+ self.pipeline_preview.add(self.fps_disp_sink)
|
||||||
|
+ self.pipeline_preview.add(self.video_rate)
|
||||||
|
+ self.pipeline_preview.add(self.video_scale)
|
||||||
|
+
|
||||||
|
+ # linking elements together
|
||||||
|
+ # -> queue 1 -> videoconvert -> fpsdisplaysink
|
||||||
|
+ # v4l2src -> video rate -> tee
|
||||||
|
+ # -> queue 2 -> videoconvert -> video scale -> appsink
|
||||||
|
+ self.v4lsrc1.link(self.video_rate)
|
||||||
|
+ self.video_rate.link(self.camerafilter1)
|
||||||
|
+ self.camerafilter1.link(self.tee)
|
||||||
|
+ self.queue1.link(self.videoformatconverter1)
|
||||||
|
+ self.videoformatconverter1.link(self.rgb16_filter)
|
||||||
|
+ self.rgb16_filter.link(self.fps_disp_sink)
|
||||||
|
+ self.queue2.link(self.videoformatconverter2)
|
||||||
|
+ self.videoformatconverter2.link(self.video_scale)
|
||||||
|
+ self.video_scale.link(self.appsink)
|
||||||
|
+ self.tee.link(self.queue1)
|
||||||
|
+ self.tee.link(self.queue2)
|
||||||
|
+
|
||||||
|
+ # set pipeline playing mode
|
||||||
|
+ self.pipeline_preview.set_state(Gst.State.PLAYING)
|
||||||
|
+ # getting pipeline bus
|
||||||
|
+ self.bus_preview = self.pipeline_preview.get_bus()
|
||||||
|
+ self.bus_preview.add_signal_watch()
|
||||||
|
+ self.bus_preview.connect('message::error', self.msg_error_cb)
|
||||||
|
+ self.bus_preview.connect('message::eos', self.msg_eos_cb)
|
||||||
|
+ self.bus_preview.connect('message::info', self.msg_info_cb)
|
||||||
|
+ self.bus_preview.connect('message::application', self.msg_application_cb)
|
||||||
|
+ self.bus_preview.connect('message::state-changed', self.msg_state_changed_cb)
|
||||||
|
+
|
||||||
|
+ return True
|
||||||
|
|
||||||
|
def camera_dual_pipeline_creation(self):
|
||||||
|
"""
|
||||||
|
creation of the gstreamer pipeline when gstwidget is created dedicated to camera stream
|
||||||
|
(in dual camera pipeline mode)
|
||||||
|
@@ -270,11 +383,14 @@ class GstWidget(Gtk.Box):
|
||||||
|
self.app.nn_inference_time = stop_time - start_time
|
||||||
|
self.app.nn_inference_fps = (1000/(self.app.nn_inference_time*1000))
|
||||||
|
self.app.keypoint_loc, self.app.keypoint_edges, self.app.edge_colors = self.app.nn.get_results()
|
||||||
|
struc = Gst.Structure.new_empty("inference-done")
|
||||||
|
msg = Gst.Message.new_application(None, struc)
|
||||||
|
- self.bus_pipeline.post(msg)
|
||||||
|
+ if (args.camera_src == "LIBCAMERA"):
|
||||||
|
+ self.bus_pipeline.post(msg)
|
||||||
|
+ elif(args.camera_src == "V4L2SRC"):
|
||||||
|
+ self.bus_preview.post(msg)
|
||||||
|
return Gst.FlowReturn.OK
|
||||||
|
|
||||||
|
def get_fps_display(self,fpsdisplaysink,fps,droprate,avgfps):
|
||||||
|
"""
|
||||||
|
measure and recover display fps
|
||||||
|
@@ -733,10 +849,12 @@ class Application:
|
||||||
|
check_camera_cmd = RESOURCES_DIRECTORY + "check_camera_preview.sh"
|
||||||
|
check_camera = subprocess.run(check_camera_cmd)
|
||||||
|
if check_camera.returncode==1:
|
||||||
|
print("no camera connected")
|
||||||
|
exit(1)
|
||||||
|
+ if (args.camera_src == "V4L2SRC"):
|
||||||
|
+ self.video_device_prev,self.camera_caps_prev,self.dcmipp_sensor, self.main_postproc = self.setup_camera()
|
||||||
|
else:
|
||||||
|
print("still picture mode activate")
|
||||||
|
self.enable_camera_preview = False
|
||||||
|
self.still_picture_next = False
|
||||||
|
# initialize the list of the file to be processed (used with the
|
||||||
|
@@ -785,10 +903,43 @@ class Application:
|
||||||
|
print("display resolution is : ",display_width, " x ", display_height)
|
||||||
|
self.window_width = int(display_width)
|
||||||
|
self.window_height = int(display_height)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
+
|
||||||
|
+ def setup_camera(self):
|
||||||
|
+ """
|
||||||
|
+ Used to configure the camera based on resolution passed as application arguments
|
||||||
|
+ """
|
||||||
|
+ width = str(args.frame_width)
|
||||||
|
+ height = str(args.frame_height)
|
||||||
|
+ framerate = str(args.framerate)
|
||||||
|
+ device = str(args.video_device)
|
||||||
|
+ nn_input_width = str(self.nn_input_width)
|
||||||
|
+ nn_input_height = str(self.nn_input_height)
|
||||||
|
+ config_camera = RESOURCES_DIRECTORY + "setup_camera.sh " + width + " " + height + " " + framerate + " " + device
|
||||||
|
+ x = subprocess.check_output(config_camera,shell=True)
|
||||||
|
+ x = x.decode("utf-8")
|
||||||
|
+ x = x.split("\n")
|
||||||
|
+ for i in x :
|
||||||
|
+ if "V4L_DEVICE_PREV" in i:
|
||||||
|
+ video_device_prev = i.lstrip('V4L_DEVICE_PREV=')
|
||||||
|
+ if "V4L2_CAPS_PREV" in i:
|
||||||
|
+ camera_caps_prev = i.lstrip('V4L2_CAPS_PREV=')
|
||||||
|
+ if "V4L_DEVICE_NN" in i:
|
||||||
|
+ video_device_nn = i.lstrip('V4L_DEVICE_NN=')
|
||||||
|
+ if "V4L2_CAPS_NN" in i:
|
||||||
|
+ camera_caps_nn = i.lstrip('V4L2_CAPS_NN=')
|
||||||
|
+ if "DCMIPP_SENSOR" in i:
|
||||||
|
+ dcmipp_sensor = i.lstrip('DCMIPP_SENSOR=')
|
||||||
|
+ if "MAIN_POSTPROC" in i:
|
||||||
|
+ main_postproc = i.lstrip('MAIN_POSTPROC=')
|
||||||
|
+ if "AUX_POSTPROC" in i:
|
||||||
|
+ aux_postproc = i.lstrip('AUX_POSTPROC=')
|
||||||
|
+ return video_device_prev, camera_caps_prev, dcmipp_sensor, main_postproc
|
||||||
|
+
|
||||||
|
+
|
||||||
|
def valid_timeout_callback(self):
|
||||||
|
"""
|
||||||
|
if timeout occurs that means that camera preview and the gtk is not
|
||||||
|
behaving as expected
|
||||||
|
"""
|
||||||
|
@@ -1059,11 +1210,11 @@ if __name__ == '__main__':
|
||||||
|
parser.add_argument("--input_mean", default=255, help="input mean")
|
||||||
|
parser.add_argument("--input_std", default=0, help="input standard deviation")
|
||||||
|
parser.add_argument("--normalize", default=False, help="input standard deviation")
|
||||||
|
parser.add_argument("--validation", action='store_true', help="enable the validation mode")
|
||||||
|
parser.add_argument("--val_run", default=50, help="set the number of draws in the validation mode")
|
||||||
|
- parser.add_argument("--camera_src", default="LIBCAMERA", help="use V4L2SRC for MP1x and LIBCAMERA for MP2x")
|
||||||
|
+ parser.add_argument("--camera_src", default="LIBCAMERA", help="use V4L2SRC for MP1x or USB camera and LIBCAMERA for MP2x")
|
||||||
|
parser.add_argument("--debug", default=False, action='store_true', help=argparse.SUPPRESS)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
application = Application(args)
|
||||||
|
--
|
||||||
|
2.34.1
|
||||||
|
|
@ -8,6 +8,7 @@ SRC_URI += " \
|
||||||
file://patches/0002-pose-estimation-reduce-font-size-for-big-screens.patch \
|
file://patches/0002-pose-estimation-reduce-font-size-for-big-screens.patch \
|
||||||
file://patches/0003-pose-estimation-set-camera-preview-to-640x480.patch \
|
file://patches/0003-pose-estimation-set-camera-preview-to-640x480.patch \
|
||||||
${@bb.utils.contains("BBFILE_COLLECTIONS", "x-linux-isp", "file://patches/0004-pose-estimation-fix-initialization-issue-with-x-linu.patch", "", d)} \
|
${@bb.utils.contains("BBFILE_COLLECTIONS", "x-linux-isp", "file://patches/0004-pose-estimation-fix-initialization-issue-with-x-linu.patch", "", d)} \
|
||||||
|
file://patches/0005-pose-estimation-restore-V4L2SRC-pipeline-for-USB-Web.patch \
|
||||||
"
|
"
|
||||||
|
|
||||||
do_install:append () {
|
do_install:append () {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
# Copyright (C) 2026, Digi International Inc.
|
||||||
|
|
||||||
|
FILESEXTRAPATHS:prepend := "${THISDIR}/files:"
|
||||||
|
|
||||||
|
SRC_URI += " \
|
||||||
|
file://patches/0001-setup_camera-fix-support-for-web-camera-with-STM32MP.patch \
|
||||||
|
"
|
||||||
|
|
@ -6,13 +6,15 @@ Subject: [PATCH] config_board: fix support for web camera with STM32MP255
|
||||||
This commit disables the dual camera pipeline support for USB cameras, as it is
|
This commit disables the dual camera pipeline support for USB cameras, as it is
|
||||||
exclusive to cameras using the DCMIPP peripheral.
|
exclusive to cameras using the DCMIPP peripheral.
|
||||||
|
|
||||||
|
https://onedigi.atlassian.net/browse/DEL-10040
|
||||||
|
|
||||||
Signed-off-by: Arturo Buzarra <arturo.buzarra@digi.com>
|
Signed-off-by: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
---
|
---
|
||||||
.../files/resources-files/config_board_npu.sh | 31 ++++++++++++++++++-
|
resources-files/config_board_npu.sh | 32 ++++++++++++++++++++++++++++-
|
||||||
1 file changed, 30 insertions(+), 1 deletion(-)
|
1 file changed, 31 insertions(+), 1 deletion(-)
|
||||||
|
|
||||||
diff --git a/resources-files/config_board_npu.sh b/resources-files/config_board_npu.sh
|
diff --git a/resources-files/config_board_npu.sh b/resources-files/config_board_npu.sh
|
||||||
index c89b5ca..af95558 100644
|
index dce3e68..65717f5 100644
|
||||||
--- a/resources-files/config_board_npu.sh
|
--- a/resources-files/config_board_npu.sh
|
||||||
+++ b/resources-files/config_board_npu.sh
|
+++ b/resources-files/config_board_npu.sh
|
||||||
@@ -16,10 +16,28 @@ STM32MP251="stm32mp251"
|
@@ -16,10 +16,28 @@ STM32MP251="stm32mp251"
|
||||||
|
|
@ -44,7 +46,7 @@ index c89b5ca..af95558 100644
|
||||||
echo "Software X-LINUX-AI installed is not compatible with the board, please install X-LINUX-AI CPU version for plateform without hardware accelerator"
|
echo "Software X-LINUX-AI installed is not compatible with the board, please install X-LINUX-AI CPU version for plateform without hardware accelerator"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
@@ -54,11 +72,22 @@ if [[ "$COMPATIBLE" == *"$STM32MP257"* ]] || [[ "$COMPATIBLE" == *"$STM32MP255"*
|
@@ -56,11 +74,23 @@ if [[ "$COMPATIBLE" == *"$STM32MP257"* ]] || [[ "$COMPATIBLE" == *"$STM32MP255"*
|
||||||
MACHINE=$STM32MP2_NPU
|
MACHINE=$STM32MP2_NPU
|
||||||
DWIDTH=760
|
DWIDTH=760
|
||||||
DHEIGHT=568
|
DHEIGHT=568
|
||||||
|
|
@ -58,6 +60,7 @@ index c89b5ca..af95558 100644
|
||||||
+ OPTIONS="--dual_camera_pipeline"
|
+ OPTIONS="--dual_camera_pipeline"
|
||||||
+ else
|
+ else
|
||||||
+ # Web camera
|
+ # Web camera
|
||||||
|
+ CAMERA_SRC="V4L2SRC"
|
||||||
+ OPTIONS=""
|
+ OPTIONS=""
|
||||||
+ DWIDTH=640
|
+ DWIDTH=640
|
||||||
+ DHEIGHT=480
|
+ DHEIGHT=480
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
From: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
|
Date: Tue, 12 May 2026 10:07:34 +0200
|
||||||
|
Subject: [PATCH] setup_camera: fix support for web camera with STM32MP255
|
||||||
|
processor
|
||||||
|
|
||||||
|
This commit fixes the function to obtain the video node for USB cameras.
|
||||||
|
|
||||||
|
https://onedigi.atlassian.net/browse/DEL-10040
|
||||||
|
|
||||||
|
Signed-off-by: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
|
---
|
||||||
|
resources-files/setup_camera.sh | 11 ++++++-----
|
||||||
|
1 file changed, 6 insertions(+), 5 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/resources-files/setup_camera.sh b/resources-files/setup_camera.sh
|
||||||
|
index 9d460a1..6f039e4 100644
|
||||||
|
--- a/resources-files/setup_camera.sh
|
||||||
|
+++ b/resources-files/setup_camera.sh
|
||||||
|
@@ -47,15 +47,16 @@ is_dcmipp_present() {
|
||||||
|
get_webcam_device() {
|
||||||
|
echo "get_webcam_device function"
|
||||||
|
found="NOTFOUND"
|
||||||
|
for video in $(find /sys/class/video4linux -name "video*" -type l | sort);
|
||||||
|
do
|
||||||
|
- if [ "$(cat $video/name)" = "dcmipp_dump_capture" ]; then
|
||||||
|
- found="FOUND"
|
||||||
|
- else
|
||||||
|
- V4L_DEVICE="$(basename $video)"
|
||||||
|
- break;
|
||||||
|
+ if ! $(cat "$video/name" | grep -q "dcmi") ; then
|
||||||
|
+ if ! $(cat "$video/name" | grep -q "stm32mp") ; then
|
||||||
|
+ V4L_DEVICE="$(basename "$video")"
|
||||||
|
+ found="FOUND"
|
||||||
|
+ break;
|
||||||
|
+ fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
--
|
||||||
|
2.34.1
|
||||||
|
|
||||||
|
|
@ -0,0 +1,250 @@
|
||||||
|
From: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
|
Date: Tue, 12 May 2026 13:25:04 +0200
|
||||||
|
Subject: [PATCH] semantic-segmentation: restore V4L2SRC pipeline for USB Web
|
||||||
|
cameras
|
||||||
|
|
||||||
|
Backport the previous V4L2SRC-based capture path so MP2x platforms can use USB
|
||||||
|
webcams again.
|
||||||
|
|
||||||
|
https://onedigi.atlassian.net/browse/DEL-10040
|
||||||
|
|
||||||
|
Signed-off-by: Arturo Buzarra <arturo.buzarra@digi.com>
|
||||||
|
---
|
||||||
|
stai-mpu/stai_mpu_semantic_segmentation.py | 167 ++++++++++++++++++++-
|
||||||
|
1 file changed, 164 insertions(+), 3 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/stai-mpu/stai_mpu_semantic_segmentation.py b/stai-mpu/stai_mpu_semantic_segmentation.py
|
||||||
|
index a4c65c2..e3601ba 100644
|
||||||
|
--- a/stai-mpu/stai_mpu_semantic_segmentation.py
|
||||||
|
+++ b/stai-mpu/stai_mpu_semantic_segmentation.py
|
||||||
|
@@ -62,12 +62,125 @@ class GstWidget(Gtk.Box):
|
||||||
|
|
||||||
|
def _on_realize(self, widget):
|
||||||
|
if(args.camera_src == "LIBCAMERA"):
|
||||||
|
self.camera_dual_pipeline_creation()
|
||||||
|
self.pipeline_preview.set_state(Gst.State.PLAYING)
|
||||||
|
+ elif(args.camera_src == "V4L2SRC"):
|
||||||
|
+ self.camera_pipeline_creation()
|
||||||
|
else:
|
||||||
|
- print("Camera source used not supported use LIBCAMERA \n")
|
||||||
|
+ print("Camera source used not supported use LIBCAMERA or V4L2SRC\n")
|
||||||
|
+
|
||||||
|
+ def camera_pipeline_creation(self):
|
||||||
|
+ """
|
||||||
|
+ Creation of the gstreamer pipeline when gstwidget is created dedicated to handle
|
||||||
|
+ camera stream and NN inference (mode single pipeline)
|
||||||
|
+ """
|
||||||
|
+ # gstreamer pipeline creation
|
||||||
|
+ self.pipeline_preview = Gst.Pipeline()
|
||||||
|
+
|
||||||
|
+ # creation of the source v4l2src
|
||||||
|
+ self.v4lsrc1 = Gst.ElementFactory.make("v4l2src", "source")
|
||||||
|
+ video_device = "/dev/" + str(self.app.video_device_prev)
|
||||||
|
+ self.v4lsrc1.set_property("device", video_device)
|
||||||
|
+
|
||||||
|
+ #creation of the v4l2src caps
|
||||||
|
+ caps = "video/x-raw,width=" + str(self.app.frame_width) + ",height=" + str(self.app.frame_height) + ", framerate=" + str(args.framerate)+ "/1"
|
||||||
|
+ print("Camera pipeline configuration : ",caps)
|
||||||
|
+ camera1caps = Gst.Caps.from_string(caps)
|
||||||
|
+ self.camerafilter1 = Gst.ElementFactory.make("capsfilter", "filter1")
|
||||||
|
+ self.camerafilter1.set_property("caps", camera1caps)
|
||||||
|
+
|
||||||
|
+ # creation of capsfilter to force RGB16 on preview branch
|
||||||
|
+ self.rgb16_filter = Gst.ElementFactory.make("capsfilter", "rgb16-filter")
|
||||||
|
+ rgb16_caps = Gst.Caps.from_string("video/x-raw,format=RGB16")
|
||||||
|
+ self.rgb16_filter.set_property("caps", rgb16_caps)
|
||||||
|
+
|
||||||
|
+ # creation of the videoconvert elements
|
||||||
|
+ self.videoformatconverter1 = Gst.ElementFactory.make("videoconvert", "video_convert1")
|
||||||
|
+ self.videoformatconverter2 = Gst.ElementFactory.make("videoconvert", "video_convert2")
|
||||||
|
+
|
||||||
|
+ self.tee = Gst.ElementFactory.make("tee", "tee")
|
||||||
|
+
|
||||||
|
+ # creation and configuration of the queue elements
|
||||||
|
+ self.queue1 = Gst.ElementFactory.make("queue", "queue-1")
|
||||||
|
+ self.queue2 = Gst.ElementFactory.make("queue", "queue-2")
|
||||||
|
+ self.queue1.set_property("max-size-buffers", 1)
|
||||||
|
+ self.queue1.set_property("leaky", 2)
|
||||||
|
+ self.queue2.set_property("max-size-buffers", 1)
|
||||||
|
+ self.queue2.set_property("leaky", 2)
|
||||||
|
+
|
||||||
|
+ # creation and configuration of the appsink element
|
||||||
|
+ self.appsink = Gst.ElementFactory.make("appsink", "appsink")
|
||||||
|
+ nn_caps = "video/x-raw,width=" + str(self.app.nn_input_width) + ",height=" + str(self.app.nn_input_height) + ",format=RGB, framerate=" + str(args.framerate)+ "/1"
|
||||||
|
+ print("Aux pipe configuration: ", nn_caps)
|
||||||
|
+ nncaps = Gst.Caps.from_string(nn_caps)
|
||||||
|
+ self.appsink.set_property("caps", nncaps)
|
||||||
|
+ self.appsink.set_property("emit-signals", True)
|
||||||
|
+ self.appsink.set_property("sync", False)
|
||||||
|
+ self.appsink.set_property("max-buffers", 1)
|
||||||
|
+ self.appsink.set_property("drop", True)
|
||||||
|
+ self.appsink.connect("new-sample", self.new_sample)
|
||||||
|
+
|
||||||
|
+ # creation of the gtkwaylandsink element to handle the gestreamer video stream
|
||||||
|
+ self.gtkwaylandsink = Gst.ElementFactory.make("gtkwaylandsink")
|
||||||
|
+ self.pack_start(self.gtkwaylandsink.props.widget, True, True, 0)
|
||||||
|
+ self.gtkwaylandsink.props.widget.show()
|
||||||
|
+
|
||||||
|
+ # creation and configuration of the fpsdisplaysink element to measure display fps
|
||||||
|
+ self.fps_disp_sink = Gst.ElementFactory.make("fpsdisplaysink", "fpsmeasure1")
|
||||||
|
+ self.fps_disp_sink.set_property("signal-fps-measurements", True)
|
||||||
|
+ self.fps_disp_sink.set_property("fps-update-interval", 2000)
|
||||||
|
+ self.fps_disp_sink.set_property("text-overlay", False)
|
||||||
|
+ self.fps_disp_sink.set_property("video-sink", self.gtkwaylandsink)
|
||||||
|
+ self.fps_disp_sink.connect("fps-measurements",self.get_fps_display)
|
||||||
|
+
|
||||||
|
+ # creation of the video rate and video scale elements
|
||||||
|
+ self.video_rate = Gst.ElementFactory.make("videorate", "video-rate")
|
||||||
|
+ self.video_scale = Gst.ElementFactory.make("videoscale", "video-scale")
|
||||||
|
+
|
||||||
|
+ # Add all elements to the pipeline
|
||||||
|
+ self.pipeline_preview.add(self.v4lsrc1)
|
||||||
|
+ self.pipeline_preview.add(self.camerafilter1)
|
||||||
|
+ self.pipeline_preview.add(self.videoformatconverter1)
|
||||||
|
+ self.pipeline_preview.add(self.videoformatconverter2)
|
||||||
|
+ self.pipeline_preview.add(self.rgb16_filter)
|
||||||
|
+ self.pipeline_preview.add(self.tee)
|
||||||
|
+ self.pipeline_preview.add(self.queue1)
|
||||||
|
+ self.pipeline_preview.add(self.queue2)
|
||||||
|
+ self.pipeline_preview.add(self.appsink)
|
||||||
|
+ self.pipeline_preview.add(self.fps_disp_sink)
|
||||||
|
+ self.pipeline_preview.add(self.video_rate)
|
||||||
|
+ self.pipeline_preview.add(self.video_scale)
|
||||||
|
+
|
||||||
|
+ # linking elements together
|
||||||
|
+ # -> queue 1 -> videoconvert -> fpsdisplaysink
|
||||||
|
+ # v4l2src -> video rate -> tee
|
||||||
|
+ # -> queue 2 -> videoconvert -> video scale -> appsink
|
||||||
|
+ self.v4lsrc1.link(self.video_rate)
|
||||||
|
+ self.video_rate.link(self.camerafilter1)
|
||||||
|
+ self.camerafilter1.link(self.tee)
|
||||||
|
+ self.queue1.link(self.videoformatconverter1)
|
||||||
|
+ self.videoformatconverter1.link(self.rgb16_filter)
|
||||||
|
+ self.rgb16_filter.link(self.fps_disp_sink)
|
||||||
|
+ self.queue2.link(self.videoformatconverter2)
|
||||||
|
+ self.videoformatconverter2.link(self.video_scale)
|
||||||
|
+ self.video_scale.link(self.appsink)
|
||||||
|
+ self.tee.link(self.queue1)
|
||||||
|
+ self.tee.link(self.queue2)
|
||||||
|
+
|
||||||
|
+ # set pipeline playing mode
|
||||||
|
+ self.pipeline_preview.set_state(Gst.State.PLAYING)
|
||||||
|
+ # getting pipeline bus
|
||||||
|
+ self.bus_preview = self.pipeline_preview.get_bus()
|
||||||
|
+ self.bus_preview.add_signal_watch()
|
||||||
|
+ self.bus_preview.connect('message::error', self.msg_error_cb)
|
||||||
|
+ self.bus_preview.connect('message::eos', self.msg_eos_cb)
|
||||||
|
+ self.bus_preview.connect('message::info', self.msg_info_cb)
|
||||||
|
+ self.bus_preview.connect('message::application', self.msg_application_cb)
|
||||||
|
+ self.bus_preview.connect('message::state-changed', self.msg_state_changed_cb)
|
||||||
|
+
|
||||||
|
+ return True
|
||||||
|
|
||||||
|
def camera_dual_pipeline_creation(self):
|
||||||
|
"""
|
||||||
|
creation of the gstreamer pipeline when gstwidget is created dedicated to camera stream
|
||||||
|
(in dual camera pipeline mode)
|
||||||
|
@@ -230,17 +343,29 @@ class GstWidget(Gtk.Box):
|
||||||
|
buffer_size = buf.get_size()
|
||||||
|
#determine the shape of the numpy array
|
||||||
|
number_of_column = caps.get_structure(0).get_value('width')
|
||||||
|
number_of_lines = caps.get_structure(0).get_value('height')
|
||||||
|
channels = 3
|
||||||
|
+
|
||||||
|
+ if (args.camera_src == "V4L2SRC"):
|
||||||
|
+ # Convert buffer, compute stride dynamically
|
||||||
|
+ # from the actual buffer size and frame height.
|
||||||
|
+ row_stride = buffer_size // number_of_lines
|
||||||
|
+ arr = np.ndarray(
|
||||||
|
+ (number_of_lines, number_of_column, channels),
|
||||||
|
+ strides=(row_stride, channels, 1),
|
||||||
|
+ buffer=buf.extract_dup(0, buffer_size),
|
||||||
|
+ dtype=np.uint8)
|
||||||
|
+ return arr
|
||||||
|
+
|
||||||
|
buffer = np.frombuffer(buf.extract_dup(0, buf.get_size()), dtype=np.uint8)
|
||||||
|
|
||||||
|
#DCMIPP pixelpacker has a constraint on the output resolution that should be multiple of 16.
|
||||||
|
# the allocated buffer may contains stride to handle the DCMIPP Hw constraints/
|
||||||
|
#The following code allow to handle both cases by anticipating the size of the
|
||||||
|
#allocated buffer according to the NN resolution
|
||||||
|
- if (self.app.nn_input_width % 16 != 0):
|
||||||
|
+ if (self.app.nn_input_width % 16 != 0) and (args.camera_src == "LIBCAMERA"):
|
||||||
|
# Calculate the nearest upper multiple of 16
|
||||||
|
upper_multiple = ((self.app.nn_input_width // 16) + 1) * 16
|
||||||
|
# Calculate the stride and offset
|
||||||
|
stride = upper_multiple * channels
|
||||||
|
offset = stride - (number_of_column * channels)
|
||||||
|
@@ -276,11 +401,14 @@ class GstWidget(Gtk.Box):
|
||||||
|
self.app.nn_inference_time = stop_time - start_time
|
||||||
|
self.app.nn_inference_fps = (1000/(self.app.nn_inference_time*1000))
|
||||||
|
self.app.unique_label, self.app.nn_seg_map = self.app.nn.get_results()
|
||||||
|
struc = Gst.Structure.new_empty("inference-done")
|
||||||
|
msg = Gst.Message.new_application(None, struc)
|
||||||
|
- self.bus_pipeline.post(msg)
|
||||||
|
+ if (args.camera_src == "LIBCAMERA"):
|
||||||
|
+ self.bus_pipeline.post(msg)
|
||||||
|
+ elif(args.camera_src == "V4L2SRC"):
|
||||||
|
+ self.bus_preview.post(msg)
|
||||||
|
return Gst.FlowReturn.OK
|
||||||
|
|
||||||
|
def get_fps_display(self,fpsdisplaysink,fps,droprate,avgfps):
|
||||||
|
"""
|
||||||
|
measure and recover display fps
|
||||||
|
@@ -863,10 +991,12 @@ class Application:
|
||||||
|
check_camera_cmd = RESOURCES_DIRECTORY + "check_camera_preview.sh"
|
||||||
|
check_camera = subprocess.run(check_camera_cmd)
|
||||||
|
if check_camera.returncode==1:
|
||||||
|
print("no camera connected")
|
||||||
|
exit(1)
|
||||||
|
+ if (args.camera_src == "V4L2SRC"):
|
||||||
|
+ self.video_device_prev,self.camera_caps_prev,self.dcmipp_sensor, self.main_postproc = self.setup_camera()
|
||||||
|
else:
|
||||||
|
print("still picture mode activate")
|
||||||
|
self.enable_camera_preview = False
|
||||||
|
self.still_picture_next = False
|
||||||
|
|
||||||
|
@@ -916,10 +1046,41 @@ class Application:
|
||||||
|
print("display resolution is : ",display_width, " x ", display_height)
|
||||||
|
self.window_width = int(display_width)
|
||||||
|
self.window_height = int(display_height)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
+ def setup_camera(self):
|
||||||
|
+ """
|
||||||
|
+ Used to configure the camera based on resolution passed as application arguments
|
||||||
|
+ """
|
||||||
|
+ width = str(args.frame_width)
|
||||||
|
+ height = str(args.frame_height)
|
||||||
|
+ framerate = str(args.framerate)
|
||||||
|
+ device = str(args.video_device)
|
||||||
|
+ nn_input_width = str(self.nn_input_width)
|
||||||
|
+ nn_input_height = str(self.nn_input_height)
|
||||||
|
+ config_camera = RESOURCES_DIRECTORY + "setup_camera.sh " + width + " " + height + " " + framerate + " " + device
|
||||||
|
+ x = subprocess.check_output(config_camera,shell=True)
|
||||||
|
+ x = x.decode("utf-8")
|
||||||
|
+ x = x.split("\n")
|
||||||
|
+ for i in x :
|
||||||
|
+ if "V4L_DEVICE_PREV" in i:
|
||||||
|
+ video_device_prev = i.lstrip('V4L_DEVICE_PREV=')
|
||||||
|
+ if "V4L2_CAPS_PREV" in i:
|
||||||
|
+ camera_caps_prev = i.lstrip('V4L2_CAPS_PREV=')
|
||||||
|
+ if "V4L_DEVICE_NN" in i:
|
||||||
|
+ video_device_nn = i.lstrip('V4L_DEVICE_NN=')
|
||||||
|
+ if "V4L2_CAPS_NN" in i:
|
||||||
|
+ camera_caps_nn = i.lstrip('V4L2_CAPS_NN=')
|
||||||
|
+ if "DCMIPP_SENSOR" in i:
|
||||||
|
+ dcmipp_sensor = i.lstrip('DCMIPP_SENSOR=')
|
||||||
|
+ if "MAIN_POSTPROC" in i:
|
||||||
|
+ main_postproc = i.lstrip('MAIN_POSTPROC=')
|
||||||
|
+ if "AUX_POSTPROC" in i:
|
||||||
|
+ aux_postproc = i.lstrip('AUX_POSTPROC=')
|
||||||
|
+ return video_device_prev, camera_caps_prev, dcmipp_sensor, main_postproc
|
||||||
|
+
|
||||||
|
def valid_timeout_callback(self):
|
||||||
|
"""
|
||||||
|
if timeout occurs that means that camera preview and the gtk is not
|
||||||
|
behaving as expected */
|
||||||
|
"""
|
||||||
|
--
|
||||||
|
2.34.1
|
||||||
|
|
@ -9,6 +9,7 @@ SRC_URI += " \
|
||||||
file://patches/0003-semantic-segmentation-adapt-sample-for-root-user.patch \
|
file://patches/0003-semantic-segmentation-adapt-sample-for-root-user.patch \
|
||||||
file://patches/0004-semantic-segmentation-set-camera-preview-to-640x480.patch \
|
file://patches/0004-semantic-segmentation-set-camera-preview-to-640x480.patch \
|
||||||
${@bb.utils.contains("BBFILE_COLLECTIONS", "x-linux-isp", "file://patches/0005-semantic-segmentation-fix-initialization-issue-with-.patch", "", d)} \
|
${@bb.utils.contains("BBFILE_COLLECTIONS", "x-linux-isp", "file://patches/0005-semantic-segmentation-fix-initialization-issue-with-.patch", "", d)} \
|
||||||
|
file://patches/0006-semantic-segmentation-restore-V4L2SRC-pipeline-for-U.patch \
|
||||||
"
|
"
|
||||||
|
|
||||||
do_install:append () {
|
do_install:append () {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue