Input of tflite-server on voxl2

[William Chung]

I am running yolov5 on tflite-server. The problem is when I change input_pipe to the rtsp-mpa, tflite-server don't show anything. Is there anything I need to process the stream?
The tflite-server works fine with built-in stereo camera (RAW8 input)

The rtsp-mpa code is here. It get rtsp video stream and output a 1280*720 RGB stream.

import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject, GLib
import numpy as np
import time
import cv2
import sys

from pympa import *

print("init gstreamer")
# Initialize GStreamer
Gst.init(None)

app_name    = 'voxl-camera-server'
output_name = 'rtsp-debug'
pympa_create_pub(output_name, app_name)

def pub_image_mpa(img, frame_id):

    #convert from BGR (opencv native) to RGB
    img_out = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) #cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    #img_out = img

    meta              = camera_image_metadata_t()
    meta.magic_number = CAMERA_MAGIC_NUMBER
    meta.timestamp_ns = int(time.time() * 1000000000) #unknown from rtsp stream, so use current timestamp
    meta.frame_id     = frame_id
    meta.width        = img_out.shape[1]
    meta.stride       = meta.width
    meta.height       = img_out.shape[0]
    meta.size_bytes   = int(meta.width * meta.height * 3) #RGB888
    meta.exposure_ns  = 0 #unknown from rtsp stream
    meta.gain         = 0 #unknown from rtsp stream
    meta.format       = 10 # 0:IMAGE_FORMAT_RAW8, 1:IMAGE_FORMAT_NV12, 10:IMAGE_FORMAT_RGB
    meta.framerate    = 0
    
    pympa_publish_image(img_out,meta)

def resize_frame(frame_data, width, height):
    resized_frame = cv2.resize(frame, (width, height))
    return resized_frame

def convertYUVI420toNV12(i420byte, nv12byte, width, height):
    nLenY = width * height
    nLenU = nLenY // 4
                
    for i in range(nLenU):
        nv12byte[nLenY + 2 * i] = i420byte[nLenY + nLenU + i]  # U
        nv12byte[nLenY + 2 * i + 1] = i420byte[nLenY + i]      # V
    return nv12byte

# Define the RTSP stream URL
stream_url = 'rtsp://169.254.4.201:554/live0'

# Create a GStreamer pipeline voxl-inspect-mavlink mavlink_onboard
pipeline = Gst.parse_launch(f"rtspsrc location={stream_url} latency=0 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink")

# Start the pipeline
pipeline.set_state(Gst.State.PLAYING)

# fourcc = cv2.VideoWriter_fourcc(*'MJPG')
# out = cv2.VideoWriter('output.avi', fourcc, 30.0, (1280, 720))

# Main loop to read frames from pipeline and publish them to MPA
frame_cntr = 0
start_t = time.time()
# while time.time() - start_t <= 5:
print("start streaming")
while True:
    # Retrieve a frame from the pipeline
    sample = pipeline.get_by_name('appsink0').emit('pull-sample')
    buffer = sample.get_buffer()
    result, info = buffer.map(Gst.MapFlags.READ)
    if result:
        # Convert the frame to numpy array
        data = np.ndarray((info.size,), dtype=np.uint8, buffer=info.data)
        # frame = np.reshape(data, (720, 640, 3))
        # Increment frame counter

        width = 1280
        height = 720

        y_size = width * height
        uv_size = int(y_size / 2)  

        y_data = data[:y_size]
        u_data = data[y_size:y_size + uv_size]
        v_data = data[y_size + uv_size:]

        yuv_data_reshaped = np.concatenate((y_data, u_data, v_data))

        yuv_data_reshaped = yuv_data_reshaped.reshape((int(height * 1.5), width))
        frame = yuv_data_reshaped
        frame = cv2.cvtColor(yuv_data_reshaped, cv2.COLOR_YUV2BGR_I420)
        #frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)
        #frame = cv2.cvtColor(frame, cv2.COLOR_YUV2YUV_I420)
        # frame = resize_frame(frame, 640, 480)
        
        def BGRtoNV12(frame):
            yuv = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_YV12)
            uv_row_cnt = yuv.shape[0] // 3
            uv_plane = np.transpose(yuv[uv_row_cnt * 2:].reshape(2, -1), [1, 0])
            yuv[uv_row_cnt * 2:] = uv_plane.reshape(uv_row_cnt, -1)
            frame = yuv
            return frame

        #BGRtoNV12(frame)


        frame_cntr += 1
        sys.stdout.write("\r")
        sys.stdout.write(f'got frame {frame_cntr} with dims {frame.shape}')
        sys.stdout.flush()
        
        pub_image_mpa(frame, frame_cntr)
        # out.write(frame)
    else:
        print("Error mapping buffer")
    
    # unmap buffer so that program would occupy the whole memoy and killed
    buffer.unmap(info)

# out.release()

# Stop the pipeline
pipeline.set_state(Gst.State.NULL)