ModalAI Forum
    • Categories
    • Recent
    • Tags
    • Popular
    • Users
    • Groups
    • Register
    • Login
    1. Home
    2. William Chung
    • Profile
    • Following 0
    • Followers 0
    • Topics 1
    • Posts 1
    • Best 0
    • Controversial 0
    • Groups 0

    William Chung

    @William Chung

    0
    Reputation
    1
    Profile views
    1
    Posts
    0
    Followers
    0
    Following
    Joined Last Online

    William Chung Unfollow Follow

    Latest posts made by William Chung

    • Input of tflite-server on voxl2

      I am running yolov5 on tflite-server. The problem is when I change input_pipe to the rtsp-mpa, tflite-server don't show anything. Is there anything I need to process the stream?
      The tflite-server works fine with built-in stereo camera (RAW8 input)

      The rtsp-mpa code is here. It get rtsp video stream and output a 1280*720 RGB stream.

      import gi
      gi.require_version('Gst', '1.0')
      from gi.repository import Gst, GObject, GLib
      import numpy as np
      import time
      import cv2
      import sys
      
      from pympa import *
      
      print("init gstreamer")
      # Initialize GStreamer
      Gst.init(None)
      
      app_name    = 'voxl-camera-server'
      output_name = 'rtsp-debug'
      pympa_create_pub(output_name, app_name)
      
      def pub_image_mpa(img, frame_id):
      
          #convert from BGR (opencv native) to RGB
          img_out = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) #cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
          #img_out = img
      
          meta              = camera_image_metadata_t()
          meta.magic_number = CAMERA_MAGIC_NUMBER
          meta.timestamp_ns = int(time.time() * 1000000000) #unknown from rtsp stream, so use current timestamp
          meta.frame_id     = frame_id
          meta.width        = img_out.shape[1]
          meta.stride       = meta.width
          meta.height       = img_out.shape[0]
          meta.size_bytes   = int(meta.width * meta.height * 3) #RGB888
          meta.exposure_ns  = 0 #unknown from rtsp stream
          meta.gain         = 0 #unknown from rtsp stream
          meta.format       = 10 # 0:IMAGE_FORMAT_RAW8, 1:IMAGE_FORMAT_NV12, 10:IMAGE_FORMAT_RGB
          meta.framerate    = 0
          
          pympa_publish_image(img_out,meta)
      
      def resize_frame(frame_data, width, height):
          resized_frame = cv2.resize(frame, (width, height))
          return resized_frame
      
      def convertYUVI420toNV12(i420byte, nv12byte, width, height):
          nLenY = width * height
          nLenU = nLenY // 4
                      
          for i in range(nLenU):
              nv12byte[nLenY + 2 * i] = i420byte[nLenY + nLenU + i]  # U
              nv12byte[nLenY + 2 * i + 1] = i420byte[nLenY + i]      # V
          return nv12byte
      
      # Define the RTSP stream URL
      stream_url = 'rtsp://169.254.4.201:554/live0'
      
      # Create a GStreamer pipeline voxl-inspect-mavlink mavlink_onboard
      pipeline = Gst.parse_launch(f"rtspsrc location={stream_url} latency=0 ! rtph264depay ! h264parse ! avdec_h264 ! videoconvert ! appsink")
      
      # Start the pipeline
      pipeline.set_state(Gst.State.PLAYING)
      
      # fourcc = cv2.VideoWriter_fourcc(*'MJPG')
      # out = cv2.VideoWriter('output.avi', fourcc, 30.0, (1280, 720))
      
      # Main loop to read frames from pipeline and publish them to MPA
      frame_cntr = 0
      start_t = time.time()
      # while time.time() - start_t <= 5:
      print("start streaming")
      while True:
          # Retrieve a frame from the pipeline
          sample = pipeline.get_by_name('appsink0').emit('pull-sample')
          buffer = sample.get_buffer()
          result, info = buffer.map(Gst.MapFlags.READ)
          if result:
              # Convert the frame to numpy array
              data = np.ndarray((info.size,), dtype=np.uint8, buffer=info.data)
              # frame = np.reshape(data, (720, 640, 3))
              # Increment frame counter
      
              width = 1280
              height = 720
      
              y_size = width * height
              uv_size = int(y_size / 2)  
      
              y_data = data[:y_size]
              u_data = data[y_size:y_size + uv_size]
              v_data = data[y_size + uv_size:]
      
              yuv_data_reshaped = np.concatenate((y_data, u_data, v_data))
      
              yuv_data_reshaped = yuv_data_reshaped.reshape((int(height * 1.5), width))
              frame = yuv_data_reshaped
              frame = cv2.cvtColor(yuv_data_reshaped, cv2.COLOR_YUV2BGR_I420)
              #frame = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)
              #frame = cv2.cvtColor(frame, cv2.COLOR_YUV2YUV_I420)
              # frame = resize_frame(frame, 640, 480)
              
              def BGRtoNV12(frame):
                  yuv = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_YV12)
                  uv_row_cnt = yuv.shape[0] // 3
                  uv_plane = np.transpose(yuv[uv_row_cnt * 2:].reshape(2, -1), [1, 0])
                  yuv[uv_row_cnt * 2:] = uv_plane.reshape(uv_row_cnt, -1)
                  frame = yuv
                  return frame
      
              #BGRtoNV12(frame)
      
      
              frame_cntr += 1
              sys.stdout.write("\r")
              sys.stdout.write(f'got frame {frame_cntr} with dims {frame.shape}')
              sys.stdout.flush()
              
              pub_image_mpa(frame, frame_cntr)
              # out.write(frame)
          else:
              print("Error mapping buffer")
          
          # unmap buffer so that program would occupy the whole memoy and killed
          buffer.unmap(info)
      
      # out.release()
      
      # Stop the pipeline
      pipeline.set_state(Gst.State.NULL)
      
      
      posted in Ask your questions right here! tflite-server
      William ChungW
      William Chung