Optimize memory management and frame conversions

fbs/CMakeLists.txt

@@ -7,8 +7,8 @@ else ()
 endif ()
 
 ExternalProject_Add(flatcc-native
-        URL https://github.com/dvidelabs/flatcc/archive/v0.6.1.tar.gz
-        URL_HASH SHA512=46ba5ca75facc7d3360dba797d24ae7bfe539a854a48831e1c7b96528cf9594d8bea22b267678fd7c6d742b6636d9e52930987119b4c6b2e38d4abe89b990cae
+        GIT_REPOSITORY https://github.com/dvidelabs/flatcc.git
+        GIT_TAG 29201734bf2d12713a7a1a035d31e5123aac9c93
         DOWNLOAD_EXTRACT_TIMESTAMP TRUE
         CMAKE_ARGS
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
@@ -53,8 +53,8 @@ else ()
 endif ()
 
 ExternalProject_Add(flatcc-target
-        URL https://github.com/dvidelabs/flatcc/archive/v0.6.1.tar.gz
-        URL_HASH SHA512=46ba5ca75facc7d3360dba797d24ae7bfe539a854a48831e1c7b96528cf9594d8bea22b267678fd7c6d742b6636d9e52930987119b4c6b2e38d4abe89b990cae
+        GIT_REPOSITORY https://github.com/dvidelabs/flatcc.git
+        GIT_TAG 29201734bf2d12713a7a1a035d31e5123aac9c93
         DOWNLOAD_EXTRACT_TIMESTAMP TRUE
         CMAKE_ARGS
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}

src/hyperion_client.c

@@ -10,6 +10,7 @@
 #include <string.h>
 #include <sys/socket.h>
 #include <sys/types.h>
+#include <sys/uio.h>
 #include <sys/un.h>
 #include <unistd.h>
 
@@ -19,6 +20,7 @@
 static int _connect_unix_socket(const char* hostname);
 static int _connect_inet_socket(const char* hostname, int port);
 static int _send_message(const void* buffer, size_t size);
+static int _write_message(const void* buffer, size_t size);
 static bool _parse_reply(hyperionnet_Reply_table_t reply);
 
 static int sockfd;
@@ -88,8 +90,8 @@ int hyperion_set_nv12_image(const uint8_t* y, const uint8_t* uv, int width, int
 {
     flatbuffers_builder_t B;
     flatcc_builder_init(&B);
-    flatbuffers_uint8_vec_ref_t yData = flatcc_builder_create_type_vector(&B, y, width * height);
-    flatbuffers_uint8_vec_ref_t uvData = flatcc_builder_create_type_vector(&B, uv, width * height / 2);
+    flatbuffers_uint8_vec_ref_t yData = flatcc_builder_create_type_vector(&B, y, stride_y * height);
+    flatbuffers_uint8_vec_ref_t uvData = flatcc_builder_create_type_vector(&B, uv, stride_uv * height / 2);
     hyperionnet_RawImage_ref_t nv12Img = hyperionnet_NV12Image_create(&B, yData, uvData, width, height, stride_y, stride_uv);
     hyperionnet_Image_ref_t imageReq = hyperionnet_Image_create(&B, hyperionnet_ImageType_as_NV12Image(nv12Img), -1);
     hyperionnet_Request_create_as_root(&B, hyperionnet_Command_as_Image(imageReq));
@@ -112,23 +114,27 @@ int hyperion_set_register(const char* origin, int priority)
 
     size_t size;
     void* buf = flatcc_builder_finalize_buffer(&B, &size);
-    uint8_t header[4] = {
+    int ret = _write_message(buf, size);
+
+    free(buf);
+    flatcc_builder_clear(&B);
+    return ret;
+}
+
+int _write_message(const void* buffer, size_t size)
+{
+    const uint8_t header[] = {
         (uint8_t)((size >> 24) & 0xFF),
         (uint8_t)((size >> 16) & 0xFF),
         (uint8_t)((size >> 8) & 0xFF),
-        (uint8_t)(size & 0xFF),
+        (uint8_t)(size & 0xFF)
+    };
+    const struct iovec iov[] = {
+        { (void*)header, sizeof(header) },
+        { (void*)buffer, size }
     };
 
-    // write message
-    int ret = 0;
-    if (write(sockfd, header, 4) < 0)
-        ret = -1;
-    if (write(sockfd, buf, size) < 0)
-        ret = -1;
-
-    free(buf);
-    flatcc_builder_clear(&B);
-    return ret;
+    return writev(sockfd, iov, 2) == (ssize_t)(sizeof(header) + size) ? 0 : -1;
 }
 
 int _send_message(const void* buffer, size_t size)
@@ -142,20 +148,7 @@ int _send_message(const void* buffer, size_t size)
         return hyperion_set_register(_origin, _priority);
     }
 
-    const uint8_t header[] = {
-        (uint8_t)((size >> 24) & 0xFF),
-        (uint8_t)((size >> 16) & 0xFF),
-        (uint8_t)((size >> 8) & 0xFF),
-        (uint8_t)(size & 0xFF)
-    };
-
-    // write message
-    int ret = 0;
-    if (write(sockfd, header, 4) < 0)
-        ret = -1;
-    if (write(sockfd, buffer, size) < 0)
-        ret = -1;
-    return ret;
+    return _write_message(buffer, size);
 }
 
 bool _parse_reply(hyperionnet_Reply_table_t reply)
@@ -267,4 +260,4 @@ int _connect_unix_socket(const char* hostname)
     _connected = true;
 
     return 0;
-}
+}

unicapture/backends/libdile_vt.c

@@ -150,9 +150,9 @@ int capture_init(cap_backend_config_t* config, void** state_p)
     }
 
     INFO("[DILE_VT] vfbs: %d; planes: %d", this->vfbcap.numVfbs, this->vfbcap.numPlanes);
-    uint32_t** ptr = calloc(sizeof(uint32_t*), this->vfbcap.numVfbs);
+    uint32_t** ptr = calloc(this->vfbcap.numVfbs, sizeof(uint32_t*));
     for (uint32_t vfb = 0; vfb < this->vfbcap.numVfbs; vfb++) {
-        ptr[vfb] = calloc(sizeof(uint32_t), this->vfbcap.numPlanes);
+        ptr[vfb] = calloc(this->vfbcap.numPlanes, sizeof(uint32_t));
     }
 
     this->vfbprop.ptr = ptr;

unicapture/backends/libvtcapture.cpp

@@ -1,4 +1,5 @@
 #include <stdexcept>
+#include <stdio.h> // snprintf()
 #include <stdlib.h> // calloc()
 #include <unistd.h> // usleep()
 
@@ -24,6 +25,7 @@ typedef struct _vtcapture_backend_state {
     _LibVtCaptureBufferInfo buff;
     _LibVtCaptureProperties props;
     char* curr_buff;
+    bool buff_valid;
     bool terminate;
     bool quirk_force_capture;
 } vtcapture_backend_state_t;
@@ -115,7 +117,7 @@ int capture_start(void* state)
     const VT_CALLER_T* caller = "hyperion-webos_service";
     vtcapture_backend_state_t* self = (vtcapture_backend_state_t*)state;
 
-    sprintf(self->client, "%s", "00");
+    snprintf(self->client, sizeof(self->client), "%s", "00");
 
     if ((ret = vtCapture_init(self->driver, caller, self->client)) == 17) {
 
@@ -182,6 +184,7 @@ int capture_start(void* state)
         plane.activeregion.a, plane.activeregion.b, plane.activeregion.c, plane.activeregion.d);
 
     self->terminate = false;
+    self->buff_valid = false;
 
     INFO("vtcapture initialization finished.");
 
@@ -213,6 +216,7 @@ int capture_terminate(void* state)
     vtcapture_backend_state_t* self = (vtcapture_backend_state_t*)state;
 
     self->terminate = true;
+    self->buff_valid = false;
 
     vtCapture_stop(self->driver, self->client);
     vtCapture_postprocess(self->driver, self->client);
@@ -224,10 +228,9 @@ int capture_terminate(void* state)
 int capture_acquire_frame(void* state, frame_info_t* frame)
 {
     vtcapture_backend_state_t* self = (vtcapture_backend_state_t*)state;
-    _LibVtCaptureBufferInfo buff;
     int ret = 0;
 
-    if ((ret = vtCapture_currentCaptureBuffInfo(self->driver, &buff)) != 0) {
+    if (!self->buff_valid && (ret = vtCapture_currentCaptureBuffInfo(self->driver, &self->buff)) != 0) {
 
         ERR("vtCapture_currentCaptureBuffInfo() failed: %d", ret);
         return -1;
@@ -236,12 +239,13 @@ int capture_acquire_frame(void* state, frame_info_t* frame)
     frame->pixel_format = PIXFMT_YUV420_SEMI_PLANAR; // ToDo: I guess?!
     frame->width = self->width;
     frame->height = self->height;
-    frame->planes[0].buffer = reinterpret_cast<uint8_t*>(buff.start_addr0);
+    frame->planes[0].buffer = reinterpret_cast<uint8_t*>(self->buff.start_addr0);
     frame->planes[0].stride = self->stride;
-    frame->planes[1].buffer = reinterpret_cast<uint8_t*>(buff.start_addr1);
+    frame->planes[1].buffer = reinterpret_cast<uint8_t*>(self->buff.start_addr1);
     frame->planes[1].stride = self->stride;
 
     self->curr_buff = self->buff.start_addr0;
+    self->buff_valid = false;
 
     return 0;
 }
@@ -305,6 +309,7 @@ int capture_wait(void* state)
     }
 
     self->curr_buff = self->buff.start_addr0;
+    self->buff_valid = true;
 
     return 0;
 }

unicapture/converter.c

@@ -1,11 +1,29 @@
 #include "converter.h"
 #include <libyuv.h>
+#include <stdlib.h>
 #include <string.h>
 
+static int converter_reserve(converter_t* converter, int idx, size_t size)
+{
+    if (converter->capacities[idx] >= size) {
+        return 0;
+    }
+
+    uint8_t* buffer = realloc(converter->buffers[idx], size);
+    if (!buffer) {
+        return -1;
+    }
+
+    converter->buffers[idx] = buffer;
+    converter->capacities[idx] = size;
+    return 0;
+}
+
 void converter_init(converter_t* this)
 {
     for (int i = 0; i < MAX_PLANES; i++) {
         this->buffers[i] = NULL;
+        this->capacities[i] = 0;
     }
 }
 
@@ -14,7 +32,9 @@ int converter_release(converter_t* converter)
     for (int i = 0; i < MAX_PLANES; i++) {
         if (converter->buffers[i] != NULL) {
             free(converter->buffers[i]);
+            converter->buffers[i] = NULL;
         }
+        converter->capacities[i] = 0;
     }
     return 0;
 }
@@ -41,7 +61,9 @@ int converter_run(converter_t* this, frame_info_t* input, frame_info_t* output,
 
             output->planes[i].stride = input->planes[i].stride;
             if (input->planes[i].buffer) {
-                this->buffers[i] = realloc(this->buffers[i], size);
+                if (converter_reserve(this, i, size) != 0) {
+                    return -1;
+                }
                 memcpy(this->buffers[i], input->planes[i].buffer, size);
                 output->planes[i].buffer = this->buffers[i];
             }
@@ -53,7 +75,9 @@ int converter_run(converter_t* this, frame_info_t* input, frame_info_t* output,
         output->width = input->width;
         output->height = input->height;
 
-        this->buffers[0] = realloc(this->buffers[0], output->width * output->height * 4);
+        if (converter_reserve(this, 0, output->width * output->height * 4) != 0) {
+            return -1;
+        }
 
         output->planes[0].buffer = this->buffers[0];
         output->planes[0].stride = output->width * 4;
@@ -85,8 +109,10 @@ int converter_run(converter_t* this, frame_info_t* input, frame_info_t* output,
                 output->width,
                 output->height);
         } else if (input->pixel_format == PIXFMT_YUV422_SEMI_PLANAR) {
-            this->buffers[1] = realloc(this->buffers[1], input->width / 2 * input->height);
-            this->buffers[2] = realloc(this->buffers[2], input->width / 2 * input->height);
+            if (converter_reserve(this, 1, input->width / 2 * input->height) != 0
+                || converter_reserve(this, 2, input->width / 2 * input->height) != 0) {
+                return -1;
+            }
             SplitUVPlane(
                 input->planes[1].buffer,
                 input->planes[1].stride,
@@ -118,8 +144,10 @@ int converter_run(converter_t* this, frame_info_t* input, frame_info_t* output,
         return 0;
     }
     if (target_format == PIXFMT_YUV420_SEMI_PLANAR && input->pixel_format == PIXFMT_ARGB) {
-        this->buffers[0] = realloc(this->buffers[0], input->width * input->height);
-        this->buffers[1] = realloc(this->buffers[1], input->width * input->height / 2);
+        if (converter_reserve(this, 0, input->width * input->height) != 0
+            || converter_reserve(this, 1, input->width * input->height / 2) != 0) {
+            return -1;
+        }
 
         output->width = input->width;
         output->height = input->height;
@@ -140,7 +168,9 @@ int converter_run(converter_t* this, frame_info_t* input, frame_info_t* output,
         return 0;
     }
     if (target_format == PIXFMT_RGB && input->pixel_format == PIXFMT_ARGB) {
-        this->buffers[0] = realloc(this->buffers[0], input->width * input->height * 3);
+        if (converter_reserve(this, 0, input->width * input->height * 3) != 0) {
+            return -1;
+        }
 
         output->width = input->width;
         output->height = input->height;
@@ -158,10 +188,12 @@ int converter_run(converter_t* this, frame_info_t* input, frame_info_t* output,
     }
 
     if (target_format == PIXFMT_YUV420_SEMI_PLANAR && input->pixel_format == PIXFMT_YUV422_SEMI_PLANAR) {
-        this->buffers[0] = realloc(this->buffers[0], input->planes[0].stride * input->height); // Y
-        this->buffers[1] = realloc(this->buffers[1], input->width / 2 * input->height); // U
-        this->buffers[2] = realloc(this->buffers[2], input->width / 2 * input->height); // V
-        this->buffers[3] = realloc(this->buffers[3], input->width / 2 * input->height); // UV
+        if (converter_reserve(this, 0, input->planes[0].stride * input->height) != 0 // Y
+            || converter_reserve(this, 1, input->width / 2 * input->height) != 0 // U
+            || converter_reserve(this, 2, input->width / 2 * input->height) != 0 // V
+            || converter_reserve(this, 3, input->width / 2 * input->height) != 0) { // UV
+            return -1;
+        }
         SplitUVPlane(
             input->planes[1].buffer,
             input->planes[1].stride,

unicapture/converter.h

@@ -1,12 +1,14 @@
 #pragma once
 
+#include <stddef.h>
 #include <stdint.h>
 
 #include "unicapture.h"
 
 typedef struct _converter {
     // Temporary conversion memory buffers
     uint8_t* buffers[4];
+    size_t capacities[4];
 } converter_t;
 
 void converter_init(converter_t* converter);