Add support for image source/sink nodes

Author: karls-science

api/datatype.proto

@@ -107,3 +107,36 @@ message AnnotatedTensor {
   // e.g. pipeline latencies
   map<string, google.protobuf.Value> metadata = 2;
 }
+
+enum PixelFormat {
+  kPixelFormatUnknown = 0;
+  kYUV420_888 = 1;
+  kRGB888 = 2;
+  kRGBA8888 = 3;
+  kGrayscale8 = 4;
+  kRAW10 = 5;
+  kRAW16 = 6;
+  kNV12 = 7;
+  kNV21 = 8;
+}
+
+message ImageFrame {
+  uint32 width = 1;
+  uint32 height = 2;
+  PixelFormat format = 3;
+
+  // Stride of the first (Y) plane for multi-plane formats
+  uint32 stride_bytes = 4;
+
+  uint64 sequence_number = 5;
+
+  // Sensor exposure midpoint, in nanoseconds
+  uint64 timestamp_ns = 6;
+  uint64 unix_timestamp_ns = 7;
+
+  // Zero if unknown or not applicable
+  int64 exposure_ns = 8;
+
+  // Empty for internal zero-copy edges, populated for tap transport
+  bytes data = 9;
+}

api/device.proto

@@ -13,6 +13,7 @@ message Peripheral {
       kElectricalStimulation = 2;
       kOpticalStimulation = 3;
       kSpikeSource = 4;
+      kCamera = 5;
     }
     string name = 1;
     string vendor = 2;

api/node.proto

@@ -11,6 +11,8 @@ import "api/nodes/disk_writer.proto";
 import "api/nodes/spike_source.proto";
 import "api/nodes/spike_binner.proto";
 import "api/nodes/application.proto";
+import "api/nodes/image_source.proto";
+import "api/nodes/image_sink.proto";
 
 enum NodeType {
   kNodeTypeUnknown = 0;
@@ -27,6 +29,8 @@ enum NodeType {
   kDiskWriter = 9;
   kSpikeBinner = 10;
   kApplication = 11;
+  kImageSource = 12;
+  kImageSink = 13;
 }
 
 message NodeConfig {
@@ -47,6 +51,8 @@ message NodeConfig {
     SpikeSourceConfig spike_source = 12;
     SpikeBinnerConfig spike_binner = 13;
     ApplicationNodeConfig application = 14;
+    ImageSourceConfig image_source = 15;
+    ImageSinkConfig image_sink = 16;
   }
 }
 
@@ -63,6 +69,8 @@ message NodeStatus {
     ApplicationNodeStatus application = 7;
     OpticalStimulationStatus optical_stimulation = 8;
     DiskWriterStatus disk_writer = 10;
+    ImageSourceStatus image_source = 11;
+    ImageSinkStatus image_sink = 12;
   }
 }
 

api/nodes/image_sink.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+package synapse;
+
+message ImageSinkConfig {
+  oneof target {
+    DisplaySinkConfig display = 1;
+    TapSinkConfig tap = 2;
+  }
+}
+
+message DisplaySinkConfig {
+  bool front_buffer = 1;
+  bool vsync = 2;
+}
+
+message TapSinkConfig {
+  ImageEncoding encoding = 1;
+
+  // 1-100, interpretation depends on encoding
+  uint32 quality = 2;
+
+  // 0 = unlimited (publish every frame)
+  float max_rate_hz = 3;
+}
+
+enum ImageEncoding {
+  kImageEncodingUnknown = 0;
+  kImageEncodingJPEG = 1;
+  kImageEncodingPNG = 2;
+  kImageEncodingRAW = 3;
+}
+
+message ImageSinkStatus {
+  uint64 frames_exported = 1;
+  float actual_rate_hz = 2;
+  float latency_ms = 3;
+}

api/nodes/image_source.proto

@@ -0,0 +1,48 @@
+syntax = "proto3";
+
+package synapse;
+
+import "api/datatype.proto";
+
+message ImageSourceConfig {
+  oneof source {
+    CameraSourceConfig camera = 1;
+    TapSourceConfig tap = 2;
+  }
+}
+
+message CameraSourceConfig {
+  uint32 peripheral_id = 1;
+  uint32 width = 2;
+  uint32 height = 3;
+  PixelFormat format = 4;
+  float frame_rate_hz = 5;
+  CameraISPConfig isp = 6;
+}
+
+message TapSourceConfig {
+}
+
+message CameraISPConfig {
+  AutoExposureMode ae_mode = 1;
+  int64 exposure_ns = 2;
+  int32 sensitivity_iso = 3;
+  bool noise_reduction = 4;
+  bool edge_enhancement = 5;
+  bool autofocus = 6;
+}
+
+enum AutoExposureMode {
+  kAEUnknown = 0;
+  kAEOff = 1;
+  kAEOn = 2;
+}
+
+message ImageSourceStatus {
+  float actual_frame_rate_hz = 1;
+  int64 actual_exposure_ns = 2;
+  float focus_diopters = 3;
+  bool lens_stationary = 4;
+  uint64 frames_produced = 5;
+  uint64 frames_dropped = 6;
+}