diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java index e6fe0a6be..77193644e 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/FFmpegProcess.java @@ -246,6 +246,7 @@ public void doStart() throws SensorException, SensorHubException { // Reinit executable. Not always necessary, but doesn't hurt. executable.init(); process.start(this::onError); + //process.start(this::onError); } catch (ProcessException e) { logger.error("Could not initialize process.", e); return; @@ -459,7 +460,7 @@ public void publishData() { } - // Listens for event from dara + // Listens for event from data protected class DataQueuePusher implements IEventListener { DataQueue dataQueue; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java index 701c49996..d0cdd3db3 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/CodecEnum.java @@ -11,15 +11,14 @@ public enum CodecEnum { //AUTO("auto"), H264(AV_CODEC_ID_H264), - H265(AV_CODEC_ID_H265), - HEVC(AV_CODEC_ID_HEVC), // HEVC and H265 are the same. Having both in this enum helps with auto codec detection. + HEVC(AV_CODEC_ID_HEVC), MJPEG(AV_CODEC_ID_MJPEG), VP8(AV_CODEC_ID_VP8), VP9(AV_CODEC_ID_VP9), MPEG2(AV_CODEC_ID_MPEG2TS), MPEG4(AV_CODEC_ID_MPEG4), - RGB(AV_PIX_FMT_RGB24), - YUV(AV_PIX_FMT_YUV420P); + RGB24(AV_PIX_FMT_RGB24), + YUV420P(AV_PIX_FMT_YUV420P); int ffmpegId; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java index 9ea520d92..1a8d7af87 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFMpegTranscoder.java @@ -18,18 +18,13 @@ import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.javacpp.BytePointer; -import org.bytedeco.javacpp.DoublePointer; -import org.bytedeco.javacpp.Pointer; -import org.sensorhub.api.common.SensorHubException; import org.sensorhub.api.processing.OSHProcessInfo; -import org.sensorhub.impl.process.video.transcoder.coders.Coder; -import org.sensorhub.impl.process.video.transcoder.coders.Decoder; -import org.sensorhub.impl.process.video.transcoder.coders.Encoder; -import org.sensorhub.impl.process.video.transcoder.coders.SwScaler; -import org.sensorhub.impl.process.video.transcoder.formatters.AVByteFormatter; -import org.sensorhub.impl.process.video.transcoder.formatters.PacketFormatter; -import org.sensorhub.impl.process.video.transcoder.formatters.RgbFormatter; -import org.sensorhub.impl.process.video.transcoder.formatters.YuvFormatter; +import org.sensorhub.impl.process.video.transcoder.coders.*; +import org.sensorhub.impl.process.video.transcoder.formatters.*; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.vast.data.DataBlockCompressed; @@ -39,13 +34,10 @@ import org.vast.swe.helper.RasterHelper; import javax.annotation.Nullable; -import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; -import static org.bytedeco.ffmpeg.global.swscale.*; /** *

@@ -62,8 +54,7 @@ public class FFMpegTranscoder extends ExecutableProcessImpl public static final OSHProcessInfo INFO = new OSHProcessInfo("video:FFMpegTranscoder", "FFMPEG Video Transcoder", null, FFMpegTranscoder.class); - AtomicBoolean doRun = new AtomicBoolean(true); - AtomicBoolean isRunning = new AtomicBoolean(false); + AtomicBoolean isInit = new AtomicBoolean(false); Time inputTimeStamp; Count inputWidth, inputHeight; DataArray imgIn; @@ -73,18 +64,14 @@ public class FFMpegTranscoder extends ExecutableProcessImpl Text inCodecParam; Text outCodecParam; - List> videoProcs; - AVByteFormatter inputFormatter, outputFormatter; - ArrayDeque inputPackets; - ArrayDeque outputPackets; - Thread outputThread; + List videoProcs; + AVByteFormatter inputFormatter, outputFormatter; - HashMap decOptions = new HashMap<>(); - HashMap encOptions = new HashMap<>(); + CodecOptions decOptions, encOptions; final boolean publish = false; - CodecEnum inCodec; - CodecEnum outCodec; + CodecInfo inCodec; + CodecInfo outCodec; RasterHelper swe = new RasterHelper(); int width, height, outWidth, outHeight; @@ -122,13 +109,13 @@ public FFMpegTranscoder() paramData.add("inCodec", inCodecParam = swe.createText() .definition(SWEHelper.getPropertyUri("Codec")) .label("Input Codec Name") - .addAllowedValues(CodecEnum.class) + //.addAllowedValues(CodecEnum.class) .build()); paramData.add("outCodec", outCodecParam = swe.createText() .definition(SWEHelper.getPropertyUri("Codec")) .label("Output Codec Name") - .addAllowedValues(CodecEnum.class) + //.addAllowedValues(CodecEnum.class) .build()); // outputs @@ -156,81 +143,86 @@ public FFMpegTranscoder() } + private void initFormatters() throws ProcessException { + inputFormatter = getFormatter(inCodec, width, height); + outputFormatter = getFormatter(outCodec, outWidth, outHeight); + } + /** * Initializes all encoder/decoder/swscaler objects. These objects are added to the {@link FFMpegTranscoder#videoProcs} * queue in the order they should process the incoming data. At most, the flow will be Decoder -> SWScale -> Encoder. */ - private void initCoders() { - try { - stopProcessThreads(); - } catch (Exception e){ - logger.error("Transcoder could not stop process threads during re-init.", e); - } + private void initCodecs() { if (videoProcs != null) { videoProcs.clear(); } videoProcs = new ArrayList<>(); + Decoder decoder = null; + Encoder encoder = null; + SwScaler swScaler = null; + if (!isUncompressed(inCodec)) { - videoProcs.add(new Decoder(inCodec.ffmpegId, decOptions)); - } - if (width != outWidth || height != outHeight || (isUncompressed(inCodec) && isUncompressed(outCodec))) { - int inFmt = isUncompressed(inCodec) ? inCodec.ffmpegId : AV_PIX_FMT_YUV420P; - int outFmt = isUncompressed(outCodec) ? outCodec.ffmpegId : AV_PIX_FMT_YUV420P; - videoProcs.add(new SwScaler(inFmt, outFmt, width, height, outWidth, outHeight)); + decoder = new Decoder(inCodec, outCodec, decOptions); + inCodec.pixelFmt = decoder.init(); + if (inCodec.pixelFmt == null) + inCodec.pixelFmt = FullPixelEnum.YUV420P; + //videoProcs.add(decoder); } + if (!isUncompressed(outCodec)) { - videoProcs.add(new Encoder(outCodec.ffmpegId, encOptions)); + encoder = new Encoder(inCodec, outCodec, encOptions); + outCodec.pixelFmt = encoder.init(); + //videoProcs.add(encoder); } - inputPackets = new ArrayDeque<>(); - if (videoProcs.get(0) != null) { - videoProcs.get(0).setInQueue(inputPackets); - } - for (int i = 1; i < videoProcs.size(); i++) { - videoProcs.get(i).setInQueue(videoProcs.get(i - 1).getOutQueue()); - } - try { - outputPackets = (ArrayDeque) videoProcs.get(videoProcs.size() - 1).getOutQueue(); - } catch (Exception e) { - logger.warn("No processes running on video input. Check codec/resolution settings.", e); - outputPackets = inputPackets; - } - } + // Always want swScaler. Decoder can output frames in a format different from what was set. + swScaler = new SwScaler(inCodec, outCodec, width, height, outWidth, outHeight); + swScaler.init(); + //videoProcs.add(swScaler); - /** - * Invoked during the first call to {@link FFMpegTranscoder#execute()} (when {@link FFMpegTranscoder#isRunning} is false). - * Start all {@link Coder} thread objects stored in {@link FFMpegTranscoder#videoProcs}. - */ - private void startProcessThreads() { - doRun.set(true); - if (videoProcs == null || videoProcs.isEmpty() || videoProcs.get(0).getState() != Thread.State.NEW) { - initCoders(); - } + if (decoder != null) { videoProcs.add(decoder); } + if (swScaler != null) { videoProcs.add(swScaler); } + if (encoder != null) { videoProcs.add(encoder); } + + logger.info("Input pixel format: {}, Output pixel format: {}", inCodec.pixelFmt, outCodec.pixelFmt); + + if (inCodec.pixelFmt == null || outCodec.pixelFmt == null) { logger.warn("Pixel format is null"); } - for (Thread process : videoProcs) { - process.start(); + } + + private void initPipeline() { + // Frame pipe between decoder, swscaler, encoder + for (int i = 0; i < videoProcs.size() - 1; i++) { + var nextProc = videoProcs.get(i + 1); + videoProcs.get(i).registerCallback(packet -> { + nextProc.submitInputPacket(packet); + }); } - outputThread.start(); - isRunning.set(true); + // Output + var finalProc = videoProcs.get(videoProcs.size() - 1); + finalProc.registerCallback(packet -> { + try { + publishFrameData(outputFormatter.convertOutput(packet)); + } finally { + finalProc.deallocateOutputPacket(packet); + } + }); } /** * Invoked on process stop and init. - * Stop all {@link Coder} thread objects stored in {@link FFMpegTranscoder#videoProcs}. + * Stop all {@link Codec} thread objects stored in {@link FFMpegTranscoder#videoProcs}. */ - private void stopProcessThreads() throws InterruptedException { - doRun.set(false); //TODO These atomic booleans may be entirely unnecessary, remove + private void stopProcessing() throws InterruptedException { + isInit.set(false); if (videoProcs != null) { - for (Thread thread : videoProcs) { - thread.interrupt(); - thread.join(); + for (Codec codec : videoProcs) { + codec.close(); } + videoProcs.clear(); } - outputThread.interrupt(); - outputThread.join(); - isRunning.set(false); } @@ -266,12 +258,10 @@ public void notifyParamChange() @Override public void stop() { - if (isRunning.get()) { - try { - stopProcessThreads(); - } catch (InterruptedException e) { - logger.warn("Interrupted while stopping process threads"); - } + try { + stopProcessing(); + } catch (InterruptedException e) { + logger.warn("Interrupted while stopping process threads"); } super.stop(); } @@ -321,32 +311,30 @@ private void setImgEncoding() { @Override public void init() throws ProcessException { - doRun.set(true); - isRunning.set(false); + if (!isInit.compareAndSet(false, true)) { + return; + } // init decoder according to configured codec // TODO: Automatically detect input codec from compression in data struct? try { - inCodec = CodecEnum.valueOf(inCodecParam.getData().getStringValue().toUpperCase()); - outCodec = CodecEnum.valueOf(outCodecParam.getData().getStringValue().toUpperCase()); + //inCodec = CodecEnum.valueOf(inCodecParam.getData().getStringValue().toUpperCase()); + //outCodec = CodecEnum.valueOf(outCodecParam.getData().getStringValue().toUpperCase()); + inCodec = CodecInfo.newCodecInfoFromName(inCodecParam.getData().getStringValue()); + outCodec = CodecInfo.newCodecInfoFromName(outCodecParam.getData().getStringValue()); setImgEncoding(); - initCodecOptions(); - - // processThreads are always running, passing available data from decoder to encoder and encoder to output - initCoders(); - outputThread = new Thread(this::outputProcess); - - inputFormatter = getFormatter(inCodec, width, height); - outputFormatter = getFormatter(outCodec, outWidth, outHeight); + initCodecs(); + initFormatters(); + initPipeline(); imgOut.setData(new DataBlockCompressed()); } catch (IllegalArgumentException e) { - reportError("Unsupported codec" + inCodecParam.getData().getStringValue() + ". Must be one of " + Arrays.toString(CodecEnum.values())); + reportError("Unsupported codec " + inCodecParam.getData().getStringValue() + ". Must be one of " + Arrays.toString(CodecEnum.values()), e); } super.init(); @@ -356,65 +344,59 @@ public void init() throws ProcessException * Creates maps of options for the encoder/decoder, including framerate, pixel format, bitrate, and image size. */ private void initCodecOptions() { - decOptions = new HashMap<>(); - encOptions = new HashMap<>(); + var decOptionBuilder = new CodecOptions.Builder(); + var encOptionBuilder = new CodecOptions.Builder(); //DataComponent temp; int fps = safeGetCountVal(inputFps); - if (fps > 0) { - decOptions.put("fps", fps); - encOptions.put("fps", fps); - } - - encOptions.put("pix_fmt", AV_PIX_FMT_YUV420P); - decOptions.put("pix_fmt", AV_PIX_FMT_YUV420P); - int bitrate = safeGetCountVal(inputBitrate); - if (bitrate > 0) { - decOptions.put("bit_rate", bitrate); - encOptions.put("bit_rate", bitrate); // Just assuming input br is the same as out, could this change? - } - width = safeGetCountVal(inputWidth); if (width <= 0) { - width = imgIn.getComponent("row").getComponentCount(); + try { + width = imgIn.getComponent("row").getComponentCount(); + } catch (Exception e) { + width = 1920; + logger.warn("Input width not specified, using default: 1920", e); + } } - decOptions.put("width", width); height = safeGetCountVal(inputHeight); if (height <= 0) { - height = imgIn.getComponentCount(); + try { + height = imgIn.getComponentCount(); + } catch (Exception e) { + height = 1080; + logger.warn("Input height not specified, using default: 1080", e); + } } - decOptions.put("height", height); outHeight = safeGetCountVal(outputHeight); if (outHeight <= 0) { try { outHeight = imgOut.getComponentCount(); - } catch (Exception ignored) { - outHeight = 0; + } catch (Exception e) { + outHeight = height; + logger.warn("Output height not specified, using input height", e); } } - if (outHeight > 0) { - encOptions.put("height", outHeight); - } else { - encOptions.put("height", height); - } outWidth = safeGetCountVal(outputWidth); if (outWidth <= 0) { try { outWidth = imgIn.getComponent("row").getComponentCount(); - } catch (Exception ignored) { - outWidth = 0; + } catch (Exception e) { + outWidth = width; + logger.warn("Output width not specified, using input width", e); } } - if (outWidth > 0) { - encOptions.put("width", outWidth); - } else { - encOptions.put("width", width); - } + + decOptions = decOptionBuilder.setFps(fps).setBitRate(bitrate) + .setWidth(width).setHeight(height).presetUltraFast().tuneZeroLatency() + .setComplianceUnofficial().build(); + encOptions = encOptionBuilder.setFps(fps).setBitRate(bitrate) + .setWidth(outWidth).setHeight(outHeight).presetUltraFast().tuneZeroLatency() + .setComplianceUnofficial().build(); } /** @@ -426,25 +408,31 @@ private void initCodecOptions() { * @return Formatter object. * @throws ProcessException Thrown when width and height are not provided for an uncompressed format. */ - private AVByteFormatter getFormatter(CodecEnum codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { + private AVByteFormatter getFormatter(CodecInfo codec, @Nullable Integer width, @Nullable Integer height) throws ProcessException { try { + if (isUncompressed(codec)) + return new FrameFormatter(width, height, codec.pixelFmt.ffmpegId); + else + return new PacketFormatter(); + /* return switch (codec) { case RGB -> new RgbFormatter(width, height); case YUV -> new YuvFormatter(width, height); default -> new PacketFormatter(); }; + */ } catch (NullPointerException e) { - reportError("Raw formatter for " + codec + " requires non-null width and height.", e); + reportError("Formatter for " + codec + " requires non-null width and height.", e); } return null; } /** * @param codec Video codec or uncompressed format. - * @return Is the codec {@link CodecEnum#RGB} or {@link CodecEnum#YUV}? + * @return Is the codec {@link FullCodecEnum#RAWVIDEO}? */ - private boolean isUncompressed(CodecEnum codec) { - return codec == CodecEnum.RGB || codec == CodecEnum.YUV; + private boolean isUncompressed(CodecInfo codec) { + return codec.codec == FullCodecEnum.RAWVIDEO; } /** @@ -480,12 +468,15 @@ private void publishFrameData(byte[] frameData) { ((DataBlockCompressed) imgOut.getData()).setUnderlyingObject(frameData.clone()); // also copy frame timestamp - double ts; + double ts = System.currentTimeMillis() / 1000d; + /* if (inputTimeStamp != null && inputTimeStamp.getData() != null) { ts = inputTimeStamp.getData().getDoubleValue(); } else { ts = System.currentTimeMillis(); } + + */ outputTimeStamp.getData().setDoubleValue(ts); try { //logger.debug("Publishing"); @@ -503,42 +494,29 @@ public void execute() throws ProcessException logger.warn("Input image is null"); return; } + // Start the threads if not already started - if (!isRunning.get()) { - startProcessThreads(); + if (!isInit.get()) { + init(); } - inputPackets.add( - inputFormatter.convertInput( - ((DataBlockCompressed)imgIn.getData()).getUnderlyingObject().clone() - )); + if (!isVideoProcChainReady()) { + logger.warn("Video processor not ready"); + return; + } + videoProcs.get(0).submitInputPacket( + inputFormatter.convertInput(((DataBlockCompressed)imgIn.getData()).getUnderlyingObject().clone()) + ); } - /** - * Runs inside a separate thread. Receives any {@link AVPacket}s or {@link AVFrame}s from the last {@link Coder} - * in the {@link FFMpegTranscoder#videoProcs} queue, converts the struct to bytes, and publishes the data. - * @see FFMpegTranscoder#publishFrameData(byte[]) - */ - private void outputProcess() { - while (doRun.get() && !Thread.currentThread().isInterrupted()) { - while (outputPackets == null || outputPackets.isEmpty()) { - if (Thread.currentThread().isInterrupted()) { - return; - } - Thread.onSpinWait(); - } - if (outputPackets != null && !outputPackets.isEmpty()) { - for (var packet : outputPackets) { - if (Thread.currentThread().isInterrupted()) { - return; - } - publishFrameData(outputFormatter.convertOutput(outputPackets.poll())); - //frameData = null; - //packet = null; - } + private boolean isVideoProcChainReady() { + for (Codec proc : videoProcs) { + if (!proc.isReady()) { + return false; } } + return true; } @Override @@ -553,24 +531,11 @@ public void dispose() { super.dispose(); - doRun.set(false); + isInit.set(false); if (videoProcs != null) { - for (Thread t : videoProcs) { - t.interrupt(); - try { - t.join(); - } catch (InterruptedException e) { - logger.error("Error waiting for process thread {} to finish", t.getName()); - } - } - } - if (outputThread != null) { - outputThread.interrupt(); - try { - outputThread.join(); - } catch (InterruptedException e) { - logger.error("Error waiting for process thread {} to finish", outputThread.getName()); + for (Codec proc : videoProcs) { + proc.close(); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java index b14c3bc5c..365093129 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderConfig.java @@ -6,17 +6,23 @@ public class FFmpegTranscoderConfig extends FFmpegProcessConfig { @DisplayInfo.Required - @DisplayInfo(label="Input Codec") + @DisplayInfo(label="Input Format") public CodecEnum inCodec = CodecEnum.H264; - @DisplayInfo(label="Automatically Detect Input Codec", desc="If enabled, process will attempt to determine the input codec." - + " If a codec could not be determined from the input data, it will fall back to the selected Input Codec.") + @DisplayInfo(label="Input Format Override") + public String inCodecOverride = ""; + + @DisplayInfo(label="Automatically Detect Input Format", desc="If enabled, process will attempt to determine the input format." + + " If a codec could not be determined from the input data, it will fall back to the selected Input Format.") public boolean detectInput = false; @DisplayInfo.Required - @DisplayInfo(label="Output Codec") + @DisplayInfo(label="Output Format") public CodecEnum outCodec = CodecEnum.H264; + @DisplayInfo(label="Output Format Override") + public String outCodecOverride = ""; + @DisplayInfo(label="Output Width") public Integer outputWidth = null; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java index a6fe55895..c6d0073a1 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/FFmpegTranscoderProcess.java @@ -19,10 +19,18 @@ public FFmpegTranscoderProcess() { @Override public void initExcProcess(IProcessExec executable) { DataBlock inCodec = new DataBlockString(1); - inCodec.setStringValue(((FFmpegTranscoderConfig)config).inCodec.toString()); + if (!((FFmpegTranscoderConfig)config).inCodecOverride.isBlank()) { + inCodec.setStringValue(((FFmpegTranscoderConfig)config).inCodecOverride); + } else { + inCodec.setStringValue(((FFmpegTranscoderConfig) config).inCodec.toString()); + } DataBlock outCodec = new DataBlockString(1); - outCodec.setStringValue(((FFmpegTranscoderConfig)config).outCodec.toString()); + if (!((FFmpegTranscoderConfig)config).outCodecOverride.isBlank()) { + outCodec.setStringValue(((FFmpegTranscoderConfig)config).outCodecOverride); + } else { + outCodec.setStringValue(((FFmpegTranscoderConfig) config).outCodec.toString()); + } DataBlockInt outWidthBlock = new DataBlockInt(1); if (((FFmpegTranscoderConfig) config).outputWidth != null) { diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md index 7ccf4a2db..fd89f33ab 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/README.md @@ -2,7 +2,8 @@ ## Overview -This module provides a process module that can decode and/or encode video. +FFmpeg video decode/encode/transcode module. +Input and output may be compressed or uncompressed video. ## Configuration @@ -24,17 +25,185 @@ When added to an OpenSensorHub node, the process has the following configuration - **Video Source:** A module with video output. Once the transcoder starts, video from this source module will be decoded/encoded and outputted from the transcoder. - - **Input Codec:** (Optional) - The codec used for decoding the incoming video data. If incoming video is uncompressed, + - **Input Format:** (Optional) + The format used for decoding the incoming video data. If incoming video is uncompressed, select either RGB or YUV. - - **Output Codec:** - The codec used for encoding the outgoing video data. If outgoing video should be uncompressed, + - **Input Format Override:** (Optional) + Manually specify any input codec or pixel format, allowing for any format not available in the short Input Codec list. + Only required if the format is not in the Input Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. + - **Output Format:** + The format used for encoding the outgoing video data. If outgoing video should be uncompressed, select either RGB or YUV. + - **Output Format Override:** (Optional) + Manually specify any output codec or pixel format, allowing for any format not available in the short Output Codec list. + Only required if the format is not in the Output Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. - **Output Width:** (Optional) The width of the output video frame. Leave this empty to avoid scaling the video frame size. - **Output Height:** (Optional) The height of the output video frame. Leave this empty to avoid scaling the video frame size. - **Auto Start:** If checked, automatically start this sensor when the OpenSensorHub node is launched. - - **Automatically Detect Input Codec:** - If checked, automatically determine the input video codec based on the input's encoding information. \ No newline at end of file + - **Automatically Detect Input Format:** + If checked, automatically determine the input video format based on the input's encoding information. + +## All Supported Formats +This list contains all the supported codecs and pixel formats. To use a format, copy it into one of the format +override config fields. + +| Codecs (Compressed Video) | Pixel Formats (Uncompressed Video) | +|---------------------------|------------------------------------| +| H264 | YUV420P | +| HEVC | YUV422P | +| MJPEG | YUV444P | +| VP8 | YUV410P | +| VP9 | YUV411P | +| MPEG2 | YUV440P | +| MPEG4 | YUVJ420P | +| AV1 | YUVJ422P | +| THEORA | YUVJ444P | +| MPEG1VIDEO | YUVJ440P | +| WMV1 | YUV420P9BE | +| WMV2 | YUV420P9LE | +| WMV3 | YUV420P10BE | +| VC1 | YUV420P10LE | +| FLV1 | YUV420P12BE | +| FLASHSV | YUV420P12LE | +| FLASHSV2 | YUV420P14BE | +| RV10 | YUV420P14LE | +| RV20 | YUV420P16BE | +| RV30 | YUV420P16LE | +| RV40 | YUV422P9BE | +| CINEPAK | YUV422P9LE | +| INDEO2 | YUV422P10BE | +| INDEO3 | YUV422P10LE | +| INDEO4 | YUV422P12BE | +| INDEO5 | YUV422P12LE | +| MSMPEG4V1 | YUV422P14BE | +| MSMPEG4V2 | YUV422P14LE | +| MSMPEG4V3 | YUV422P16BE | +| H261 | YUV422P16LE | +| H263 | YUV444P9BE | +| H263I | YUV444P9LE | +| H263P | YUV444P10BE | +| SNOW | YUV444P10LE | +| SVQ1 | YUV444P12BE | +| SVQ3 | YUV444P12LE | +| DVVIDEO | YUV444P14BE | +| HUFFYUV | YUV444P14LE | +| FFVHUFF | YUV444P16BE | +| FFV1 | YUV444P16LE | +| ASV1 | YUYV422 | +| ASV2 | UYVY422 | +| VCR1 | YVYU422 | +| CLJR | UYYVYY411 | +| MDEC | NV12 | +| ROQ | NV21 | +| INTERPLAY_VIDEO | NV16 | +| XAN_WC3 | NV20LE | +| XAN_WC4 | NV20BE | +| RPZA | NV24 | +| SMC | NV42 | +| GIF | RGB24 | +| PNG | BGR24 | +| PPM | ARGB | +| PBM | RGBA | +| PGM | ABGR | +| PAM | BGRA | +| BMP | RGB0 | +| TIFF | BGR0 | +| SGI | RGB8 | +| ALIAS_PIX | BGR8 | +| DPX | RGB4 | +| EXR | BGR4 | +| WEBP | RGB4_BYTE | +| DIRAC | BGR4_BYTE | +| DNXHD | RGB48BE | +| PRORES | RGB48LE | +| JPEG2000 | RGBA64BE | +| JPEGLS | RGBA64LE | +| HAP | BGR48BE | +| | BGR48LE | +| | BGRA64BE | +| | BGRA64LE | +| | RGB565BE | +| | RGB565LE | +| | RGB555BE | +| | RGB555LE | +| | RGB444BE | +| | RGB444LE | +| | BGR565BE | +| | BGR565LE | +| | BGR555BE | +| | BGR555LE | +| | BGR444BE | +| | BGR444LE | +| | GRAY8 | +| | GRAY8A | +| | GRAY9BE | +| | GRAY9LE | +| | GRAY10BE | +| | GRAY10LE | +| | GRAY12BE | +| | GRAY12LE | +| | GRAY14BE | +| | GRAY14LE | +| | GRAY16BE | +| | GRAY16LE | +| | MONOWHITE | +| | MONOBLACK | +| | YA8 | +| | YA16BE | +| | YA16LE | +| | YUVA420P | +| | YUVA422P | +| | YUVA444P | +| | YUVA420P9BE | +| | YUVA420P9LE | +| | YUVA422P9BE | +| | YUVA422P9LE | +| | YUVA444P9BE | +| | YUVA444P9LE | +| | YUVA420P10BE | +| | YUVA420P10LE | +| | YUVA422P10BE | +| | YUVA422P10LE | +| | YUVA444P10BE | +| | YUVA444P10LE | +| | YUVA420P16BE | +| | YUVA420P16LE | +| | YUVA422P16BE | +| | YUVA422P16LE | +| | YUVA444P16BE | +| | YUVA444P16LE | +| | BAYER_BGGR8 | +| | BAYER_RGGB8 | +| | BAYER_GBRG8 | +| | BAYER_GRBG8 | +| | BAYER_BGGR16LE | +| | BAYER_BGGR16BE | +| | BAYER_RGGB16LE | +| | BAYER_RGGB16BE | +| | BAYER_GBRG16LE | +| | BAYER_GBRG16BE | +| | BAYER_GRBG16LE | +| | BAYER_GRBG16BE | +| | GRAYF32BE | +| | GRAYF32LE | +| | PAL8 | +| | XYZ12LE | +| | XYZ12BE | +| | XTOP | +| | P010LE | +| | P010BE | +| | P016LE | +| | P016BE | +| | P210BE | +| | P210LE | +| | P410BE | +| | P410LE | +| | P216BE | +| | P216LE | +| | P416BE | +| | P416LE | \ No newline at end of file diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java new file mode 100644 index 000000000..f1e01f8a7 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Codec.java @@ -0,0 +1,387 @@ +package org.sensorhub.impl.process.video.transcoder.coders; + +import static org.bytedeco.ffmpeg.global.avutil.*; + +import java.util.HashMap; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.bytedeco.ffmpeg.global.avcodec.*; + +import org.bytedeco.ffmpeg.avcodec.AVCodec; +import org.bytedeco.ffmpeg.avcodec.AVCodecContext; +import org.bytedeco.ffmpeg.avcodec.AVPacket; +import org.bytedeco.ffmpeg.avutil.AVFrame; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.PointerPointer; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Consumes AVPackets/AVFrames and produces AVPackets/AVFrames. Asynchronously submit packets to the codec with + * {@link #submitInputPacket(Pointer input)}. Listen for output by submitting a {@link CodecCallback} (typically a lambda) + * to {@link #registerCallback(CodecCallback callback)}. If transcoding, use callbacks to connect the output of one + * {@link #Codec} instance to the {@link #submitInputPacket(Pointer input)} method of the next. + * + *

AVPacket/AVFrame ownership is transferred to the consumer/listener. The consumer shall be responsible for + * deallocating AVPackets/AVFrames by passing them to the {@link #deallocateOutputPacket(Pointer packet)} method of the + * codec which most recently produced them. ONLY deallocate AVPackets/AVFrames that are not in use by a + * {@link #Codec}. {@link #submitInputPacket(Pointer input)} will automatically deallocate its input. + * + *

Codec/Callback Sample: + * 

+ * {@code
+ * // Packet pipe between decoder, swscaler, encoder
+ *         for (int i = 0; i < codecList.size() - 1; i++) {
+ *             var nextCodec = codecList.get(i + 1);
+ *             codecList.get(i).registerCallback(packet -> {
+ *                 nextCodec.submitInputPacket(packet);
+ *             });
+ *         }
+ *
+ *         // Output
+ *         var finalProc = codecList.get(codecList.size() - 1);
+ *         finalProc.registerCallback(packet -> {
+ *             try {
+ *                 publishFrameData(outputFormatter.convertOutput(packet));
+ *             } finally {
+ *                 finalProc.deallocateOutputPacket(packet);
+ *             }
+ *         });
+ * }
+ *
+ * @param Input class (either AVFrame or AVPacket) + * @param Output class (either AVFrame or AVPacket) + */ +public abstract class Codec implements AutoCloseable { + + + public interface CodecCallback { + // The recipient does not need to deallocate the output; this is done automatically + public abstract void onPacket(O packet); + } + + protected static final Logger logger = LoggerFactory.getLogger(Codec.class); + + private static int codecCount = 0; + private final int codecNum = codecCount++; + private final ExecutorService executor = Executors.newSingleThreadExecutor( + r -> new Thread(r, "ffmpeg-codec-thread-" + codecNum)); + private final ExecutorService outputExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-output-thread-" + codecNum)); + private final Map, ExecutorService> callbackMap = new HashMap<>(); + + CodecInfo inputFormat; + CodecInfo outputFormat; + protected AVCodecContext codec_ctx; + protected AVCodec codec; + protected final Queue inQueue = new ConcurrentLinkedQueue<>(); + protected final Queue outQueue = new ConcurrentLinkedQueue<>(); + protected final int maxQueue = 10; + private final AtomicBoolean isProcessing = new AtomicBoolean(false); + private final AtomicBoolean isAlive = new AtomicBoolean(false); // Set false to indicate packets should no longer be accepted + final Object contextLock = new Object(); + Class inputClass; + Class outputClass; + CodecOptions options; + AtomicBoolean isNotifying = new AtomicBoolean(false); + byte[] headers; + + public Codec(CodecInfo inFormatInfo, CodecInfo outFormatInfo, Class inputClass, Class outputClass, CodecOptions options) { + super(); + + if ((inputClass != AVPacket.class && inputClass != AVFrame.class) + || (outputClass != AVPacket.class && outputClass != AVFrame.class)) { + throw new IllegalArgumentException("Input and output classes must be either AVPacket or AVFrame"); + } + + this.inputFormat = inFormatInfo.clone(); + this.inputClass = inputClass; + this.outputFormat = outFormatInfo.clone(); + this.outputClass = outputClass; + this.options = options; + } + + public FullPixelEnum init() { + initContext(); + initOptions(); + var pixFmt = openContext(); + isAlive.set(true); + return pixFmt; + } + + private void submitTask(Runnable task) { + if (!executor.isShutdown()) { + executor.submit(() -> { + try { + task.run(); + } catch (Throwable t) { + // Prevent silent thread replacement — log and propagate + // only after codec state is consistent + logger.error("Fatal error in codec runner thread", t); + //throw t; // will kill this task but not spawn a new thread silently + } + }); + } + } + + protected void addOutPacket(O outPacket) { + while (outQueue.size() >= maxQueue) { + var packet = outQueue.poll(); + deallocateOutputPacket(packet); + } + outQueue.add(outPacket); + } + + protected void addInPacket(I inPacket) { + while (inQueue.size() >= maxQueue) { + var packet = inQueue.poll(); + deallocateInputPacket(packet); + } + inQueue.add(inPacket); + } + + public boolean isReady() { + return isAlive.get(); + } + + public Class getOutputClass() { + return outputClass; + } + + public Class getInputClass() { + return inputClass; + } + + protected abstract void initContext(); + + protected FullPixelEnum openContext() { + int ret; + FullPixelEnum pixelFmt; + + codec_ctx.flags(codec_ctx.flags() | AV_CODEC_FLAG_GLOBAL_HEADER); + + if ((ret = avcodec_open2(codec_ctx, codec, (PointerPointer) null)) < 0) { + logFFmpeg(ret); + throw new IllegalStateException("Error opening codec " + codec.name().getString()); + } + + headers = getExtradata(codec_ctx); + + try { + var desc = av_pix_fmt_desc_get(codec_ctx.pix_fmt()); + pixelFmt = FullPixelEnum.valueOf(desc.name().getString().toUpperCase()); + inputFormat.pixelFmt = pixelFmt; + outputFormat.pixelFmt = pixelFmt; + desc.deallocate(); + } catch (Exception e) { + pixelFmt = null; + logger.warn("Could not determine codec info for " + codec.name().getString(), e); + } + return pixelFmt; + } + + protected static byte[] getExtradata(AVCodecContext codecCtx) { + if (codecCtx.extradata() == null || codecCtx.extradata_size() == 0) + return null; + + byte[] data = new byte[codecCtx.extradata_size()]; + codecCtx.extradata().get(data); + return data; + } + + protected static void logFFmpeg(int retCode) { + BytePointer buf = new BytePointer(AV_ERROR_MAX_STRING_SIZE); + av_strerror(retCode, buf, buf.capacity()); + logger.warn("FFmpeg returned error code {}: {}", retCode, buf.getString()); + } + + protected static void setCodecPixFmt(AVCodecContext codec_ctx, FullPixelEnum desiredFmt) { + String codecString = codec_ctx.codec().name().getString(); + PointerPointer pixelFmts = new PointerPointer<>(1); + + avcodec_get_supported_config(codec_ctx, null, AV_CODEC_CONFIG_PIX_FORMAT, 0, pixelFmts, (IntPointer) null); + IntPointer fmts = pixelFmts.get(IntPointer.class, 0); + // If null, all formats are supported + if (fmts == null || fmts.isNull()) { + if (desiredFmt == FullPixelEnum.NONE) { + codec_ctx.pix_fmt(AV_PIX_FMT_YUV420P); + } else { + codec_ctx.pix_fmt(desiredFmt.ffmpegId); + } + } + else { + boolean found = false; + for (int i = 0; fmts.get(i) != AV_PIX_FMT_NONE; i++) { + if (fmts.get(i) == desiredFmt.ffmpegId) { + found = true; + codec_ctx.pix_fmt(fmts.get(i)); + break; + } + } + if (!found) { + logger.warn("Preferred pixel format for codec {} could not be found", codecString); + IntPointer loss = new IntPointer(1); + int fmt = avcodec_find_best_pix_fmt_of_list(fmts, desiredFmt.ffmpegId, 0, loss); + codec_ctx.pix_fmt(fmt); + } + } + pixelFmts.deallocate(); + } + + /** + * Set certain options in the codec context. + * Context must be allocated first using {@link #initContext()}. + */ + protected void initOptions() { + + if (options.bitRate() > 0) { + codec_ctx.bit_rate(options.bitRate() * 1000); + } else { + codec_ctx.bit_rate(0); + } + + codec_ctx.width(options.width()); + codec_ctx.height(options.height()); + + codec_ctx.time_base(av_make_q(1, 90000)); + + if (options.fps() > 0) { + codec_ctx.framerate(av_make_q(options.fps(), 1)); + } else { + codec_ctx.framerate(av_make_q(25, 1)); + } + + if (inputFormat.codec.ffmpegId == AV_CODEC_ID_H264) { + // OpenH264 only supports Baseline (66) and Main (77) + codec_ctx.profile(AV_PROFILE_H264_MAIN); + + // Enable frame skip so bitrate control works correctly, + // or it falls back to quality mode and ignores the bitrate setting + av_opt_set(codec_ctx.priv_data(), "skip_frames", "1", 0); + + // OpenH264 uses slice_mode instead of preset + av_opt_set(codec_ctx.priv_data(), "slice_mode", "auto", 0); + } + + //av_opt_set(codec_ctx.priv_data(), "preset", options.preset(), 0); + //av_opt_set(codec_ctx.priv_data(), "tune", options.tune(), 0); + codec_ctx.strict_std_compliance(options.compliance()); // Needed so that yuvj420p works (used for mjpeg, must be set to unofficial) + } + + public abstract void deallocateInputPacket(I packet); + public abstract void deallocateOutputPacket(O packet); + protected abstract O cloneOutput(O packet); + protected abstract void processInputPacket(I inputPacket); + + + public void submitInputPacket(I inputPacket) { + synchronized (contextLock) { + if (!isAlive.get()) { + return; + } + if (inputPacket != null) { + addInPacket(inputPacket); + } + if (isProcessing.compareAndSet(false, true)) { + submitTask(() -> { + // Process the input + while (!inQueue.isEmpty()) { + I inPacket = inQueue.poll(); + processInputPacket(inPacket); + deallocateInputPacket(inPacket); + } + + if (!outQueue.isEmpty() && isNotifying.compareAndSet(false, true)) { + outputExecutor.submit(() -> { + while (!outQueue.isEmpty()) { + O outputPacket = outQueue.poll(); + notifyCallbacks(outputPacket); + } + isNotifying.set(false); + }); + } + + isProcessing.set(false); + }); + } + } + } + + private void notifyCallbacks(O outputPacket) { + for (var entry: callbackMap.entrySet()) { + var clonedOutputPacket = cloneOutput(outputPacket); + entry.getValue().submit(() -> { + entry.getKey().onPacket(clonedOutputPacket); + }); + } + deallocateOutputPacket(outputPacket); + } + + public void registerCallback(CodecCallback callback) { + if (!callbackMap.containsKey(callback)) { + callbackMap.put(callback, Executors.newSingleThreadExecutor(r -> new Thread(r, "ffmpeg-codec-" + codecNum + "-callback-thread"))); + } else { + logger.warn("This callback was already registered for codec " + codecNum); + } + } + + public void unregisterCallback(CodecCallback callback) { + callbackMap.remove(callback); + } + + public void unregisterAllCallbacks() { + callbackMap.clear(); + } + + @Override + public void close() { + synchronized (contextLock) { + if (isAlive.compareAndSet(true, false)) { + + // Submit cleanup *before* shutdown so it is the last task to run + executor.submit(this::cleanup); + executor.shutdownNow(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException ignored) { + logger.warn("Interrupted while waiting for ffmpeg thread to finish"); + Thread.currentThread().interrupt(); + } + + outputExecutor.shutdownNow(); + try { + outputExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException ignored) { + logger.warn("Interrupted while waiting for ffmpeg thread to finish"); + Thread.currentThread().interrupt(); + } + + unregisterAllCallbacks(); + } + } + } + + private void cleanup() { + if (codec_ctx != null) { + avcodec_free_context(codec_ctx); + } + codec_ctx = null; + codec = null; + + unregisterAllCallbacks(); + + for (var packet : outQueue) { + deallocateOutputPacket(packet); + } + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java deleted file mode 100644 index 5cc663d6d..000000000 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Coder.java +++ /dev/null @@ -1,191 +0,0 @@ -package org.sensorhub.impl.process.video.transcoder.coders; - -import static org.bytedeco.ffmpeg.global.avutil.*; - -import java.util.ArrayDeque; -import java.util.HashMap; -import java.util.Queue; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.bytedeco.ffmpeg.global.avcodec.*; - -import org.bytedeco.ffmpeg.avcodec.AVCodecContext; -import org.bytedeco.ffmpeg.avcodec.AVPacket; -import org.bytedeco.ffmpeg.avutil.AVFrame; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class Coder extends Thread { - - protected static final Logger logger = LoggerFactory.getLogger(Coder.class); - - int codecId; - private static final int MAX_QUEUE_SIZE = 500; - protected AVCodecContext codec_ctx; - protected Queue inPackets; // ONLY allow the main loop to poll - protected final Queue outPackets; // ONLY allow the main loop to add - protected I inPacket; - protected O outPacket; - //volatile boolean isProcessing = false; // Set to true at the start of the loop, false at the end - volatile boolean isGettingPackets = false; - final Object waitingObj = new Object(); - Class inputClass; - Class outputClass; - HashMap options; - - public AtomicBoolean doRun = new AtomicBoolean(true); - - public Coder(int codecId, Class inputClass, Class outputClass, HashMap options) { - super(); - - assert inputClass == AVPacket.class || inputClass == AVFrame.class; - assert outputClass == AVPacket.class || outputClass == AVFrame.class; - assert options != null; - - this.codecId = codecId; - this.inPackets = new ArrayDeque<>(); - this.outPackets = new ArrayDeque<>(); - this.inputClass = inputClass; - this.outputClass = outputClass; - this.options = options; - } - - /** - * Gets reference to output queue. Typically used as the input queue for another {@link Coder}. - * @return Output queue containing either {@link AVPacket}s or {@link AVFrame}s. - */ - public Queue getOutQueue() { - return outPackets; - } - - /** - * Sets input queue. - * @param inPackets Typically the output queue of another {@link Coder}, containing either - * {@link AVPacket}s or {@link AVFrame}s. - */ - public void setInQueue(Queue inPackets) { - this.inPackets = (Queue) inPackets; - } - - // Safety net queue purging. - // Sometimes, queues can grow very large (like when a thread hangs up or is paused in the debugger). - // Can recover, so we don't want memory issues from too many items in the queues. - private void queuePurge() { - - synchronized (outPackets) { - if (outPackets.size() > MAX_QUEUE_SIZE) { logger.warn("Output queue is larger than max ({} > {}). Purging queue.", outPackets.size(), MAX_QUEUE_SIZE); } - while (outPackets.size() > MAX_QUEUE_SIZE) { - deallocateOutputPacket(outPackets.poll()); - } - } - } - - protected abstract void deallocateInputPacket(I packet); - - protected abstract void deallocateOutputPacket(O packet); - - protected abstract void deallocateOutQueue(); - - // To be implemented by subclasses, encoder/decoder - protected abstract void initContext(); - - // Take data from input queue and send to encoder/decoder - protected abstract void sendInPacket(); - - // Take data from encoder/decoder and send to output queue - protected abstract void receiveOutPacket(); - - // Allocate packets/frames - protected abstract void allocatePackets(); - - // Deallocate packets/frames - protected abstract void deallocatePackets(); - - /** - * Set certain options in the codec context. - * @param codec_ctx Codec context. Context must be allocated first. - */ - protected void initOptions(AVCodecContext codec_ctx) { - - codec_ctx.time_base(av_make_q(1, options.getOrDefault("fps", 30))); - - if (options.containsKey("bit_rate")) { - codec_ctx.bit_rate(options.get("bit_rate") * 1000); - } else { - //codec_ctx.bit_rate(150*1000); - } - - codec_ctx.width(options.getOrDefault("width", 1920)); - - codec_ctx.height(options.getOrDefault("height", 1080)); - - if (options.containsKey("pix_fmt")) { - codec_ctx.pix_fmt(options.get("pix_fmt")); - } else { - if (codecId == AV_CODEC_ID_MJPEG) { - codec_ctx.pix_fmt(AV_PIX_FMT_YUVJ420P); - } else { - codec_ctx.pix_fmt(AV_PIX_FMT_YUV420P); - } - } - - av_opt_set(codec_ctx.priv_data(), "preset", "ultrafast", 0); - av_opt_set(codec_ctx.priv_data(), "tune", "zerolatency", 0); - codec_ctx.strict_std_compliance(FF_COMPLIANCE_UNOFFICIAL); // Needed so that yuvj420p works (used for mjpeg) - } - - @Override - public void run() { - initContext(); - inPacket = (I)(new Object()); - outPacket = (O)(new Object()); - - allocatePackets(); - - // init FFMPEG logging - av_log_set_level(logger.isDebugEnabled() ? AV_LOG_INFO : AV_LOG_FATAL); - //av_log_set_level(AV_LOG_DEBUG); - - // Actual start of run - // Codec should be initialized by now - doRun.set(true); - - while (doRun.get() && !Thread.currentThread().isInterrupted()) { - while (inPackets == null || inPackets.isEmpty()) { - if (!doRun.get() || Thread.currentThread().isInterrupted()) { break; } - Thread.onSpinWait(); - } - while (inPackets != null && !inPackets.isEmpty()) { - if (!doRun.get() || Thread.currentThread().isInterrupted()) { break; } - - queuePurge(); - - //logger.debug("Queue size: {}", inPackets.size()); - // Get data from in queue - // Send data to encoder/decoder - //logger.debug("{}: Sending packet", this.getClass().getName()); - sendInPacket(); - - // Receive data from encoder/decoder - // Add data to out queue - //logger.debug("{}: Receiving packet", this.getClass().getName()); - receiveOutPacket(); - - //isProcessing = false; - // Wake a thread waiting for packets. - - } - } - - // End of coding - synchronized (waitingObj) { - waitingObj.notifyAll(); - } - - deallocatePackets(); - if (codec_ctx != null) { - avcodec_free_context(codec_ctx); - } - codec_ctx = null; - } -} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java index fc5709954..f8340644c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Decoder.java @@ -1,102 +1,74 @@ package org.sensorhub.impl.process.video.transcoder.coders; -import org.bytedeco.ffmpeg.avcodec.AVCodec; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; -import org.bytedeco.javacpp.PointerPointer; - -import java.util.HashMap; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; -public class Decoder extends Coder { - public Decoder(int codecId, HashMap options) { - super(codecId, AVPacket.class, AVFrame.class, options); +public class Decoder extends Codec { + + public Decoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { + super(inFormatInfo, outFormatInfo, AVPacket.class, AVFrame.class, options); } @Override protected void initContext() { - AVCodec codec = avcodec_find_decoder(codecId); - codec_ctx = avcodec_alloc_context3(codec); - - initOptions(codec_ctx); - - if (avcodec_open2(codec_ctx, codec, (PointerPointer)null) < 0) { - throw new IllegalStateException("Error initializing " + codec.name().getString() + " decoder"); + synchronized (contextLock) { + codec = avcodec_find_decoder(inputFormat.codec.ffmpegId); + codec_ctx = avcodec_alloc_context3(codec); + codec_ctx.codec_id(inputFormat.codec.ffmpegId); + //setCodecPixFmt(codec_ctx, outputFormat.pixelFmt); + //codec_ctx.pix_fmt(outputFormat.pixelFmt().ffmpegId); } } - // Get compressed packet, send to decoder - @Override - protected void sendInPacket() { - inPacket = inPackets.poll(); - //logger.debug("decode send:"); - //logger.debug(" data[0]: {}", inPacket.data()); - //logger.debug("Sent frame to encoder"); - avcodec_send_packet(codec_ctx, av_packet_clone(inPacket)); - //av_packet_free(inPacket); - } - - // Receive uncompressed frame from decoder @Override - protected void receiveOutPacket() { - synchronized (outPackets) { - while (avcodec_receive_frame(codec_ctx, outPacket) >= 0) { - //av_packet_free(inPacket); - outPackets.add(av_frame_clone(outPacket)); - //av_frame_free(outPacket); - //logger.debug("Decode Packet added"); - } + public void deallocateInputPacket(AVPacket packet) { + if (packet != null) { + av_packet_free(packet); } } @Override - protected void deallocateInputPacket(AVPacket packet) { - av_packet_free(packet); - packet = null; + public void deallocateOutputPacket(AVFrame packet) { + if (packet != null) { + av_frame_free(packet); + } } @Override - protected void deallocateOutputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + protected AVFrame cloneOutput(AVFrame packet) { + if (packet != null) { + return av_frame_clone(packet); + } else { + return null; + } } @Override - protected void deallocateOutQueue() { - outPackets.clear(); - } + protected synchronized void processInputPacket(AVPacket inputPacket) { + if (inputPacket != null && !inputPacket.isNull()) { + int ret; + if ((ret = avcodec_send_packet(codec_ctx, inputPacket)) < 0) { + //logger.warn("Error sending packet to decoder"); + //logFFmpeg(ret); + //avcodec_flush_buffers(codec_ctx); + return; + } - @Override - protected void allocatePackets() { - inPacket = new AVPacket(); - inPacket = av_packet_alloc(); - av_init_packet(inPacket); - outPacket = new AVFrame(); - outPacket = av_frame_alloc(); - } + AVFrame outputPacket = av_frame_alloc(); - @Override - protected void deallocatePackets() { - if (inPacket != null) { - av_packet_free(inPacket); - } - if (outPacket != null) { - av_frame_free(outPacket); - } - if (outPackets != null) { - for (AVFrame frame : outPackets) { - av_frame_free(frame); + while (avcodec_receive_frame(codec_ctx, outputPacket) >= 0) { + if (!outputPacket.isNull()) { + addOutPacket(outputPacket); + } + outputPacket = av_frame_alloc(); } - outPackets.clear(); - } - if (inPackets != null) { - for (AVPacket packet : inPackets) { - av_packet_free(packet); - } - inPackets.clear(); + av_frame_free(outputPacket); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java index da47fcdfc..df1666f9c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/Encoder.java @@ -1,111 +1,139 @@ package org.sensorhub.impl.process.video.transcoder.coders; -import org.bytedeco.ffmpeg.avcodec.AVCodec; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.javacpp.BytePointer; -import org.bytedeco.javacpp.PointerPointer; - -import java.util.HashMap; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecOptions; +import org.sensorhub.impl.process.video.transcoder.helpers.FullCodecEnum; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; -public class Encoder extends Coder { - long pts = 0; +public class Encoder extends Codec { + + volatile long pts = 0; + long timebaseNum, timebaseDen, framerateNum, framerateDen; + final int GOP_SIZE = 10; + volatile long frameSinceGop = 0; + final Object timestampLock = new Object(); - public Encoder(int codecId, HashMap options) { - super(codecId, AVFrame.class, AVPacket.class, options); + public Encoder(CodecInfo inFormatInfo, CodecInfo outFormatInfo, CodecOptions options) { + super(inFormatInfo, outFormatInfo, AVFrame.class, AVPacket.class, options); } + // Override so we can get the timebase @Override - protected void deallocateOutQueue() { - outPackets.clear(); + public FullPixelEnum init() { + var pixfmt = super.init(); + timebaseNum = codec_ctx.time_base().num(); + timebaseDen = codec_ctx.time_base().den(); + framerateNum = codec_ctx.framerate().num(); + framerateDen = codec_ctx.framerate().den(); + //GOP_SIZE = codec_ctx.gop_size(); + return pixfmt; } @Override protected void initContext() { - pts = 0; - AVCodec codec = avcodec_find_encoder(codecId); - codec_ctx = avcodec_alloc_context3(codec); - - initOptions(codec_ctx); - int ret = avcodec_open2(codec_ctx, codec, (PointerPointer)null); - if (ret < 0) { - BytePointer errorBuffer = new BytePointer(AV_ERROR_MAX_STRING_SIZE); - - av_strerror(ret, errorBuffer, AV_ERROR_MAX_STRING_SIZE); - logger.debug("Receive Error: {}", errorBuffer.getString()); - throw new IllegalStateException("Error initializing " + codec.name().getString() + " encoder"); - } - } + synchronized (contextLock) { + pts = 0; - @Override - protected void sendInPacket() { - inPacket = inPackets.poll(); - //pts++; - //inPacket.pts(pts); + // For H264, prefer x264 over OpenH264 — better option compatibility + if (outputFormat.codec == FullCodecEnum.H264) { + codec = avcodec_find_encoder_by_name("libx264"); + } - avcodec_send_frame(codec_ctx, av_frame_clone(inPacket)); - //av_frame_free(inPacket); + // Fall back to the default encoder for this codec ID + if (codec == null || codec.isNull()) { + codec = avcodec_find_encoder(outputFormat.codec.ffmpegId); + } - } + if (codec == null || codec.isNull()) { + throw new IllegalStateException("Could not find encoder for: " + outputFormat.codec); + } - @Override - protected void receiveOutPacket() { - /* - synchronized (outPackets) { + codec_ctx = avcodec_alloc_context3(codec); - } + if (codec_ctx == null || codec_ctx.isNull()) { + throw new IllegalStateException("Could not allocate encoder context for: " + codec.name().getString()); + } + + codec_ctx.gop_size(GOP_SIZE); + codec_ctx.max_b_frames(0); - */ - int ret = 0; - while (( ret = avcodec_receive_packet(codec_ctx, outPacket) ) >= 0) { - outPackets.add(av_packet_clone(outPacket)); + setCodecPixFmt(codec_ctx, inputFormat.pixelFmt); + + logger.debug("Using encoder: {}", codec.name().getString()); } - //BytePointer errorBuffer = new BytePointer(AV_ERROR_MAX_STRING_SIZE); - //av_strerror(ret, errorBuffer, AV_ERROR_MAX_STRING_SIZE); - //logger.debug("Receive Error: {}", errorBuffer.getString()); } @Override - protected void deallocateInputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + public void deallocateInputPacket(AVFrame packet) { + if (packet != null) { + av_frame_free(packet); + } } @Override - protected void deallocateOutputPacket(AVPacket packet) { - av_packet_free(packet); - packet = null; + public void deallocateOutputPacket(AVPacket packet) { + if (packet != null) { + av_packet_free(packet); + } } @Override - protected void allocatePackets() { - inPacket = av_frame_alloc(); - outPacket = av_packet_alloc(); - av_init_packet(outPacket); + protected AVPacket cloneOutput(AVPacket packet) { + if (packet != null) { + return av_packet_clone(packet); + } else { + return null; + } } @Override - protected void deallocatePackets() { - if (inPacket != null) { - av_frame_free(inPacket); - } - if (outPacket != null) { - av_packet_free(outPacket); - } - if (outPackets != null) { - for (AVPacket packet : outPackets) { - av_packet_free(packet); + protected synchronized void processInputPacket(AVFrame inputPacket) { + if (inputPacket != null && !inputPacket.isNull()) { + int ret; + if (inputPacket.format() != codec_ctx.pix_fmt()) { + throw new IllegalArgumentException("AVFrame pixel format: " + FullPixelEnum.fromId(inputPacket.format()) + + " incompatible with codec pixel format: " + FullPixelEnum.fromId(codec_ctx.pix_fmt())); } - outPackets.clear(); - } - if (inPackets != null) { - for (AVFrame frame : inPackets) { - av_frame_free(frame); + + if (frameSinceGop++ >= GOP_SIZE) { + frameSinceGop = 0; + inputPacket.pict_type(AV_PICTURE_TYPE_I); + inputPacket.flags(inputPacket.flags() | AVFrame.AV_FRAME_FLAG_KEY); + } + + if ((ret = avcodec_send_frame(codec_ctx, inputPacket)) < 0) { + //logger.warn("Error sending packet to encoder"); + //logFFmpeg(ret); + //avcodec_flush_buffers(codec_ctx); + return; } - inPackets.clear(); + + AVPacket outputPacket = av_packet_alloc(); + + while (avcodec_receive_packet(codec_ctx, outputPacket) >= 0) { + if (!outputPacket.isNull()) { + // Add headers if necessary/available + if (((outputPacket.flags() & AV_PKT_FLAG_KEY) != 0) && headers != null) { + int originalSize = outputPacket.size(); + byte[] original = new byte[originalSize]; + outputPacket.data().capacity(originalSize).position(0).get(original); + av_grow_packet(outputPacket, headers.length); + BytePointer dst = outputPacket.data().capacity((long) headers.length + originalSize); + dst.position(0).put(headers); + dst.position(headers.length).put(original); + } + addOutPacket(outputPacket); + } + outputPacket = av_packet_alloc(); + } + + av_packet_free(outputPacket); } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java index 3bf44c047..a87beac4c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/coders/SwScaler.java @@ -1,113 +1,90 @@ package org.sensorhub.impl.process.video.transcoder.coders; -import org.bytedeco.ffmpeg.avcodec.AVCodec; -import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.ffmpeg.swscale.SwsContext; -import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.DoublePointer; -import org.bytedeco.javacpp.PointerPointer; +import org.sensorhub.impl.process.video.transcoder.helpers.CodecInfo; +import org.sensorhub.impl.process.video.transcoder.helpers.FullPixelEnum; -import java.util.HashMap; - -import static org.bytedeco.ffmpeg.global.avcodec.*; import static org.bytedeco.ffmpeg.global.avutil.*; import static org.bytedeco.ffmpeg.global.swscale.*; -public class SwScaler extends Coder { +public class SwScaler extends Codec { long pts = 0; SwsContext swsContext; - final int inPixFmt, outPixFmt, inWidth, inHeight, outWidth, outHeight; + final int inWidth, inHeight, outWidth, outHeight; - public SwScaler(int inPixFmt, int outPixFmt, int inWidth, int inHeight, int outWidth, int outHeight) { - super(0, AVFrame.class, AVFrame.class, new HashMap()); - this.inPixFmt = inPixFmt; - this.outPixFmt = outPixFmt; + public SwScaler(CodecInfo inputFormat, CodecInfo outputFormat, int inWidth, int inHeight, int outWidth, int outHeight) { + super(inputFormat, outputFormat, AVFrame.class, AVFrame.class, null); this.inWidth = inWidth; this.inHeight = inHeight; this.outWidth = outWidth; this.outHeight = outHeight; } - @Override - protected void deallocateOutQueue() { - outPackets.clear(); - } - @Override protected void initContext() { - pts = 0; - swsContext = sws_getContext(inWidth, inHeight, inPixFmt, - outWidth, outHeight, outPixFmt, + swsContext = sws_getContext(inWidth, inHeight, inputFormat.pixelFmt.ffmpegId, + outWidth, outHeight, outputFormat.pixelFmt.ffmpegId, SWS_BICUBIC, null, null, (DoublePointer) null); - } @Override - protected void sendInPacket() { - inPacket = inPackets.poll(); - //pts++; - //inPacket.pts(pts); - sws_scale_frame(swsContext, outPacket, inPacket); - //avcodec_send_frame(codec_ctx, av_frame_clone(inPacket)); - //av_frame_free(inPacket); + protected void initOptions() {} // no options for swscaler - } + @Override + protected FullPixelEnum openContext() { + return outputFormat.pixelFmt; + } // no codec to open @Override - protected void receiveOutPacket() { - // We already have outPacket at this point (one method needed for scaling rather than two) - // Just use this to add the output to the out queue and make the buffers writable - outPackets.add(av_frame_clone(outPacket)); - av_frame_make_writable(outPacket); + public void deallocateInputPacket(AVFrame packet) { + if (packet != null) { + av_frame_free(packet); + } } @Override - protected void deallocateInputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + public void deallocateOutputPacket(AVFrame packet) { + if (packet != null) { + av_frame_free(packet); + } } @Override - protected void deallocateOutputPacket(AVFrame packet) { - av_frame_free(packet); - packet = null; + protected AVFrame cloneOutput(AVFrame packet) { + if (packet != null) { + return av_frame_clone(packet); + } else { + return null; + } } @Override - protected void allocatePackets() { - inPacket = av_frame_alloc(); + protected synchronized void processInputPacket(AVFrame inputPacket) { + if (inputPacket == null) { return; } + + // FFmpeg can be weird when it comes to decoder pixfmt. May need to change + // on the fly (usually first frame). + if (inputPacket.format() != inputFormat.pixelFmt.ffmpegId) { + inputFormat.pixelFmt = FullPixelEnum.fromId(inputPacket.format()); + initContext(); + } - outPacket = av_frame_alloc(); - outPacket.format(outPixFmt); - outPacket.width(outWidth); - outPacket.height(outHeight); - av_image_alloc(outPacket.data(), outPacket.linesize(), - outWidth, outHeight, outPixFmt, 1); + AVFrame outputPacket = av_frame_alloc(); + outputPacket.format(outputFormat.pixelFmt.ffmpegId); + outputPacket.width(outWidth); + outputPacket.height(outHeight); + av_frame_get_buffer(outputPacket, 0); + sws_scale_frame(swsContext, outputPacket, inputPacket); + addOutPacket(outputPacket); } @Override - protected void deallocatePackets() { - if (swsContext != null) { + public void close() { + synchronized (contextLock) { + super.close(); sws_freeContext(swsContext); } - if (inPacket != null) { - av_frame_free(inPacket); - } - if (outPacket != null) { - av_frame_free(outPacket); - } - if (outPackets != null) { - for (AVFrame frame : outPackets) { - av_frame_free(frame); - } - outPackets.clear(); - } - if (inPackets != null) { - for (AVFrame frame : inPackets) { - av_frame_free(frame); - } - inPackets.clear(); - } } } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java index 60227176d..c9b71f651 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/AVByteFormatter.java @@ -5,7 +5,7 @@ import org.bytedeco.javacpp.Pointer; import org.sensorhub.impl.process.video.transcoder.CodecEnum; -public abstract class AVByteFormatter { +public interface AVByteFormatter { public abstract T convertInput(byte[] inputData); diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java new file mode 100644 index 000000000..dd2e30b31 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/FrameFormatter.java @@ -0,0 +1,231 @@ +package org.sensorhub.impl.process.video.transcoder.formatters; + +import org.bytedeco.ffmpeg.avutil.*; +import org.bytedeco.ffmpeg.global.avutil; +import org.bytedeco.javacpp.BytePointer; + +import java.util.HashSet; +import java.util.Set; + +import static org.bytedeco.ffmpeg.global.avutil.*; + +// This class is intended to replace both RgbFormatter and YuvFormatter as a generic solution +public class FrameFormatter implements AVByteFormatter { + + private final int width; + private final int height; + private final int pixFmt; + private final AVPixFmtDescriptor desc; + private final int planeCount; + private final int[] planeWidths; + private final int[] planeHeights; + private final int[] planeSizes; + private final int totalSize; + boolean isPlanar; + + public FrameFormatter(int width, int height, int pixFmt) { + this.width = width; + this.height = height; + this.pixFmt = pixFmt; + this.desc = avutil.av_pix_fmt_desc_get(pixFmt); + isPlanar = (desc.flags() & AV_PIX_FMT_FLAG_PLANAR) != 0; + + if (isPlanar) { + this.planeCount = countPlanes(desc); + + planeWidths = new int[planeCount]; + planeHeights = new int[planeCount]; + planeSizes = new int[planeCount]; + + calculatePlaneSizes(desc, width, height, planeWidths, planeHeights, planeSizes); + } else { + planeCount = 1; + planeWidths = new int[1]; + planeHeights = new int[1]; + planeSizes = new int[1]; + planeWidths[0] = width; + planeHeights[0] = height; + planeSizes[0] = width * height * ((av_get_bits_per_pixel(desc) + 7) / 8); + } + totalSize = calcByteSize(planeSizes); + } + + public int getPixFmt() { + return pixFmt; + } + + public int getTotalSize() { + return totalSize; + } + + private static int calcByteSize(int[] planeSizes) { + int sum = 0; + for (int s : planeSizes) sum += s; + return sum; + } + + @Override + public AVFrame convertInput(byte[] inputData) { + AVFrame newFrame = generateFrame(); + + if (isPlanar) + setFrameDataPlanar(newFrame, inputData); + else + setFrameDataPacked(newFrame, inputData); + + return newFrame; + } + + private void setFrameDataPlanar(AVFrame newFrame, byte[] inputData) { + int offset = 0; + + for (int p = 0; p < planeCount; p++) { + int w = planeWidths[p]; + int h = planeHeights[p]; + int stride = newFrame.linesize(p); + + BytePointer dst = newFrame.data(p) + .capacity((long) stride * h) + .position(0); + + int bitsPerComponent = desc.comp(p < 1 ? 0 : 1).depth(); + int rowBytes = w * ((bitsPerComponent + 7) / 8); + + for (int y = 0; y < h; y++) { + dst.position((long) y * stride); + dst.put(inputData, offset + y * rowBytes, rowBytes); + } + + offset += planeSizes[p]; + } + } + + private void setFrameDataPacked(AVFrame newFrame, byte[] inputData) { + int linesize = newFrame.linesize(0); + BytePointer dst = newFrame.data(0) + .capacity((long) linesize * height) + .position(0); + + int bytesPerPixel = (av_get_bits_per_pixel(desc) + 7) / 8; + + int rowBytes = width * bytesPerPixel; + + int offset = 0; + + for (int y = 0; y < height; y++) { + dst.position((long) y * linesize) + .put(inputData, offset, rowBytes); + + offset += rowBytes; + } + } + + private AVFrame generateFrame() { + AVFrame newFrame = av_frame_alloc(); + newFrame.format(pixFmt); + newFrame.width(width); + newFrame.height(height); + int ret = av_frame_get_buffer(newFrame, 0); + if (ret < 0) { + av_frame_free(newFrame); + throw new IllegalStateException("Could not allocate AVFrame buffer, ffmpeg error: " + ret); + } + return newFrame; + } + + @Override + public byte[] convertOutput(AVFrame outputFrame) { + if (outputFrame.format() != pixFmt) { + throw new IllegalArgumentException( + "Unexpected frame pixel format: " + outputFrame.format() + ", expected " + pixFmt); + } + + byte[] out = new byte[totalSize]; + + if (isPlanar) { + getFrameDataPlanar(outputFrame, out); + } else { + getFrameDataPacked(outputFrame, out); + } + return out; + } + + private void getFrameDataPlanar(AVFrame outputFrame, byte[] out) { + int offset = 0; + for (int plane = 0; plane < planeCount; plane++) { + int w = planeWidth(plane); + int h = planeHeight(plane); + int srcStride = outputFrame.linesize(plane); + + BytePointer src = outputFrame.data(plane) + .capacity((long) srcStride * h) + .position(0); + + int bitsPerComponent = desc.comp(plane < 1 ? 0 : 1).depth(); + int rowBytes = w * ((bitsPerComponent + 7) / 8); + + for (int y = 0; y < h; y++) { + src.position((long) y * srcStride); + src.get(out, offset, rowBytes); + offset += rowBytes; + } + } + } + + private void getFrameDataPacked(AVFrame outputFrame, byte[] out) { + int bytesPerPixel = (av_get_bits_per_pixel(desc) + 7) / 8; + int rowBytes = width * bytesPerPixel; + int linesize = outputFrame.linesize(0); + + BytePointer src = outputFrame.data(0) + .capacity((long) linesize * height); + + int offset = 0; + for (int y = 0; y < height; y++) { + src.position((long) y * linesize) + .get(out, offset, rowBytes); + offset += rowBytes; + } + } + + private int planeWidth(int plane) { + if (plane == 0) + return width; + else + return width >> desc.log2_chroma_w(); + } + + private int planeHeight(int plane) { + if (plane == 0) + return height; + else + return height >> desc.log2_chroma_h(); + } + + // Helper functions for PLANAR formats. Untested with packed formats. + private static int countPlanes(AVPixFmtDescriptor desc) { + Set planes = new HashSet<>(); + for (int i = 0; i < desc.nb_components(); i++) { + planes.add(desc.comp(i).plane()); + } + return planes.size(); + } + + private static void calculatePlaneSizes(AVPixFmtDescriptor desc, int width, int height, int[] planeWidths, int[] planeHeights, int[] planeSizes) { + for (int c = 0; c < desc.nb_components(); c++) { + AVComponentDescriptor comp = desc.comp(c); + int p = comp.plane(); + + int shiftW = (p == 0) ? 0 : desc.log2_chroma_w(); + int shiftH = (p == 0) ? 0 : desc.log2_chroma_h(); + + planeWidths[p] = width >> shiftW; + planeHeights[p] = height >> shiftH; + } + + for (int p = 0; p < planeSizes.length; p++) { + int bytesPerSample = (desc.comp(p == 0 ? 0 : 1).depth() + 7) / 8; + planeSizes[p] = planeWidths[p] * planeHeights[p] * bytesPerSample; + } + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java index b3b09149b..46699ff2c 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/PacketFormatter.java @@ -10,7 +10,7 @@ import static org.bytedeco.ffmpeg.global.avutil.*; import static org.bytedeco.ffmpeg.global.swscale.*; -public class PacketFormatter extends AVByteFormatter { +public class PacketFormatter implements AVByteFormatter { /** * Converts an array of bytes from compressed video into an {@link AVPacket}. @@ -22,7 +22,8 @@ public AVPacket convertInput(byte[] inputData) { AVPacket newPacket = av_packet_alloc(); av_new_packet(newPacket, inputData.length); newPacket.data().position(0); - newPacket.data().put(inputData.clone(), 0, inputData.length); + newPacket.data().limit(inputData.length); + newPacket.data().put(inputData); return newPacket; } diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java index 2e0eb7fd8..5d5766c83 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/formatters/RgbFormatter.java @@ -7,7 +7,7 @@ import static org.bytedeco.ffmpeg.global.avutil.*; import static org.bytedeco.ffmpeg.global.swscale.*; -public class RgbFormatter extends AVByteFormatter { +public class RgbFormatter implements AVByteFormatter { protected final int width, height, size; protected final int pixFormat; diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java new file mode 100644 index 000000000..f4cc69bf1 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecInfo.java @@ -0,0 +1,37 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; + +public class CodecInfo implements Cloneable { + public FullCodecEnum codec; + public FullPixelEnum pixelFmt; + + public static CodecInfo newCodecInfoFromName(String name) { + FullCodecEnum codec; + FullPixelEnum pixel; + try { + codec = Enum.valueOf(FullCodecEnum.class, name); + pixel = FullPixelEnum.YUVJ420P; + } catch (Exception e) { + codec = FullCodecEnum.RAWVIDEO; + pixel = Enum.valueOf(FullPixelEnum.class, name); + } + + return new CodecInfo(codec, pixel); + } + + public CodecInfo(FullCodecEnum codec, FullPixelEnum pixelFmt) { + this.codec = codec; + this.pixelFmt = pixelFmt; + } + + @Override + public CodecInfo clone() { + try { + CodecInfo clone = (CodecInfo) super.clone(); + clone.codec = codec; + clone.pixelFmt = pixelFmt; + return clone; + } catch (CloneNotSupportedException e) { + throw new AssertionError(); + } + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java new file mode 100644 index 000000000..64e0771b3 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/CodecOptions.java @@ -0,0 +1,79 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; +import static org.bytedeco.ffmpeg.global.avcodec.*; + +public record CodecOptions(int fps, int bitRate, int width, int height, int compliance, String preset, String tune) { + public CodecOptions(int fps, int bitRate, int width, int height, int compliance, String preset, String tune) { + this.fps = Math.max(fps, 1); + this.bitRate = bitRate; + this.width = Math.max(width, 1); + this.height = Math.max(height, 1); + this.compliance = compliance; + this.preset = preset; + this.tune = tune; + } + + public static class Builder { + private int fps = 30; + private int bitRate = -1; + private int width = 1920; + private int height = 1080; + private int compliance = FF_COMPLIANCE_UNOFFICIAL; + private String preset = null; + private String tune = null; + + public CodecOptions build() { + return new CodecOptions(fps, bitRate, width, height, compliance, preset, tune); + } + + public Builder setFps(int fps) { + this.fps = fps; + return this; + } + + public Builder setBitRate(int bitRate) { + this.bitRate = bitRate; + return this; + } + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setCompliance(int compliance) { + this.compliance = compliance; + return this; + } + + // Convenience, unofficial provides widest range of pixel formats + public Builder setComplianceUnofficial() { + this.compliance = FF_COMPLIANCE_UNOFFICIAL; + return this; + } + + public Builder setPreset(String preset) { + this.preset = preset; + return this; + } + + public Builder presetUltraFast() { + this.preset = "ultrafast"; + return this; + } + + public Builder setTune(String tune) { + this.tune = tune; + return this; + } + + public Builder tuneZeroLatency() { + this.tune = "zerolatency"; + return this; + } + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java new file mode 100644 index 000000000..1db8625f6 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullCodecEnum.java @@ -0,0 +1,100 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; + +import org.bytedeco.ffmpeg.global.avcodec; + +import java.util.HashMap; +import java.util.Map; + +public enum FullCodecEnum { + H264(avcodec.AV_CODEC_ID_H264), + HEVC(avcodec.AV_CODEC_ID_HEVC), + MJPEG(avcodec.AV_CODEC_ID_MJPEG), + VP8(avcodec.AV_CODEC_ID_VP8), + VP9(avcodec.AV_CODEC_ID_VP9), + MPEG2(avcodec.AV_CODEC_ID_MPEG2VIDEO), + MPEG4(avcodec.AV_CODEC_ID_MPEG4), + AV1(avcodec.AV_CODEC_ID_AV1), + THEORA(avcodec.AV_CODEC_ID_THEORA), + MPEG1VIDEO(avcodec.AV_CODEC_ID_MPEG1VIDEO), + WMV1(avcodec.AV_CODEC_ID_WMV1), + WMV2(avcodec.AV_CODEC_ID_WMV2), + WMV3(avcodec.AV_CODEC_ID_WMV3), + VC1(avcodec.AV_CODEC_ID_VC1), + FLV1(avcodec.AV_CODEC_ID_FLV1), + FLASHSV(avcodec.AV_CODEC_ID_FLASHSV), + FLASHSV2(avcodec.AV_CODEC_ID_FLASHSV2), + RV10(avcodec.AV_CODEC_ID_RV10), + RV20(avcodec.AV_CODEC_ID_RV20), + RV30(avcodec.AV_CODEC_ID_RV30), + RV40(avcodec.AV_CODEC_ID_RV40), + CINEPAK(avcodec.AV_CODEC_ID_CINEPAK), + INDEO2(avcodec.AV_CODEC_ID_INDEO2), + INDEO3(avcodec.AV_CODEC_ID_INDEO3), + INDEO4(avcodec.AV_CODEC_ID_INDEO4), + INDEO5(avcodec.AV_CODEC_ID_INDEO5), + MSMPEG4V1(avcodec.AV_CODEC_ID_MSMPEG4V1), + MSMPEG4V2(avcodec.AV_CODEC_ID_MSMPEG4V2), + MSMPEG4V3(avcodec.AV_CODEC_ID_MSMPEG4V3), + H261(avcodec.AV_CODEC_ID_H261), + H263(avcodec.AV_CODEC_ID_H263), + H263I(avcodec.AV_CODEC_ID_H263I), + H263P(avcodec.AV_CODEC_ID_H263P), + SNOW(avcodec.AV_CODEC_ID_SNOW), + SVQ1(avcodec.AV_CODEC_ID_SVQ1), + SVQ3(avcodec.AV_CODEC_ID_SVQ3), + DVVIDEO(avcodec.AV_CODEC_ID_DVVIDEO), + HUFFYUV(avcodec.AV_CODEC_ID_HUFFYUV), + FFVHUFF(avcodec.AV_CODEC_ID_FFVHUFF), + FFV1(avcodec.AV_CODEC_ID_FFV1), + ASV1(avcodec.AV_CODEC_ID_ASV1), + ASV2(avcodec.AV_CODEC_ID_ASV2), + VCR1(avcodec.AV_CODEC_ID_VCR1), + CLJR(avcodec.AV_CODEC_ID_CLJR), + MDEC(avcodec.AV_CODEC_ID_MDEC), + ROQ(avcodec.AV_CODEC_ID_ROQ), + INTERPLAY_VIDEO(avcodec.AV_CODEC_ID_INTERPLAY_VIDEO), + XAN_WC3(avcodec.AV_CODEC_ID_XAN_WC3), + XAN_WC4(avcodec.AV_CODEC_ID_XAN_WC4), + RPZA(avcodec.AV_CODEC_ID_RPZA), + SMC(avcodec.AV_CODEC_ID_SMC), + GIF(avcodec.AV_CODEC_ID_GIF), + RAWVIDEO(avcodec.AV_CODEC_ID_RAWVIDEO), + PNG(avcodec.AV_CODEC_ID_PNG), + PPM(avcodec.AV_CODEC_ID_PPM), + PBM(avcodec.AV_CODEC_ID_PBM), + PGM(avcodec.AV_CODEC_ID_PGM), + PAM(avcodec.AV_CODEC_ID_PAM), + BMP(avcodec.AV_CODEC_ID_BMP), + TIFF(avcodec.AV_CODEC_ID_TIFF), + SGI(avcodec.AV_CODEC_ID_SGI), + ALIAS_PIX(avcodec.AV_CODEC_ID_ALIAS_PIX), + DPX(avcodec.AV_CODEC_ID_DPX), + EXR(avcodec.AV_CODEC_ID_EXR), + WEBP(avcodec.AV_CODEC_ID_WEBP), + DIRAC(avcodec.AV_CODEC_ID_DIRAC), + DNXHD(avcodec.AV_CODEC_ID_DNXHD), + PRORES(avcodec.AV_CODEC_ID_PRORES), + JPEG2000(avcodec.AV_CODEC_ID_JPEG2000), + JPEGLS(avcodec.AV_CODEC_ID_JPEGLS), + HAP(avcodec.AV_CODEC_ID_HAP); + + public int ffmpegId; + + private static final Map BY_ID; + + static { + BY_ID = new HashMap<>(); + for (FullCodecEnum fmt : values()) { + BY_ID.put(fmt.ffmpegId, fmt); + } + } + + public static FullCodecEnum fromId(int ffmpegId) { + return BY_ID.getOrDefault(ffmpegId, RAWVIDEO); + } + + FullCodecEnum(int ffmpegId) + { + this.ffmpegId = ffmpegId; + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java new file mode 100644 index 000000000..be7002944 --- /dev/null +++ b/processing/sensorhub-process-ffmpeg/src/main/java/org/sensorhub/impl/process/video/transcoder/helpers/FullPixelEnum.java @@ -0,0 +1,205 @@ +package org.sensorhub.impl.process.video.transcoder.helpers; + +import org.bytedeco.ffmpeg.global.avutil; + +import java.util.HashMap; +import java.util.Map; + +public enum FullPixelEnum { + + NONE(avutil.AV_PIX_FMT_NONE), + + // --- Planar YUV --- + YUV420P(avutil.AV_PIX_FMT_YUV420P), + YUV422P(avutil.AV_PIX_FMT_YUV422P), + YUV444P(avutil.AV_PIX_FMT_YUV444P), + YUV410P(avutil.AV_PIX_FMT_YUV410P), + YUV411P(avutil.AV_PIX_FMT_YUV411P), + YUV440P(avutil.AV_PIX_FMT_YUV440P), + YUVJ420P(avutil.AV_PIX_FMT_YUVJ420P), + YUVJ422P(avutil.AV_PIX_FMT_YUVJ422P), + YUVJ444P(avutil.AV_PIX_FMT_YUVJ444P), + YUVJ440P(avutil.AV_PIX_FMT_YUVJ440P), + YUV420P9BE(avutil.AV_PIX_FMT_YUV420P9BE), + YUV420P9LE(avutil.AV_PIX_FMT_YUV420P9LE), + YUV420P10BE(avutil.AV_PIX_FMT_YUV420P10BE), + YUV420P10LE(avutil.AV_PIX_FMT_YUV420P10LE), + YUV420P12BE(avutil.AV_PIX_FMT_YUV420P12BE), + YUV420P12LE(avutil.AV_PIX_FMT_YUV420P12LE), + YUV420P14BE(avutil.AV_PIX_FMT_YUV420P14BE), + YUV420P14LE(avutil.AV_PIX_FMT_YUV420P14LE), + YUV420P16BE(avutil.AV_PIX_FMT_YUV420P16BE), + YUV420P16LE(avutil.AV_PIX_FMT_YUV420P16LE), + YUV422P9BE(avutil.AV_PIX_FMT_YUV422P9BE), + YUV422P9LE(avutil.AV_PIX_FMT_YUV422P9LE), + YUV422P10BE(avutil.AV_PIX_FMT_YUV422P10BE), + YUV422P10LE(avutil.AV_PIX_FMT_YUV422P10LE), + YUV422P12BE(avutil.AV_PIX_FMT_YUV422P12BE), + YUV422P12LE(avutil.AV_PIX_FMT_YUV422P12LE), + YUV422P14BE(avutil.AV_PIX_FMT_YUV422P14BE), + YUV422P14LE(avutil.AV_PIX_FMT_YUV422P14LE), + YUV422P16BE(avutil.AV_PIX_FMT_YUV422P16BE), + YUV422P16LE(avutil.AV_PIX_FMT_YUV422P16LE), + YUV444P9BE(avutil.AV_PIX_FMT_YUV444P9BE), + YUV444P9LE(avutil.AV_PIX_FMT_YUV444P9LE), + YUV444P10BE(avutil.AV_PIX_FMT_YUV444P10BE), + YUV444P10LE(avutil.AV_PIX_FMT_YUV444P10LE), + YUV444P12BE(avutil.AV_PIX_FMT_YUV444P12BE), + YUV444P12LE(avutil.AV_PIX_FMT_YUV444P12LE), + YUV444P14BE(avutil.AV_PIX_FMT_YUV444P14BE), + YUV444P14LE(avutil.AV_PIX_FMT_YUV444P14LE), + YUV444P16BE(avutil.AV_PIX_FMT_YUV444P16BE), + YUV444P16LE(avutil.AV_PIX_FMT_YUV444P16LE), + + // --- Packed YUV --- + YUYV422(avutil.AV_PIX_FMT_YUYV422), + UYVY422(avutil.AV_PIX_FMT_UYVY422), + YVYU422(avutil.AV_PIX_FMT_YVYU422), + UYYVYY411(avutil.AV_PIX_FMT_UYYVYY411), + NV12(avutil.AV_PIX_FMT_NV12), + NV21(avutil.AV_PIX_FMT_NV21), + NV16(avutil.AV_PIX_FMT_NV16), + NV20LE(avutil.AV_PIX_FMT_NV20LE), + NV20BE(avutil.AV_PIX_FMT_NV20BE), + NV24(avutil.AV_PIX_FMT_NV24), + NV42(avutil.AV_PIX_FMT_NV42), + + // --- RGB --- + RGB24(avutil.AV_PIX_FMT_RGB24), + BGR24(avutil.AV_PIX_FMT_BGR24), + ARGB(avutil.AV_PIX_FMT_ARGB), + RGBA(avutil.AV_PIX_FMT_RGBA), + ABGR(avutil.AV_PIX_FMT_ABGR), + BGRA(avutil.AV_PIX_FMT_BGRA), + RGB0(avutil.AV_PIX_FMT_RGB0), + BGR0(avutil.AV_PIX_FMT_BGR0), + RGB8(avutil.AV_PIX_FMT_RGB8), + BGR8(avutil.AV_PIX_FMT_BGR8), + RGB4(avutil.AV_PIX_FMT_RGB4), + BGR4(avutil.AV_PIX_FMT_BGR4), + RGB4_BYTE(avutil.AV_PIX_FMT_RGB4_BYTE), + BGR4_BYTE(avutil.AV_PIX_FMT_BGR4_BYTE), + RGB48BE(avutil.AV_PIX_FMT_RGB48BE), + RGB48LE(avutil.AV_PIX_FMT_RGB48LE), + RGBA64BE(avutil.AV_PIX_FMT_RGBA64BE), + RGBA64LE(avutil.AV_PIX_FMT_RGBA64LE), + BGR48BE(avutil.AV_PIX_FMT_BGR48BE), + BGR48LE(avutil.AV_PIX_FMT_BGR48LE), + BGRA64BE(avutil.AV_PIX_FMT_BGRA64BE), + BGRA64LE(avutil.AV_PIX_FMT_BGRA64LE), + RGB565BE(avutil.AV_PIX_FMT_RGB565BE), + RGB565LE(avutil.AV_PIX_FMT_RGB565LE), + RGB555BE(avutil.AV_PIX_FMT_RGB555BE), + RGB555LE(avutil.AV_PIX_FMT_RGB555LE), + RGB444BE(avutil.AV_PIX_FMT_RGB444BE), + RGB444LE(avutil.AV_PIX_FMT_RGB444LE), + BGR565BE(avutil.AV_PIX_FMT_BGR565BE), + BGR565LE(avutil.AV_PIX_FMT_BGR565LE), + BGR555BE(avutil.AV_PIX_FMT_BGR555BE), + BGR555LE(avutil.AV_PIX_FMT_BGR555LE), + BGR444BE(avutil.AV_PIX_FMT_BGR444BE), + BGR444LE(avutil.AV_PIX_FMT_BGR444LE), + + // --- Grayscale --- + GRAY8(avutil.AV_PIX_FMT_GRAY8), + GRAY8A(avutil.AV_PIX_FMT_GRAY8A), + GRAY9BE(avutil.AV_PIX_FMT_GRAY9BE), + GRAY9LE(avutil.AV_PIX_FMT_GRAY9LE), + GRAY10BE(avutil.AV_PIX_FMT_GRAY10BE), + GRAY10LE(avutil.AV_PIX_FMT_GRAY10LE), + GRAY12BE(avutil.AV_PIX_FMT_GRAY12BE), + GRAY12LE(avutil.AV_PIX_FMT_GRAY12LE), + GRAY14BE(avutil.AV_PIX_FMT_GRAY14BE), + GRAY14LE(avutil.AV_PIX_FMT_GRAY14LE), + GRAY16BE(avutil.AV_PIX_FMT_GRAY16BE), + GRAY16LE(avutil.AV_PIX_FMT_GRAY16LE), + MONOWHITE(avutil.AV_PIX_FMT_MONOWHITE), + MONOBLACK(avutil.AV_PIX_FMT_MONOBLACK), + + // --- YA --- + YA8(avutil.AV_PIX_FMT_YA8), + YA16BE(avutil.AV_PIX_FMT_YA16BE), + YA16LE(avutil.AV_PIX_FMT_YA16LE), + + // --- YUVA --- + YUVA420P(avutil.AV_PIX_FMT_YUVA420P), + YUVA422P(avutil.AV_PIX_FMT_YUVA422P), + YUVA444P(avutil.AV_PIX_FMT_YUVA444P), + YUVA420P9BE(avutil.AV_PIX_FMT_YUVA420P9BE), + YUVA420P9LE(avutil.AV_PIX_FMT_YUVA420P9LE), + YUVA422P9BE(avutil.AV_PIX_FMT_YUVA422P9BE), + YUVA422P9LE(avutil.AV_PIX_FMT_YUVA422P9LE), + YUVA444P9BE(avutil.AV_PIX_FMT_YUVA444P9BE), + YUVA444P9LE(avutil.AV_PIX_FMT_YUVA444P9LE), + YUVA420P10BE(avutil.AV_PIX_FMT_YUVA420P10BE), + YUVA420P10LE(avutil.AV_PIX_FMT_YUVA420P10LE), + YUVA422P10BE(avutil.AV_PIX_FMT_YUVA422P10BE), + YUVA422P10LE(avutil.AV_PIX_FMT_YUVA422P10LE), + YUVA444P10BE(avutil.AV_PIX_FMT_YUVA444P10BE), + YUVA444P10LE(avutil.AV_PIX_FMT_YUVA444P10LE), + YUVA420P16BE(avutil.AV_PIX_FMT_YUVA420P16BE), + YUVA420P16LE(avutil.AV_PIX_FMT_YUVA420P16LE), + YUVA422P16BE(avutil.AV_PIX_FMT_YUVA422P16BE), + YUVA422P16LE(avutil.AV_PIX_FMT_YUVA422P16LE), + YUVA444P16BE(avutil.AV_PIX_FMT_YUVA444P16BE), + YUVA444P16LE(avutil.AV_PIX_FMT_YUVA444P16LE), + + // --- Bayer --- + BAYER_BGGR8(avutil.AV_PIX_FMT_BAYER_BGGR8), + BAYER_RGGB8(avutil.AV_PIX_FMT_BAYER_RGGB8), + BAYER_GBRG8(avutil.AV_PIX_FMT_BAYER_GBRG8), + BAYER_GRBG8(avutil.AV_PIX_FMT_BAYER_GRBG8), + BAYER_BGGR16LE(avutil.AV_PIX_FMT_BAYER_BGGR16LE), + BAYER_BGGR16BE(avutil.AV_PIX_FMT_BAYER_BGGR16BE), + BAYER_RGGB16LE(avutil.AV_PIX_FMT_BAYER_RGGB16LE), + BAYER_RGGB16BE(avutil.AV_PIX_FMT_BAYER_RGGB16BE), + BAYER_GBRG16LE(avutil.AV_PIX_FMT_BAYER_GBRG16LE), + BAYER_GBRG16BE(avutil.AV_PIX_FMT_BAYER_GBRG16BE), + BAYER_GRBG16LE(avutil.AV_PIX_FMT_BAYER_GRBG16LE), + BAYER_GRBG16BE(avutil.AV_PIX_FMT_BAYER_GRBG16BE), + + // --- Floating Point --- + GRAYF32BE(avutil.AV_PIX_FMT_GRAYF32BE), + GRAYF32LE(avutil.AV_PIX_FMT_GRAYF32LE), + + // --- Palette --- + PAL8(avutil.AV_PIX_FMT_PAL8), + + // --- XYZ --- + XYZ12LE(avutil.AV_PIX_FMT_XYZ12LE), + XYZ12BE(avutil.AV_PIX_FMT_XYZ12BE), + + // --- Miscellaneous --- + XTOP(avutil.AV_PIX_FMT_X2RGB10LE), + P010LE(avutil.AV_PIX_FMT_P010LE), + P010BE(avutil.AV_PIX_FMT_P010BE), + P016LE(avutil.AV_PIX_FMT_P016LE), + P016BE(avutil.AV_PIX_FMT_P016BE), + P210BE(avutil.AV_PIX_FMT_P210BE), + P210LE(avutil.AV_PIX_FMT_P210LE), + P410BE(avutil.AV_PIX_FMT_P410BE), + P410LE(avutil.AV_PIX_FMT_P410LE), + P216BE(avutil.AV_PIX_FMT_P216BE), + P216LE(avutil.AV_PIX_FMT_P216LE), + P416BE(avutil.AV_PIX_FMT_P416BE), + P416LE(avutil.AV_PIX_FMT_P416LE); + + public int ffmpegId; + private static final Map BY_ID; + + static { + BY_ID = new HashMap<>(); + for (FullPixelEnum fmt : values()) { + BY_ID.put(fmt.ffmpegId, fmt); + } + } + + public static FullPixelEnum fromId(int ffmpegId) { + return BY_ID.getOrDefault(ffmpegId, NONE); + } + + FullPixelEnum(int ffmpegId) + { + this.ffmpegId = ffmpegId; + } +} diff --git a/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md b/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md index 3ba1e0c72..fd89f33ab 100644 --- a/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md +++ b/processing/sensorhub-process-ffmpeg/src/main/resources/org/sensorhub/impl/process/video/transcoder/README.md @@ -2,11 +2,12 @@ ## Overview -This module provides a process module that will can decode and encode video. +FFmpeg video decode/encode/transcode module. +Input and output may be compressed or uncompressed video. ## Configuration -When added to an OpenSensorHub node, the process has the following configuration properties: +When added to an OpenSensorHub node, the process has the following configuration options: - **General:** - **Module ID:** *Not editable.* @@ -24,17 +25,185 @@ When added to an OpenSensorHub node, the process has the following configuration - **Video Source:** A module with video output. Once the transcoder starts, video from this source module will be decoded/encoded and outputted from the transcoder. - - **Input Codec:** (Optional) - The codec used for decoding the incoming video data. If incoming video is uncompressed, + - **Input Format:** (Optional) + The format used for decoding the incoming video data. If incoming video is uncompressed, select either RGB or YUV. - - **Output Codec:** - The codec used for encoding the outgoing video data. If outgoing video should be uncompressed, + - **Input Format Override:** (Optional) + Manually specify any input codec or pixel format, allowing for any format not available in the short Input Codec list. + Only required if the format is not in the Input Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. + - **Output Format:** + The format used for encoding the outgoing video data. If outgoing video should be uncompressed, select either RGB or YUV. + - **Output Format Override:** (Optional) + Manually specify any output codec or pixel format, allowing for any format not available in the short Output Codec list. + Only required if the format is not in the Output Format list. Otherwise, leave blank. + A full list of codecs and pixel formats is available at the end of this file. - **Output Width:** (Optional) The width of the output video frame. Leave this empty to avoid scaling the video frame size. - **Output Height:** (Optional) The height of the output video frame. Leave this empty to avoid scaling the video frame size. - **Auto Start:** If checked, automatically start this sensor when the OpenSensorHub node is launched. - - **Automatically Detect Input Codec:** - If checked, automatically determine the input video codec based on the input's encoding information. \ No newline at end of file + - **Automatically Detect Input Format:** + If checked, automatically determine the input video format based on the input's encoding information. + +## All Supported Formats +This list contains all the supported codecs and pixel formats. To use a format, copy it into one of the format +override config fields. + +| Codecs (Compressed Video) | Pixel Formats (Uncompressed Video) | +|---------------------------|------------------------------------| +| H264 | YUV420P | +| HEVC | YUV422P | +| MJPEG | YUV444P | +| VP8 | YUV410P | +| VP9 | YUV411P | +| MPEG2 | YUV440P | +| MPEG4 | YUVJ420P | +| AV1 | YUVJ422P | +| THEORA | YUVJ444P | +| MPEG1VIDEO | YUVJ440P | +| WMV1 | YUV420P9BE | +| WMV2 | YUV420P9LE | +| WMV3 | YUV420P10BE | +| VC1 | YUV420P10LE | +| FLV1 | YUV420P12BE | +| FLASHSV | YUV420P12LE | +| FLASHSV2 | YUV420P14BE | +| RV10 | YUV420P14LE | +| RV20 | YUV420P16BE | +| RV30 | YUV420P16LE | +| RV40 | YUV422P9BE | +| CINEPAK | YUV422P9LE | +| INDEO2 | YUV422P10BE | +| INDEO3 | YUV422P10LE | +| INDEO4 | YUV422P12BE | +| INDEO5 | YUV422P12LE | +| MSMPEG4V1 | YUV422P14BE | +| MSMPEG4V2 | YUV422P14LE | +| MSMPEG4V3 | YUV422P16BE | +| H261 | YUV422P16LE | +| H263 | YUV444P9BE | +| H263I | YUV444P9LE | +| H263P | YUV444P10BE | +| SNOW | YUV444P10LE | +| SVQ1 | YUV444P12BE | +| SVQ3 | YUV444P12LE | +| DVVIDEO | YUV444P14BE | +| HUFFYUV | YUV444P14LE | +| FFVHUFF | YUV444P16BE | +| FFV1 | YUV444P16LE | +| ASV1 | YUYV422 | +| ASV2 | UYVY422 | +| VCR1 | YVYU422 | +| CLJR | UYYVYY411 | +| MDEC | NV12 | +| ROQ | NV21 | +| INTERPLAY_VIDEO | NV16 | +| XAN_WC3 | NV20LE | +| XAN_WC4 | NV20BE | +| RPZA | NV24 | +| SMC | NV42 | +| GIF | RGB24 | +| PNG | BGR24 | +| PPM | ARGB | +| PBM | RGBA | +| PGM | ABGR | +| PAM | BGRA | +| BMP | RGB0 | +| TIFF | BGR0 | +| SGI | RGB8 | +| ALIAS_PIX | BGR8 | +| DPX | RGB4 | +| EXR | BGR4 | +| WEBP | RGB4_BYTE | +| DIRAC | BGR4_BYTE | +| DNXHD | RGB48BE | +| PRORES | RGB48LE | +| JPEG2000 | RGBA64BE | +| JPEGLS | RGBA64LE | +| HAP | BGR48BE | +| | BGR48LE | +| | BGRA64BE | +| | BGRA64LE | +| | RGB565BE | +| | RGB565LE | +| | RGB555BE | +| | RGB555LE | +| | RGB444BE | +| | RGB444LE | +| | BGR565BE | +| | BGR565LE | +| | BGR555BE | +| | BGR555LE | +| | BGR444BE | +| | BGR444LE | +| | GRAY8 | +| | GRAY8A | +| | GRAY9BE | +| | GRAY9LE | +| | GRAY10BE | +| | GRAY10LE | +| | GRAY12BE | +| | GRAY12LE | +| | GRAY14BE | +| | GRAY14LE | +| | GRAY16BE | +| | GRAY16LE | +| | MONOWHITE | +| | MONOBLACK | +| | YA8 | +| | YA16BE | +| | YA16LE | +| | YUVA420P | +| | YUVA422P | +| | YUVA444P | +| | YUVA420P9BE | +| | YUVA420P9LE | +| | YUVA422P9BE | +| | YUVA422P9LE | +| | YUVA444P9BE | +| | YUVA444P9LE | +| | YUVA420P10BE | +| | YUVA420P10LE | +| | YUVA422P10BE | +| | YUVA422P10LE | +| | YUVA444P10BE | +| | YUVA444P10LE | +| | YUVA420P16BE | +| | YUVA420P16LE | +| | YUVA422P16BE | +| | YUVA422P16LE | +| | YUVA444P16BE | +| | YUVA444P16LE | +| | BAYER_BGGR8 | +| | BAYER_RGGB8 | +| | BAYER_GBRG8 | +| | BAYER_GRBG8 | +| | BAYER_BGGR16LE | +| | BAYER_BGGR16BE | +| | BAYER_RGGB16LE | +| | BAYER_RGGB16BE | +| | BAYER_GBRG16LE | +| | BAYER_GBRG16BE | +| | BAYER_GRBG16LE | +| | BAYER_GRBG16BE | +| | GRAYF32BE | +| | GRAYF32LE | +| | PAL8 | +| | XYZ12LE | +| | XYZ12BE | +| | XTOP | +| | P010LE | +| | P010BE | +| | P016LE | +| | P016BE | +| | P210BE | +| | P210LE | +| | P410BE | +| | P410LE | +| | P216BE | +| | P216LE | +| | P416BE | +| | P416LE | \ No newline at end of file diff --git a/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java b/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java index 1825381dd..11359cc00 100644 --- a/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java +++ b/services/sensorhub-service-video/src/main/java/org/sensorhub/impl/service/sos/video/MP4Serializer.java @@ -72,7 +72,7 @@ public void sendNextFrame(DataBlock nextFrame, OutputStream os) throws IOExcepti ByteBuffer nals = ByteBuffer.wrap(frameData); // debug - //os.write(frameData); + os.write(frameData); //os.flush(); // look for next nal unit