From 9d9882dd84c9422ff24072fa4caf4b986310e9c5 Mon Sep 17 00:00:00 2001 From: peerless2012 Date: Wed, 10 Jun 2026 16:46:17 +0800 Subject: [PATCH] Add maxRenderPixels for subtitle render downscaling Add AssHandlerConfig.maxRenderPixels to limit subtitle render resolution. When frame size exceeds this pixel count, rendering is proportionally downscaled and then scaled back up during display. This reduces CPU and memory usage on high-resolution devices (e.g., 4K TVs). Changes: - AssHandlerConfig: add maxRenderPixels parameter (default 0 = no limit) - AssHandler: add computeRenderSize() utility, apply to all setFrameSize calls, log downscaling when triggered from onSurfaceSizeChanged - AssSubtitleTextureView: scale glViewport coordinates from renderSize to surfaceSize - AssSubtitleCanvasView: scale drawBitmap with RectF when downscaled - AssTexOverlay: create smaller FBO, override getVertexTransformation to scale up in OverlayEffect pipeline - AssCanvasOverlay: scale canvas draw coordinates - AssPlayer.kt: expose config parameter in buildWithAssSupport() - README: document configuration and downscaling behavior Refs #27 --- .../peerless2012/ass/demo/MainActivity.kt | 2 + lib_ass_media/README.md | 58 +++++++++++++++++++ .../peerless2012/ass/media/AssHandler.kt | 43 ++++++++++++-- .../ass/media/AssHandlerConfig.kt | 22 ++++++- .../peerless2012/ass/media/kt/AssPlayer.kt | 4 +- .../ass/media/render/AssCanvasOverlay.kt | 22 ++++++- .../ass/media/render/AssTexOverlay.kt | 30 ++++++++-- .../ass/media/widget/AssSubtitleCanvasView.kt | 21 ++++++- .../media/widget/AssSubtitleTextureView.kt | 14 ++++- 9 files changed, 197 insertions(+), 19 deletions(-) diff --git a/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt b/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt index 1f7ebf0..1c4dfe3 100644 --- a/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt +++ b/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList import io.github.peerless2012.ass.media.kt.buildWithAssSupport import io.github.peerless2012.ass.media.type.AssRenderType import androidx.core.net.toUri +import io.github.peerless2012.ass.media.AssHandlerConfig class MainActivity : AppCompatActivity() { @@ -51,6 +52,7 @@ class MainActivity : AppCompatActivity() { .buildWithAssSupport( this, AssRenderType.OVERLAY_OPEN_GL, + AssHandlerConfig(maxRenderPixels = 720*480), playerView.subtitleView ) playerView.player = player diff --git a/lib_ass_media/README.md b/lib_ass_media/README.md index bfad0e1..2938aad 100644 --- a/lib_ass_media/README.md +++ b/lib_ass_media/README.md @@ -102,3 +102,61 @@ And the `libass` render and `OpenGL` draw on another separate thread, it will no .setSubtitleConfigurations(ImmutableList.of(enConfig, jpConfig, zhConfig)) ``` NOTE: Make sure the `id` is set and different from media self track size. Recommend bigger than 128 or more bigger. + +## Configuration + +`AssHandlerConfig` provides options to tune subtitle rendering behavior. Pass it to `buildWithAssSupport`: + +```kotlin +player = ExoPlayer.Builder(this) + .buildWithAssSupport( + context = this, + renderType = AssRenderType.OVERLAY_OPEN_GL, + config = AssHandlerConfig( + maxRenderPixels = 1920 * 1080 + ) + ) +``` + +### Parameters + +| Parameter | Default | Description | +| :----: | :----: | :---- | +| `glyphSize` | 10000 | Maximum number of glyph cache entries in libass | +| `cacheSize` | 128 | Maximum bitmap cache size in MB for libass | +| `maxRenderPixels` | 0 | Maximum pixel count for subtitle rendering (0 = no limit) | + +### Render Downscaling (`maxRenderPixels`) + +On high-resolution devices (e.g., 4K TVs), rendering subtitles at full resolution can be very CPU and memory intensive, especially for complex ASS animations. `maxRenderPixels` limits the resolution at which libass renders subtitles. + +**How it works:** + +``` +Target frame size: 3840 x 2160 = 8,294,400 pixels +maxRenderPixels: 1920 x 1080 = 2,073,600 + +scale = sqrt(2,073,600 / 8,294,400) = 0.5 +Actual render size: 1920 x 1080 +``` + +1. libass renders subtitle bitmaps at the downscaled size (e.g., 1080p) +2. The rendered subtitle images are then scaled up to the actual surface/video size during display +3. GPU hardware handles the upscaling, which is nearly free + +**Recommended values:** + +| Value | Equivalent | Use case | +| :---- | :---- | :---- | +| `0` | No limit | Default, render at full resolution | +| `2_073_600` | 1080p | 4K devices with normal CPU | +| `3_686_400` | 1440p | 4K devices with strong CPU | +| `921_600` | 720p | Low-end devices or complex ASS animations | + +**Mode-specific behavior:** + +- **OVERLAY_OPEN_GL**: Renders at reduced size, scales `glViewport` coordinates to surface size +- **OVERLAY_CANVAS**: Renders at reduced size, uses `drawBitmap` with scaled `RectF` +- **EFFECTS_OPEN_GL**: Creates smaller FBO, uses `getVertexTransformation` to scale up in the OverlayEffect pipeline +- **EFFECTS_CANVAS**: Renders at reduced size, scales canvas draw coordinates +- **CUES**: Not affected (pre-renders all subtitles at parse time) diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt index b0e9779..159034d 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt @@ -169,9 +169,11 @@ class AssHandler( val render = requireNotNull(render) render.setStorageSize(videoSize.width, videoSize.height) if (renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) { - render.setFrameSize(surfaceSize.width, surfaceSize.height) + val renderSize = computeRenderSize(surfaceSize.width, surfaceSize.height) + render.setFrameSize(renderSize.width, renderSize.height) } else { - render.setFrameSize(videoSize.width, videoSize.height) + val renderSize = computeRenderSize(videoSize.width, videoSize.height) + render.setFrameSize(renderSize.width, renderSize.height) } render.setTrack(track) @@ -193,7 +195,11 @@ class AssHandler( if (surfaceSize.width == width && surfaceSize.height == height) return surfaceSize = Size(width, height) if ((renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) && surfaceSize.isValid) { - render?.setFrameSize(surfaceSize.width, surfaceSize.height) + val renderSize = computeRenderSize(width, height) + if (renderSize.width != width || renderSize.height != height) { + Log.i("AssHandler", "Downscaling render: ${width}x${height} -> ${renderSize.width}x${renderSize.height} (maxPixels=${config.maxRenderPixels})") + } + render?.setFrameSize(renderSize.width, renderSize.height) } } @@ -277,15 +283,18 @@ class AssHandler( render.setStorageSize(videoSize.width, videoSize.height) } if (videoSize.isValid) { - render.setFrameSize(videoSize.width, videoSize.height) + val renderSize = computeRenderSize(videoSize.width, videoSize.height) + render.setFrameSize(renderSize.width, renderSize.height) } if (renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) { if (surfaceSize.isValid) { - render.setFrameSize(surfaceSize.width, surfaceSize.height) + val renderSize = computeRenderSize(surfaceSize.width, surfaceSize.height) + render.setFrameSize(renderSize.width, renderSize.height) } } else { if (videoSize.isValid) { - render.setFrameSize(videoSize.width, videoSize.height) + val renderSize = computeRenderSize(videoSize.width, videoSize.height) + render.setFrameSize(renderSize.width, renderSize.height) } } Log.i("AssHandler", "Ass cacheSize: ${config.cacheSize}MB") @@ -367,4 +376,26 @@ class AssHandler( */ private val Size.isValid get() = width > 0 && height > 0 + + /** + * Computes the actual render size, downscaling proportionally if the frame + * exceeds [AssHandlerConfig.maxRenderPixels]. + * + * The result is aligned to even numbers for consistent libass internal layout. + * + * @param width Target frame width (surface or video size) + * @param height Target frame height + * @return The (possibly downscaled) render size + */ + fun computeRenderSize(width: Int, height: Int): Size { + val max = config.maxRenderPixels + if (max <= 0) return Size(width, height) + val pixels = width.toLong() * height + if (pixels <= max) return Size(width, height) + val scale = Math.sqrt(max.toDouble() / pixels).toFloat() + // Align to even numbers for libass internal layout + val w = ((width * scale).toInt() and 0x7FFFFFFE).coerceAtLeast(2) + val h = ((height * scale).toInt() and 0x7FFFFFFE).coerceAtLeast(2) + return Size(w, h) + } } \ No newline at end of file diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt index 58656c1..71ca5c0 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt @@ -2,5 +2,23 @@ package io.github.peerless2012.ass.media data class AssHandlerConfig( val glyphSize: Int = 10000, - val cacheSize: Int = 128){ -} + val cacheSize: Int = 128, + + /** + * Maximum number of pixels (width * height) for subtitle rendering. + * + * When the target frame size exceeds this limit, the render size will be + * proportionally downscaled while maintaining the aspect ratio. + * + * This reduces CPU and memory usage on high-resolution displays (e.g., 4K TVs) + * at the cost of slightly lower subtitle sharpness. + * + * Examples: + * - 1920 * 1080 = 2_073_600 (limit to 1080p) + * - 2560 * 1440 = 3_686_400 (limit to 1440p) + * - 0 = no limit, render at full frame size (default) + * + * Only applies to OVERLAY and EFFECTS render types. CUES mode is not affected. + */ + val maxRenderPixels: Int = 0 +) diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/kt/AssPlayer.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/kt/AssPlayer.kt index d096250..98bdd7f 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/kt/AssPlayer.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/kt/AssPlayer.kt @@ -14,6 +14,7 @@ import androidx.media3.extractor.ExtractorsFactory import androidx.media3.extractor.mkv.MatroskaExtractor import androidx.media3.ui.SubtitleView import io.github.peerless2012.ass.media.AssHandler +import io.github.peerless2012.ass.media.AssHandlerConfig import io.github.peerless2012.ass.media.extractor.AssMatroskaExtractor import io.github.peerless2012.ass.media.factory.AssRenderersFactory import io.github.peerless2012.ass.media.parser.AssSubtitleParserFactory @@ -24,12 +25,13 @@ import io.github.peerless2012.ass.media.widget.AssSubtitleView fun ExoPlayer.Builder.buildWithAssSupport( context: Context, renderType: AssRenderType = AssRenderType.CUES, + config: AssHandlerConfig = AssHandlerConfig(), subtitleView: SubtitleView? = null, dataSourceFactory: DataSource.Factory = DefaultDataSource.Factory(context), extractorsFactory: ExtractorsFactory = DefaultExtractorsFactory(), renderersFactory: RenderersFactory = DefaultRenderersFactory(context) ): ExoPlayer { - val assHandler = AssHandler(renderType) + val assHandler = AssHandler(renderType, config) val assSubtitleParserFactory = AssSubtitleParserFactory(assHandler) val mediaSourceFactory = DefaultMediaSourceFactory( diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssCanvasOverlay.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssCanvasOverlay.kt index af84fd6..c5e7211 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssCanvasOverlay.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssCanvasOverlay.kt @@ -26,10 +26,16 @@ class AssCanvasOverlay(private val handler: AssHandler, private val render: AssR private var texDirty = true + private var renderSize = Size.ZERO + + private var videoSize = Size.ZERO + override fun configure(videoSize: Size) { super.configure(videoSize) + this.videoSize = videoSize + renderSize = handler.computeRenderSize(videoSize.width, videoSize.height) executor = AssExecutor(render) - render.setFrameSize(videoSize.width, videoSize.height) + render.setFrameSize(renderSize.width, renderSize.height) } override fun onDraw(canvas: Canvas, presentationTimeUs: Long) { @@ -53,6 +59,8 @@ class AssCanvasOverlay(private val handler: AssHandler, private val render: AssR assFrame?.images?.let { frames -> texDirty = true + val scaleX = videoSize.width.toFloat() / renderSize.width + val scaleY = videoSize.height.toFloat() / renderSize.height frames.forEach { frame -> frame.bitmap?.let { bitmap -> val r = frame.color shr 24 and 0xFF @@ -62,7 +70,17 @@ class AssCanvasOverlay(private val handler: AssHandler, private val render: AssR val color = (a shl 24) or (r shl 16) or (g shl 8) or b paint.color = color - canvas.drawBitmap(bitmap, frame.x.toFloat(), frame.y.toFloat(), paint) + if (scaleX == 1f && scaleY == 1f) { + canvas.drawBitmap(bitmap, frame.x.toFloat(), frame.y.toFloat(), paint) + } else { + val dst = android.graphics.RectF( + frame.x * scaleX, + frame.y * scaleY, + (frame.x + bitmap.width) * scaleX, + (frame.y + bitmap.height) * scaleY + ) + canvas.drawBitmap(bitmap, null, dst, paint) + } } } } diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssTexOverlay.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssTexOverlay.kt index 9d8f517..90ec7fe 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssTexOverlay.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssTexOverlay.kt @@ -1,6 +1,7 @@ package io.github.peerless2012.ass.media.render import android.opengl.GLES20 +import android.opengl.Matrix import androidx.annotation.OptIn import androidx.media3.common.util.GlProgram import androidx.media3.common.util.GlUtil @@ -68,6 +69,10 @@ class AssTexOverlay(private val handler: AssHandler, private val render: AssRend private var texSize = Size.ZERO + private var renderSize = Size.ZERO + + private var vertexTransformMatrix = GlUtil.create4x4IdentityMatrix() + private var fboId = 0 private lateinit var glProgram: GlProgram @@ -147,7 +152,7 @@ class AssTexOverlay(private val handler: AssHandler, private val render: AssRend GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, txt) GlUtil.checkGlError() - GLES20.glViewport(frame.x, texSize.height - bitmap.height - frame.y, bitmap.width, bitmap.height) + GLES20.glViewport(frame.x, renderSize.height - bitmap.height - frame.y, bitmap.width, bitmap.height) GLES20.glUniform4f(glProgram.getUniformLocation("u_Color"), r / 255f, g / 255f, b / 255f, a / 255f) GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) @@ -173,13 +178,30 @@ class AssTexOverlay(private val handler: AssHandler, private val render: AssRend return texSize } + override fun getVertexTransformation(presentationTimeUs: Long): FloatArray { + return vertexTransformMatrix + } + override fun configure(videoSize: Size) { super.configure(videoSize) - this.texSize = videoSize + renderSize = handler.computeRenderSize(videoSize.width, videoSize.height) + this.texSize = renderSize executor = AssExecutor(render) - render.setFrameSize(videoSize.width, videoSize.height) - texId = GlUtil.createTexture(videoSize.width, videoSize.height, false) + render.setFrameSize(renderSize.width, renderSize.height) + texId = GlUtil.createTexture(renderSize.width, renderSize.height, false) fboId = GlUtil.createFboForTexture(texId) + + // Compute vertex transform to scale the overlay texture up to cover the full video frame + if (renderSize.width != videoSize.width || renderSize.height != videoSize.height) { + vertexTransformMatrix = GlUtil.create4x4IdentityMatrix() + Matrix.scaleM( + vertexTransformMatrix, 0, + videoSize.width.toFloat() / renderSize.width, + videoSize.height.toFloat() / renderSize.height, + 1f + ) + } + glProgram = GlProgram(vertexShaderCode, fragmentShaderCode) GlUtil.checkGlError() diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleCanvasView.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleCanvasView.kt index 3a34d99..e3d7b8b 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleCanvasView.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleCanvasView.kt @@ -5,8 +5,10 @@ import android.graphics.Canvas import android.graphics.Paint import android.graphics.PorterDuff import android.graphics.PorterDuffXfermode +import android.graphics.RectF import android.util.AttributeSet import android.view.View +import androidx.media3.common.util.Size import io.github.peerless2012.ass.AssFrame import io.github.peerless2012.ass.AssTexType import io.github.peerless2012.ass.media.AssHandler @@ -31,6 +33,8 @@ class AssSubtitleCanvasView : View, AssSubtitleRender { private var assFrame: AssFrame? = null + private var renderSize = Size.ZERO + // Use a local param, avoid create each time. private val invalidateCallback = Runnable { invalidate() } @@ -66,7 +70,8 @@ class AssSubtitleCanvasView : View, AssSubtitleRender { override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) { super.onSizeChanged(w, h, oldw, oldh) if (w > 0 && h > 0) { - assHandler.render?.setFrameSize(w, h) + renderSize = assHandler.computeRenderSize(w, h) + assHandler.render?.setFrameSize(renderSize.width, renderSize.height) } } @@ -92,6 +97,8 @@ class AssSubtitleCanvasView : View, AssSubtitleRender { override fun onDraw(canvas: Canvas) { super.onDraw(canvas) assFrame?.images?.let { frames -> + val scaleX = if (renderSize.width > 0) width.toFloat() / renderSize.width else 1f + val scaleY = if (renderSize.height > 0) height.toFloat() / renderSize.height else 1f frames.forEach { frame -> frame.bitmap?.let { bitmap -> val r = frame.color shr 24 and 0xFF @@ -100,7 +107,17 @@ class AssSubtitleCanvasView : View, AssSubtitleRender { val a = 0xFF - frame.color and 0xFF val color = (a shl 24) or (r shl 16) or (g shl 8) or b paint.color = color - canvas.drawBitmap(bitmap, frame.x.toFloat(), frame.y.toFloat(), paint) + if (scaleX == 1f && scaleY == 1f) { + canvas.drawBitmap(bitmap, frame.x.toFloat(), frame.y.toFloat(), paint) + } else { + val dst = RectF( + frame.x * scaleX, + frame.y * scaleY, + (frame.x + bitmap.width) * scaleX, + (frame.y + bitmap.height) * scaleY + ) + canvas.drawBitmap(bitmap, null, dst, paint) + } } } } diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleTextureView.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleTextureView.kt index 2b6b681..77883c4 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleTextureView.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/widget/AssSubtitleTextureView.kt @@ -247,6 +247,8 @@ class AssSubtitleTextureView : TextureView, AssSubtitleRender, TextureView.Surfa private var surfaceSize = Size.ZERO + private var renderSize = Size.ZERO + private lateinit var glProgram: GlProgram private var vertexBufferId = 0 @@ -306,7 +308,8 @@ class AssSubtitleTextureView : TextureView, AssSubtitleRender, TextureView.Surfa override fun onSurfaceChanged(width: Int, height: Int) { surfaceSize = Size(width, height) - assHandler.render?.setFrameSize(width, height) + renderSize = assHandler.computeRenderSize(width, height) + assHandler.render?.setFrameSize(renderSize.width, renderSize.height) GLES20.glViewport(0, 0, width, height) forceNextRender = true } @@ -362,7 +365,14 @@ class AssSubtitleTextureView : TextureView, AssSubtitleRender, TextureView.Surfa GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId) GlUtil.checkGlError() - GLES20.glViewport(frame.x, surfaceSize.height - frame.y - frame.h, frame.w, frame.h) + // Scale coordinates from render size to surface size + val scaleX = surfaceSize.width.toFloat() / renderSize.width + val scaleY = surfaceSize.height.toFloat() / renderSize.height + val scaledX = (frame.x * scaleX).toInt() + val scaledY = (frame.y * scaleY).toInt() + val scaledW = (frame.w * scaleX).toInt() + val scaledH = (frame.h * scaleY).toInt() + GLES20.glViewport(scaledX, surfaceSize.height - scaledY - scaledH, scaledW, scaledH) GLES20.glUniform4f(glProgram.getUniformLocation("u_Color"), r / 255f, g / 255f, b / 255f, a / 255f) GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)