media: mtk-vcodec: vdec: clamp OUTPUT resolution to hardware limits

Calling S_FMT or TRY_FMT on the OUTPUT queue should adjust the
resolution to the limits supported by the hardware. Until now this was
only done on the CAPTURE queue, which could make clients believe that
unsupported resolutions can be used when they set the coded size on the
OUTPUT queue.

In the case of the stateless decoder, the problem was even bigger since
subsequently calling G_FMT on the CAPTURE queue would result in the
unclamped resolution being returned, further inducing the client into
error.

Signed-off-by: Alexandre Courbot <acourbot@chromium.org>
Signed-off-by: Tzung-Bi Shih <tzungbi@google.com>
Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>

diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c
index 1a633b485a69c89962ef7f30e134662cdf6a126b..16e057f73789a149e1786d7819d00adae827b076 100644 (file)
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c
@@ -649,19 +649,17 @@ static int vidioc_try_fmt(struct v4l2_format *f,
 
        pix_fmt_mp->field = V4L2_FIELD_NONE;
 
+       pix_fmt_mp->width =
+               clamp(pix_fmt_mp->width, MTK_VDEC_MIN_W, MTK_VDEC_MAX_W);
+       pix_fmt_mp->height =
+               clamp(pix_fmt_mp->height, MTK_VDEC_MIN_H, MTK_VDEC_MAX_H);
+
        if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
                pix_fmt_mp->num_planes = 1;
                pix_fmt_mp->plane_fmt[0].bytesperline = 0;
        } else if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
                int tmp_w, tmp_h;
 
-               pix_fmt_mp->height = clamp(pix_fmt_mp->height,
-                                       MTK_VDEC_MIN_H,
-                                       MTK_VDEC_MAX_H);
-               pix_fmt_mp->width = clamp(pix_fmt_mp->width,
-                                       MTK_VDEC_MIN_W,
-                                       MTK_VDEC_MAX_W);
-
                /*
                 * Find next closer width align 64, heign align 64, size align
                 * 64 rectangle