drm/vmwgfx: validate the screen formats

[ Upstream commit 8bb75aeb58bd688d70827ae179bd3da57b6d975b ]

The kms code wasn't validating the modifiers and was letting through
unsupported formats. rgb8 was never properly supported and has no
matching svga screen target format so remove it.
This fixes format/modifier failures in kms_addfb_basic from IGT.

Signed-off-by: Zack Rusin <zackr@vmware.com>
Reviewed-by: Martin Krastev <krastevm@vmware.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20220318174332.440068-4-zack@kde.org
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 50c64e7813be148785190e313e464a051ca81fcc..171e90c4b9f3fc24f27655d29902942a9d45e2db 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -916,6 +916,15 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
         * Sanity checks.
         */
 
+       if (!drm_any_plane_has_format(&dev_priv->drm,
+                                     mode_cmd->pixel_format,
+                                     mode_cmd->modifier[0])) {
+               drm_dbg(&dev_priv->drm,
+                       "unsupported pixel format %p4cc / modifier 0x%llx\n",
+                       &mode_cmd->pixel_format, mode_cmd->modifier[0]);
+               return -EINVAL;
+       }
+
        /* Surface must be marked as a scanout. */
        if (unlikely(!surface->metadata.scanout))
                return -EINVAL;
@@ -1229,20 +1238,13 @@ static int vmw_kms_new_framebuffer_bo(struct vmw_private *dev_priv,
                return -EINVAL;
        }
 
-       /* Limited framebuffer color depth support for screen objects */
-       if (dev_priv->active_display_unit == vmw_du_screen_object) {
-               switch (mode_cmd->pixel_format) {
-               case DRM_FORMAT_XRGB8888:
-               case DRM_FORMAT_ARGB8888:
-                       break;
-               case DRM_FORMAT_XRGB1555:
-               case DRM_FORMAT_RGB565:
-                       break;
-               default:
-                       DRM_ERROR("Invalid pixel format: %p4cc\n",
-                                 &mode_cmd->pixel_format);
-                       return -EINVAL;
-               }
+       if (!drm_any_plane_has_format(&dev_priv->drm,
+                                     mode_cmd->pixel_format,
+                                     mode_cmd->modifier[0])) {
+               drm_dbg(&dev_priv->drm,
+                       "unsupported pixel format %p4cc / modifier 0x%llx\n",
+                       &mode_cmd->pixel_format, mode_cmd->modifier[0]);
+               return -EINVAL;
        }
 
        vfbd = kzalloc(sizeof(*vfbd), GFP_KERNEL);

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index bbc809f7bd8a914a6af98ccaa9c517b15b1a8e30..8c8ee87fd3ac77777a75cd612244b8550a3529f2 100644 (file)
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -248,7 +248,6 @@ struct vmw_framebuffer_bo {
 static const uint32_t __maybe_unused vmw_primary_plane_formats[] = {
        DRM_FORMAT_XRGB1555,
        DRM_FORMAT_RGB565,
-       DRM_FORMAT_RGB888,
        DRM_FORMAT_XRGB8888,
        DRM_FORMAT_ARGB8888,
 };