lavc/qsv: Fix MSDK initialization failure in system memory mode

MSDK does not create internal acceleration device on Linux,
So MFXVideoCORE_SetHandle() is necessary.
It has been added for ff_qsv_init_session_device().
But missed for ff_qsv_init_internal_session() due to commit
1f26a23 overwrited commit db89f45

Fix #7030

Signed-off-by: Zhong Li <zhong.li@intel.com>

6 files changed, 109 insertions, 31 deletions

diff --git a/libavcodec/qsv.c b/libavcodec/qsv.c
index 65ad070e1f..c4aef70b1e 100644
--- a/libavcodec/qsv.c
+++ b/libavcodec/qsv.c
@@ -348,7 +348,41 @@ load_plugin_fail:
 
 }
 
-int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
+//This code is only required for Linux since a display handle is required.
+//For Windows the session is complete and ready to use.
+
+#ifdef AVCODEC_QSV_LINUX_SESSION_HANDLE
+static int ff_qsv_set_display_handle(AVCodecContext *avctx, QSVSession *qs)
+{
+    AVDictionary *child_device_opts = NULL;
+    AVVAAPIDeviceContext *hwctx;
+    int ret;
+
+    av_dict_set(&child_device_opts, "kernel_driver", "i915", 0);
+    av_dict_set(&child_device_opts, "driver",        "iHD",  0);
+
+    ret = av_hwdevice_ctx_create(&qs->va_device_ref, AV_HWDEVICE_TYPE_VAAPI, NULL, child_device_opts, 0);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to create a VAAPI device.\n");
+        return ret;
+    } else {
+        qs->va_device_ctx = (AVHWDeviceContext*)qs->va_device_ref->data;
+        hwctx = qs->va_device_ctx->hwctx;
+
+        ret = MFXVideoCORE_SetHandle(qs->session,
+                (mfxHandleType)MFX_HANDLE_VA_DISPLAY, (mfxHDL)hwctx->display);
+        if (ret < 0) {
+            return ff_qsv_print_error(avctx, ret, "Error during set display handle\n");
+        }
+    }
+
+    av_dict_free(&child_device_opts);
+
+    return 0;
+}
+#endif //AVCODEC_QSV_LINUX_SESSION_HANDLE
+
+int ff_qsv_init_internal_session(AVCodecContext *avctx, QSVSession *qs,
                                  const char *load_plugins)
 {
     mfxIMPL impl   = MFX_IMPL_AUTO_ANY;
@@ -357,18 +391,24 @@ int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
     const char *desc;
     int ret;
 
-    ret = MFXInit(impl, &ver, session);
+    ret = MFXInit(impl, &ver, &qs->session);
     if (ret < 0)
         return ff_qsv_print_error(avctx, ret,
                                   "Error initializing an internal MFX session");
 
-    ret = qsv_load_plugins(*session, load_plugins, avctx);
+#ifdef AVCODEC_QSV_LINUX_SESSION_HANDLE
+    ret = ff_qsv_set_display_handle(avctx, qs);
+    if (ret < 0)
+        return ret;
+#endif
+
+    ret = qsv_load_plugins(qs->session, load_plugins, avctx);
     if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
         return ret;
     }
 
-    MFXQueryIMPL(*session, &impl);
+    MFXQueryIMPL(qs->session, &impl);
 
     switch (MFX_IMPL_BASETYPE(impl)) {
     case MFX_IMPL_SOFTWARE:
@@ -758,3 +798,17 @@ int ff_qsv_init_session_frames(AVCodecContext *avctx, mfxSession *psession,
     *psession = session;
     return 0;
 }
+
+int ff_qsv_close_internal_session(QSVSession *qs)
+{
+    if (qs->session) {
+        MFXClose(qs->session);
+        qs->session = NULL;
+    }
+#ifdef AVCODEC_QSV_LINUX_SESSION_HANDLE
+    if (qs->va_device_ctx) {
+        qs->va_device_ctx->free(qs->va_device_ctx);
+    }
+#endif
+    return 0;
+}
diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index c50e9c792c..62885134b1 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -21,6 +21,22 @@
 #ifndef AVCODEC_QSV_INTERNAL_H
 #define AVCODEC_QSV_INTERNAL_H
 
+#if CONFIG_VAAPI
+#define AVCODEC_QSV_LINUX_SESSION_HANDLE
+#endif //CONFIG_VAAPI
+
+#ifdef AVCODEC_QSV_LINUX_SESSION_HANDLE
+#include <stdio.h>
+#include <string.h>
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <fcntl.h>
+#include <va/va.h>
+#include <va/va_drm.h>
+#include "libavutil/hwcontext_vaapi.h"
+#endif
+
 #include <mfx/mfxvideo.h>
 
 #include "libavutil/frame.h"
@@ -64,6 +80,14 @@ typedef struct QSVFrame {
     struct QSVFrame *next;
 } QSVFrame;
 
+typedef struct QSVSession {
+    mfxSession session;
+#ifdef AVCODEC_QSV_LINUX_SESSION_HANDLE
+    AVBufferRef *va_device_ref;
+    AVHWDeviceContext *va_device_ctx;
+#endif
+} QSVSession;
+
 typedef struct QSVFramesContext {
     AVBufferRef *hw_frames_ctx;
     void *logctx;
@@ -99,9 +123,11 @@ enum AVPictureType ff_qsv_map_pictype(int mfx_pic_type);
 
 enum AVFieldOrder ff_qsv_map_picstruct(int mfx_pic_struct);
 
-int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
+int ff_qsv_init_internal_session(AVCodecContext *avctx, QSVSession *qs,
                                  const char *load_plugins);
 
+int ff_qsv_close_internal_session(QSVSession *qs);
+
 int ff_qsv_init_session_device(AVCodecContext *avctx, mfxSession *psession,
                                AVBufferRef *device_ref, const char *load_plugins);
 
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index eef4fe7138..2fce478d63 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -62,9 +62,9 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
     if (session) {
         q->session = session;
     } else if (hw_frames_ref) {
-        if (q->internal_session) {
-            MFXClose(q->internal_session);
-            q->internal_session = NULL;
+        if (q->internal_qs.session) {
+            MFXClose(q->internal_qs.session);
+            q->internal_qs.session = NULL;
         }
         av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
 
@@ -72,7 +72,7 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
         if (!q->frames_ctx.hw_frames_ctx)
             return AVERROR(ENOMEM);
 
-        ret = ff_qsv_init_session_frames(avctx, &q->internal_session,
+        ret = ff_qsv_init_session_frames(avctx, &q->internal_qs.session,
                                          &q->frames_ctx, q->load_plugins,
                                          q->iopattern == MFX_IOPATTERN_OUT_OPAQUE_MEMORY);
         if (ret < 0) {
@@ -80,28 +80,28 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
             return ret;
         }
 
-        q->session = q->internal_session;
+        q->session = q->internal_qs.session;
     } else if (hw_device_ref) {
-        if (q->internal_session) {
-            MFXClose(q->internal_session);
-            q->internal_session = NULL;
+        if (q->internal_qs.session) {
+            MFXClose(q->internal_qs.session);
+            q->internal_qs.session = NULL;
         }
 
-        ret = ff_qsv_init_session_device(avctx, &q->internal_session,
+        ret = ff_qsv_init_session_device(avctx, &q->internal_qs.session,
                                          hw_device_ref, q->load_plugins);
         if (ret < 0)
             return ret;
 
-        q->session = q->internal_session;
+        q->session = q->internal_qs.session;
     } else {
-        if (!q->internal_session) {
-            ret = ff_qsv_init_internal_session(avctx, &q->internal_session,
+        if (!q->internal_qs.session) {
+            ret = ff_qsv_init_internal_session(avctx, &q->internal_qs,
                                                q->load_plugins);
             if (ret < 0)
                 return ret;
         }
 
-        q->session = q->internal_session;
+        q->session = q->internal_qs.session;
     }
 
     /* make sure the decoder is uninitialized */
@@ -529,8 +529,7 @@ int ff_qsv_decode_close(QSVContext *q)
     av_fifo_free(q->async_fifo);
     q->async_fifo = NULL;
 
-    if (q->internal_session)
-        MFXClose(q->internal_session);
+    ff_qsv_close_internal_session(&q->internal_qs);
 
     av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
     av_buffer_unref(&q->frames_ctx.mids_buf);
diff --git a/libavcodec/qsvdec.h b/libavcodec/qsvdec.h
index c057bc6722..64dc8d2f47 100644
--- a/libavcodec/qsvdec.h
+++ b/libavcodec/qsvdec.h
@@ -42,7 +42,7 @@ typedef struct QSVContext {
 
     // the session we allocated internally, in case the caller did not provide
     // one
-    mfxSession internal_session;
+    QSVSession internal_qs;
 
     QSVFramesContext frames_ctx;
 
diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index 9bf8574e30..207cdc1d61 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -954,7 +954,7 @@ static int qsvenc_init_session(AVCodecContext *avctx, QSVEncContext *q)
         if (!q->frames_ctx.hw_frames_ctx)
             return AVERROR(ENOMEM);
 
-        ret = ff_qsv_init_session_frames(avctx, &q->internal_session,
+        ret = ff_qsv_init_session_frames(avctx, &q->internal_qs.session,
                                          &q->frames_ctx, q->load_plugins,
                                          q->param.IOPattern == MFX_IOPATTERN_IN_OPAQUE_MEMORY);
         if (ret < 0) {
@@ -962,21 +962,21 @@ static int qsvenc_init_session(AVCodecContext *avctx, QSVEncContext *q)
             return ret;
         }
 
-        q->session = q->internal_session;
+        q->session = q->internal_qs.session;
     } else if (avctx->hw_device_ctx) {
-        ret = ff_qsv_init_session_device(avctx, &q->internal_session,
+        ret = ff_qsv_init_session_device(avctx, &q->internal_qs.session,
                                          avctx->hw_device_ctx, q->load_plugins);
         if (ret < 0)
             return ret;
 
-        q->session = q->internal_session;
+        q->session = q->internal_qs.session;
     } else {
-        ret = ff_qsv_init_internal_session(avctx, &q->internal_session,
+        ret = ff_qsv_init_internal_session(avctx, &q->internal_qs,
                                            q->load_plugins);
         if (ret < 0)
             return ret;
 
-        q->session = q->internal_session;
+        q->session = q->internal_qs.session;
     }
 
     return 0;
@@ -1507,10 +1507,9 @@ int ff_qsv_enc_close(AVCodecContext *avctx, QSVEncContext *q)
 
     if (q->session)
         MFXVideoENCODE_Close(q->session);
-    if (q->internal_session)
-        MFXClose(q->internal_session);
+
     q->session          = NULL;
-    q->internal_session = NULL;
+    ff_qsv_close_internal_session(&q->internal_qs);
 
     av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
     av_buffer_unref(&q->frames_ctx.mids_buf);
diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
index f2f4d38503..ec8b5419cb 100644
--- a/libavcodec/qsvenc.h
+++ b/libavcodec/qsvenc.h
@@ -102,7 +102,7 @@ typedef struct QSVEncContext {
     QSVFrame *work_frames;
 
     mfxSession session;
-    mfxSession internal_session;
+    QSVSession internal_qs;
 
     int packet_size;
     int width_align;