path: root/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

#include "qffmpeghwaccel_p.h"
#if QT_CONFIG(vaapi)
#include "qffmpeghwaccel_vaapi_p.h"
#endif
#ifdef Q_OS_DARWIN
#include "qffmpeghwaccel_videotoolbox_p.h"
#endif
#if QT_CONFIG(wmf)
#include "qffmpeghwaccel_d3d11_p.h"
#endif
#ifdef Q_OS_ANDROID
#    include "qffmpeghwaccel_mediacodec_p.h"
#endif
#include "qffmpeg_p.h"
#include "qffmpegvideobuffer_p.h"

#include <private/qrhi_p.h>
#include <qloggingcategory.h>
#include <set>

/* Infrastructure for HW acceleration goes into this file. */

QT_BEGIN_NAMESPACE

static Q_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel");

namespace QFFmpeg {

static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = {
#if defined(Q_OS_ANDROID)
    AV_HWDEVICE_TYPE_MEDIACODEC,
#elif defined(Q_OS_LINUX)
    AV_HWDEVICE_TYPE_VAAPI,
    AV_HWDEVICE_TYPE_VDPAU,
    AV_HWDEVICE_TYPE_CUDA,
#elif defined (Q_OS_WIN)
    AV_HWDEVICE_TYPE_D3D11VA,
#elif defined (Q_OS_DARWIN)
    AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
#endif
};

static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
{
    std::vector<AVHWDeviceType> result;

    const auto definedDeviceTypes = qgetenv(envVarName);
    if (!definedDeviceTypes.isNull()) {
        const auto definedDeviceTypesString = QString::fromUtf8(definedDeviceTypes).toLower();
        for (const auto &deviceType : definedDeviceTypesString.split(',')) {
            constexpr std::array<std::pair<AVHWDeviceType, const char *>, 11> map = {
                std::make_pair(AV_HWDEVICE_TYPE_CUDA, "cuda"),
                std::make_pair(AV_HWDEVICE_TYPE_VAAPI, "vaapi"),
                std::make_pair(AV_HWDEVICE_TYPE_DXVA2, "dxva2"),
                std::make_pair(AV_HWDEVICE_TYPE_QSV, "qsv"),
                std::make_pair(AV_HWDEVICE_TYPE_VAAPI, "vaapi"),
                std::make_pair(AV_HWDEVICE_TYPE_VIDEOTOOLBOX, "videotoolbox"),
                std::make_pair(AV_HWDEVICE_TYPE_D3D11VA, "d3d11va"),
                std::make_pair(AV_HWDEVICE_TYPE_DRM, "drm"),
                std::make_pair(AV_HWDEVICE_TYPE_OPENCL, "opencl"),
                std::make_pair(AV_HWDEVICE_TYPE_MEDIACODEC, "mediacodec"),
                std::make_pair(AV_HWDEVICE_TYPE_VULKAN, "vulkan")
            };

            const auto found =
                    std::find_if(map.begin(), map.end(), [&deviceType](const auto &hwTypeToStr) {
                        return deviceType == hwTypeToStr.second;
                    });

            if (found != map.end())
                result.emplace_back(found->first);
        }

        return result;
    } else {
        std::set<AVHWDeviceType> deviceTypesSet;
        AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
        while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
            deviceTypesSet.insert(type);

        for (const auto preffered : preferredHardwareAccelerators)
            if (deviceTypesSet.erase(preffered))
                result.push_back(preffered);

        result.insert(result.end(), deviceTypesSet.begin(), deviceTypesSet.end());
    }

    result.shrink_to_fit();
    return result;
}

static AVBufferRef *loadHWContext(const AVHWDeviceType type)
{
    AVBufferRef *hwContext = nullptr;
    int ret = av_hwdevice_ctx_create(&hwContext, type, nullptr, nullptr, 0);
    qCDebug(qLHWAccel) << "    Checking HW context:" << av_hwdevice_get_type_name(type);
    if (ret == 0) {
        qCDebug(qLHWAccel) << "    Using above hw context.";
        return hwContext;
    }
    qCDebug(qLHWAccel) << "    Could not create hw context:" << ret << strerror(-ret);
    return nullptr;
}

template<typename CodecFinder>
std::pair<const AVCodec *, std::unique_ptr<HWAccel>>
findCodecWithHwAccel(AVCodecID id, const std::vector<AVHWDeviceType> &deviceTypes,
                     CodecFinder codecFinder,
                     const std::function<bool(const HWAccel &)> &hwAccelPredicate)
{
    for (auto type : deviceTypes) {
        const auto codec = codecFinder(id, type, {});

        if (!codec)
            continue;

        qCDebug(qLHWAccel) << "Found potential codec" << codec->name << "for hw accel" << type
                           << "; Checking the hw device...";

        auto hwAccel = QFFmpeg::HWAccel::create(type);

        if (!hwAccel)
            continue;

        if (hwAccelPredicate && !hwAccelPredicate(*hwAccel)) {
            qCDebug(qLHWAccel) << "HW device is available but doesn't suit due to restrictions";
            continue;
        }

        qCDebug(qLHWAccel) << "HW device is OK";

        return { codec, std::move(hwAccel) };
    }

    qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id;

    return { nullptr, nullptr };
}

// Used for the AVCodecContext::get_format callback
AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *fmt)
{
    // First check HW accelerated codecs, the HW device context must be set
    if (codecContext->hw_device_ctx) {
        auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data;
        auto formatForDeprecatedHw = AV_PIX_FMT_NONE;
        auto format = AV_PIX_FMT_NONE;
        for (int i = 0;
             const AVCodecHWConfig *config = avcodec_get_hw_config(codecContext->codec, i); i++) {
            if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
                continue;

            if (device_ctx->type != config->device_type)
                continue;

            const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0;

            for (int n = 0; fmt[n] != AV_PIX_FMT_NONE && format == AV_PIX_FMT_NONE; n++) {
                const auto pixFmt = config->pix_fmt;
                if ((pixFmt != AV_PIX_FMT_NONE && pixFmt == fmt[n])
                    || (pixFmt == AV_PIX_FMT_NONE
                        && isAVFormatSupported(codecContext->codec, fmt[n]))) {

                    if (!isDeprecated)
                        format = *fmt;
                    else if (formatForDeprecatedHw == AV_PIX_FMT_NONE)
                        formatForDeprecatedHw = *fmt;
                }
            }

            if (format != AV_PIX_FMT_NONE)
                break;
        }

        if (format == AV_PIX_FMT_NONE)
            format = formatForDeprecatedHw;

        if (format != AV_PIX_FMT_NONE) {
#if QT_CONFIG(wmf)
            if (format == AV_PIX_FMT_D3D11)
                QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(codecContext);
#endif
#ifdef Q_OS_ANDROID
            if (format == AV_PIX_FMT_MEDIACODEC)
                QFFmpeg::MediaCodecTextureConverter::setupDecoderSurface(codecContext);
#endif
            qCDebug(qLHWAccel) << "Selected format" << format << "for hw" << device_ctx->type;
            return format;
        }
    }

    // prefer video formats we can handle directly
    for (int n = 0; fmt[n] != AV_PIX_FMT_NONE; n++) {
        bool needsConversion = true;
        QFFmpegVideoBuffer::toQtPixelFormat(fmt[n], &needsConversion);
        if (!needsConversion) {
            qCDebug(qLHWAccel) << "Selected format with no conversion" << fmt[n];
            return fmt[n];
        }
    }

    qCDebug(qLHWAccel) << "Selected format with conversion" << *fmt;

    // take the native format, this will involve one additional format conversion on the CPU side
    return *fmt;
}

TextureConverter::Data::~Data()
{
    delete backend;
}

HWAccel::~HWAccel() = default;

std::unique_ptr<HWAccel> HWAccel::create(AVHWDeviceType deviceType)
{
    if (auto *ctx = loadHWContext(deviceType))
        return std::unique_ptr<HWAccel>(new HWAccel(ctx));
    else
        return {};
}

AVPixelFormat HWAccel::format(AVFrame *frame)
{
    if (!frame->hw_frames_ctx)
        return AVPixelFormat(frame->format);

    auto *hwFramesContext = (AVHWFramesContext *)frame->hw_frames_ctx->data;
    Q_ASSERT(hwFramesContext);
    return AVPixelFormat(hwFramesContext->sw_format);
}

const std::vector<AVHWDeviceType> &HWAccel::encodingDeviceTypes()
{
    static const auto &result = deviceTypes("QT_FFMPEG_ENCODING_HW_DEVICE_TYPES");
    return result;
}

const std::vector<AVHWDeviceType> &HWAccel::decodingDeviceTypes()
{
    static const auto &result = deviceTypes("QT_FFMPEG_DECODING_HW_DEVICE_TYPES");
    return result;
}

AVHWDeviceContext *HWAccel::hwDeviceContext() const
{
    return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr;
}

AVPixelFormat HWAccel::hwFormat() const
{
    switch (deviceType()) {
    case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
        return AV_PIX_FMT_VIDEOTOOLBOX;
    case AV_HWDEVICE_TYPE_VAAPI:
        return AV_PIX_FMT_VAAPI;
    case AV_HWDEVICE_TYPE_MEDIACODEC:
        return AV_PIX_FMT_MEDIACODEC;
    case AV_HWDEVICE_TYPE_CUDA:
        return AV_PIX_FMT_CUDA;
    case AV_HWDEVICE_TYPE_VDPAU:
        return AV_PIX_FMT_VDPAU;
    case AV_HWDEVICE_TYPE_OPENCL:
        return AV_PIX_FMT_OPENCL;
    case AV_HWDEVICE_TYPE_VULKAN:
        return AV_PIX_FMT_VULKAN;
    case AV_HWDEVICE_TYPE_QSV:
        return AV_PIX_FMT_QSV;
    case AV_HWDEVICE_TYPE_D3D11VA:
        return AV_PIX_FMT_D3D11;
    case AV_HWDEVICE_TYPE_DXVA2:
        return AV_PIX_FMT_DXVA2_VLD;
    case AV_HWDEVICE_TYPE_DRM:
        return AV_PIX_FMT_DRM_PRIME;
    default:
        return AV_PIX_FMT_NONE;
    }
}

std::pair<const AVCodec *, std::unique_ptr<HWAccel>>
HWAccel::findEncoderWithHwAccel(AVCodecID id, const std::function<bool(const HWAccel &)>& hwAccelPredicate)
{
    return findCodecWithHwAccel(id, encodingDeviceTypes(), &QFFmpeg::findAVEncoder,
                                hwAccelPredicate);
}

std::pair<const AVCodec *, std::unique_ptr<HWAccel>>
HWAccel::findDecoderWithHwAccel(AVCodecID id, const std::function<bool(const HWAccel &)>& hwAccelPredicate)
{
    return findCodecWithHwAccel(id, decodingDeviceTypes(), &QFFmpeg::findAVDecoder,
                                hwAccelPredicate);
}

AVHWDeviceType HWAccel::deviceType() const
{
    return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE;
}

void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size)
{
    if (m_hwFramesContext) {
        qWarning() << "Frames context has been already created!";
        return;
    }

    if (!m_hwDeviceContext)
        return;

    m_hwFramesContext.reset(av_hwframe_ctx_alloc(m_hwDeviceContext.get()));
    auto *c = (AVHWFramesContext *)m_hwFramesContext->data;
    c->format = hwFormat();
    c->sw_format = swFormat;
    c->width = size.width();
    c->height = size.height();
    qCDebug(qLHWAccel) << "init frames context";
    int err = av_hwframe_ctx_init(m_hwFramesContext.get());
    if (err < 0)
        qWarning() << "failed to init HW frame context" << err << err2str(err);
    else
        qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format;
}

AVHWFramesContext *HWAccel::hwFramesContext() const
{
    return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr;
}


TextureConverter::TextureConverter(QRhi *rhi)
    : d(new Data)
{
    d->rhi = rhi;
}

TextureSet *TextureConverter::getTextures(AVFrame *frame)
{
    if (!frame || isNull())
        return nullptr;

    Q_ASSERT(frame->format == d->format);
    return d->backend->getTextures(frame);
}

void TextureConverter::updateBackend(AVPixelFormat fmt)
{
    d->backend = nullptr;
    if (!d->rhi)
        return;
    switch (fmt) {
#if QT_CONFIG(vaapi)
    case AV_PIX_FMT_VAAPI:
        d->backend = new VAAPITextureConverter(d->rhi);
        break;
#endif
#ifdef Q_OS_DARWIN
    case AV_PIX_FMT_VIDEOTOOLBOX:
        d->backend = new VideoToolBoxTextureConverter(d->rhi);
        break;
#endif
#if QT_CONFIG(wmf)
    case AV_PIX_FMT_D3D11:
        d->backend = new D3D11TextureConverter(d->rhi);
        break;
#endif
#ifdef Q_OS_ANDROID
    case AV_PIX_FMT_MEDIACODEC:
        d->backend = new MediaCodecTextureConverter(d->rhi);
        break;
#endif
    default:
        break;
    }
    d->format = fmt;
}

} // namespace QFFmpeg

QT_END_NAMESPACE