/*
 * Copyright (c) 2017-2025, NVIDIA CORPORATION. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#include "nvidia-drm-conftest.h"

#if defined(NV_DRM_AVAILABLE)

#include "nvidia-drm-gem-nvkms-memory.h"
#include "nvidia-drm-helper.h"
#include "nv_drm_common_ioctl.h"

#include <drm/drm_drv.h>
#include <drm/drm_prime.h>

#include <linux/io.h>
#if defined(NV_BSD)
#include <vm/vm_pageout.h>
#endif

static void __nv_drm_gem_nvkms_memory_free(struct nv_drm_gem_object *nv_gem)
{
    struct nv_drm_device *nv_dev = nv_gem->nv_dev;
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
        to_nv_nvkms_memory(nv_gem);

    if (nv_nvkms_memory->physically_mapped) {
        if (nv_nvkms_memory->pWriteCombinedIORemapAddress != NULL) {
            iounmap(nv_nvkms_memory->pWriteCombinedIORemapAddress);
        }

#if defined(NVCPU_AARCH64)
        if (nv_nvkms_memory->pNumaVMapAddress != NULL) {
            nv_drm_vunmap(nv_nvkms_memory->pNumaVMapAddress);
        }
#endif

        nvKms->unmapMemory(nv_dev->pDevice,
                           nv_nvkms_memory->base.pMemory,
                           NVKMS_KAPI_MAPPING_TYPE_USER,
                           nv_nvkms_memory->pPhysicalAddress);
    }

    if (nv_nvkms_memory->pages_count != 0) {
        nvKms->freeMemoryPages((NvU64 *)nv_nvkms_memory->pages);
    }

    /* Decrement GC6 blocker if it's still held */
    if (nv_nvkms_memory->was_mmapped) {
        nvKms->gc6BlockerRefCntDec(nv_dev->pDevice);
    }

    /* Free NvKmsKapiMemory handle associated with this gem object */

    nvKms->freeMemory(nv_dev->pDevice, nv_nvkms_memory->base.pMemory);

    nv_drm_free(nv_nvkms_memory);
}

static int __nv_drm_gem_nvkms_map(
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory);

static int __nv_drm_gem_nvkms_mmap(struct nv_drm_gem_object *nv_gem,
                                   struct vm_area_struct *vma)
{
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
        to_nv_nvkms_memory(nv_gem);

    int ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
    if (ret) {
       return ret;
    }

    if (nvKms->isVidmem(nv_nvkms_memory->base.pMemory)) {
        /*
         * Usermode clients can't be trusted not to access mappings while the GPU
         * is in GC6. Increment the GC6 blocker refcount to keep the GPU awake
         * while this mapping is active.
         */
        mutex_lock(&nv_nvkms_memory->map_lock);
        if (!nv_nvkms_memory->was_mmapped) {
            if (!nvKms->gc6BlockerRefCntInc(nv_gem->nv_dev->pDevice)) {
                mutex_unlock(&nv_nvkms_memory->map_lock);
                return -ENOMEM;
            }
            nv_nvkms_memory->was_mmapped = true;
        }
        mutex_unlock(&nv_nvkms_memory->map_lock);
    }

    return drm_gem_mmap_obj(&nv_gem->base,
                drm_vma_node_size(&nv_gem->base.vma_node) << PAGE_SHIFT, vma);
}

static vm_fault_t __nv_drm_gem_nvkms_handle_vma_fault(
    struct nv_drm_gem_object *nv_gem,
    struct vm_area_struct *vma,
    struct vm_fault *vmf)
{
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
        to_nv_nvkms_memory(nv_gem);
    unsigned long address = vmf->address;
    struct drm_gem_object *gem = vma->vm_private_data;
    unsigned long page_offset, pfn;
    vm_fault_t ret;

    page_offset = vmf->pgoff - drm_vma_node_start(&gem->vma_node);

    if (nv_nvkms_memory->pages_count == 0) {
        pfn = (unsigned long)(uintptr_t)nv_nvkms_memory->pPhysicalAddress;
        pfn >>= PAGE_SHIFT;
#if defined(NV_LINUX)
        /*
         * FreeBSD doesn't set pgoff. We instead have pfn be the base physical
         * address, and we will calculate the index pidx from the virtual address.
         *
         * This only works because linux_cdev_pager_populate passes the pidx as
         * vmf->virtual_address. Then we turn the virtual address
         * into a physical page number.
         */
        pfn += page_offset;
#endif
    } else {
        BUG_ON(page_offset >= nv_nvkms_memory->pages_count);
        pfn = page_to_pfn(nv_nvkms_memory->pages[page_offset]);
    }

#if defined(NV_VMF_INSERT_PFN_PRESENT)
    ret = vmf_insert_pfn(vma, address, pfn);
#else
    ret = vm_insert_pfn(vma, address, pfn);
    switch (ret) {
        case 0:
        case -EBUSY:
            /*
             * EBUSY indicates that another thread already handled
             * the faulted range.
             */
            ret = VM_FAULT_NOPAGE;
            break;
        case -ENOMEM:
            ret = VM_FAULT_OOM;
            break;
        default:
            WARN_ONCE(1, "Unhandled error in %s: %d\n", __FUNCTION__, ret);
            ret = VM_FAULT_SIGBUS;
            break;
    }
#endif /* defined(NV_VMF_INSERT_PFN_PRESENT) */
    return ret;
}

static struct drm_gem_object *__nv_drm_gem_nvkms_prime_dup(
    struct drm_device *dev,
    const struct nv_drm_gem_object *nv_gem_src);

static int __nv_drm_gem_nvkms_map(
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory)
{
    int ret = 0;
    struct nv_drm_device *nv_dev = nv_nvkms_memory->base.nv_dev;
    struct NvKmsKapiMemory *pMemory = nv_nvkms_memory->base.pMemory;

    mutex_lock(&nv_nvkms_memory->map_lock);

    if (nv_nvkms_memory->physically_mapped) {
        goto done;
    }

    if (!nvKms->isVidmem(pMemory)) {
        goto done;
    }

    /*
     * XXX Physical mapping currently broken in cases where we can't guarantee
     * that the mapping is contiguous. Fail on platforms that don't have
     * guaranteed contiguous physical mappings if the memory itself isn't
     * physically contiguous.
     */
    if (!nv_dev->contiguousPhysicalMappings
        && !nvKms->isContiguous(pMemory)
    ) {
        NV_DRM_DEV_LOG_INFO(
            nv_dev,
            "Mapping non-contiguous vidmem NvKmsKapiMemory 0x%p is currently "
            "unsupported on coherent GPU memory configurations",
            pMemory);
        ret = -ENOMEM;
        goto done;
    }

    if (!nvKms->mapMemory(nv_dev->pDevice,
                          pMemory,
                          NVKMS_KAPI_MAPPING_TYPE_USER,
                          &nv_nvkms_memory->pPhysicalAddress)) {
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to map NvKmsKapiMemory 0x%p",
            pMemory);
        ret = -ENOMEM;
        goto done;
    }

#if defined(NVCPU_AARCH64)
    if (!nv_aarch64_pfn_is_map_memory(
            __phys_to_pfn((phys_addr_t) nv_nvkms_memory->pPhysicalAddress)))
#endif
    {
        nv_nvkms_memory->pWriteCombinedIORemapAddress = ioremap_wc(
                (uintptr_t)nv_nvkms_memory->pPhysicalAddress,
                nv_nvkms_memory->base.base.size);

        if (!nv_nvkms_memory->pWriteCombinedIORemapAddress) {
            NV_DRM_DEV_LOG_INFO(
                nv_dev,
                "Failed to ioremap_wc NvKmsKapiMemory 0x%p",
                pMemory);
        }
    }
#if defined(NVCPU_AARCH64)
    else {
        /*
         * Handle NUMA on aarch64. In this case, video memory allocations may also
         * be kernel managed system memory, requiring vmap() instead of ioremap().
         */
        unsigned long i;
        unsigned long pages_count = nv_nvkms_memory->base.base.size / PAGE_SIZE;
        struct page **pages = nv_drm_calloc(pages_count, sizeof(struct page *));

        WARN_ON((nv_nvkms_memory->base.base.size % PAGE_SIZE) != 0);

        if (pages == NULL) {
            NV_DRM_DEV_LOG_INFO(
                nv_dev,
                "Failed to allocate page list for vmap NvKmsKapiMemory 0x%p",
                pMemory);
        } else {
            /*
             * Since this is a video memory allocation from the perspective of
             * RM, we can't use nvKms->getMemoryPages(). Instead, derive a list
             * of contiguous pages from the physical address.
             */
            const unsigned long base_pfn =
                __phys_to_pfn((phys_addr_t) nv_nvkms_memory->pPhysicalAddress);
            for (i = 0; i < pages_count; i++) {
                unsigned long pfn = base_pfn + i;
                WARN_ON(!pfn_valid(pfn));
                pages[i] = pfn_to_page(pfn);
            }

            nv_nvkms_memory->pNumaVMapAddress = nv_drm_vmap(pages, pages_count, false);
            if (!nv_nvkms_memory->pNumaVMapAddress) {
                NV_DRM_DEV_LOG_INFO(
                    nv_dev,
                    "Failed to vmap NvKmsKapiMemory 0x%p",
                    pMemory);
            }

            nv_drm_free(pages);
        }
    }
#endif

    nv_nvkms_memory->physically_mapped = true;

done:
    mutex_unlock(&nv_nvkms_memory->map_lock);
    return ret;
}

static void *__nv_drm_gem_nvkms_prime_vmap(
    struct nv_drm_gem_object *nv_gem)
{
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
        to_nv_nvkms_memory(nv_gem);

    int ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
    if (ret) {
       return ERR_PTR(ret);
    }

    if (nv_nvkms_memory->physically_mapped) {
        void *address;

        if (nv_nvkms_memory->pWriteCombinedIORemapAddress != NULL) {
            address = nv_nvkms_memory->pWriteCombinedIORemapAddress;
        }
#if defined(NVCPU_AARCH64)
        else if (nv_nvkms_memory->pNumaVMapAddress != NULL) {
            address = nv_nvkms_memory->pNumaVMapAddress;
        }
#endif
        else {
            return ERR_PTR(-ENOMEM);
        }

        /*
         * For CPU mappings of vidmem, increment the GC6 blocker refcount to
         * keep the GPU awake while the kernel is using the mapping.
         */
        if (nvKms->isVidmem(nv_gem->pMemory) &&
            !nvKms->gc6BlockerRefCntInc(nv_gem->nv_dev->pDevice)) {
            return ERR_PTR(-ENOMEM);
        }

        return address;
    }

    /*
     * If this buffer isn't physically mapped, it might be backed by struct
     * pages. Use vmap in that case. Do a noncached mapping for system memory
     * as display is non io-coherent device in case of Tegra.
     */
    if (nv_nvkms_memory->pages_count > 0) {
         return nv_drm_vmap(nv_nvkms_memory->pages,
                            nv_nvkms_memory->pages_count,
                            false);
    }

    return ERR_PTR(-ENOMEM);
}

static void __nv_drm_gem_nvkms_prime_vunmap(
    struct nv_drm_gem_object *nv_gem,
    void *address)
{
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
        to_nv_nvkms_memory(nv_gem);

    if (!nv_nvkms_memory->physically_mapped &&
        nv_nvkms_memory->pages_count > 0) {
        nv_drm_vunmap(address);
    }

    if (nv_nvkms_memory->physically_mapped &&
        nvKms->isVidmem(nv_gem->pMemory)) {
        /*
         * Decrement the GC6 blocker refcount now that the mapping is no longer
         * needed.
         */
        nvKms->gc6BlockerRefCntDec(nv_gem->nv_dev->pDevice);
    }
}

static int __nv_drm_gem_map_nvkms_memory_offset(
    struct nv_drm_device *nv_dev,
    struct nv_drm_gem_object *nv_gem,
    uint64_t *offset)
{
    return nv_drm_gem_create_mmap_offset(nv_gem, offset);
}

static struct sg_table *__nv_drm_gem_nvkms_memory_prime_get_sg_table(
    struct nv_drm_gem_object *nv_gem)
{
    struct nv_drm_device *nv_dev = nv_gem->nv_dev;
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory =
        to_nv_nvkms_memory(nv_gem);
    struct sg_table *sg_table;

    if (nv_nvkms_memory->pages_count == 0) {
        NV_DRM_DEV_DEBUG_DRIVER(
                nv_dev,
                "Cannot create sg_table for NvKmsKapiMemory 0x%p",
                nv_gem->pMemory);
        return ERR_PTR(-ENOMEM);
    }

    sg_table = nv_drm_prime_pages_to_sg(nv_dev->dev,
                                        nv_nvkms_memory->pages,
                                        nv_nvkms_memory->pages_count);

    return sg_table;
}

const struct nv_drm_gem_object_funcs nv_gem_nvkms_memory_ops = {
    .free = __nv_drm_gem_nvkms_memory_free,
    .prime_dup = __nv_drm_gem_nvkms_prime_dup,
    .prime_vmap = __nv_drm_gem_nvkms_prime_vmap,
    .prime_vunmap = __nv_drm_gem_nvkms_prime_vunmap,
    .mmap = __nv_drm_gem_nvkms_mmap,
    .handle_vma_fault = __nv_drm_gem_nvkms_handle_vma_fault,
    .create_mmap_offset = __nv_drm_gem_map_nvkms_memory_offset,
    .prime_get_sg_table = __nv_drm_gem_nvkms_memory_prime_get_sg_table,
};

static int __nv_drm_nvkms_gem_obj_init(
    struct nv_drm_device *nv_dev,
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory,
    struct NvKmsKapiMemory *pMemory,
    uint64_t size)
{
    NvU64 *pages = NULL;
    NvU32 numPages = 0;

    if ((size % PAGE_SIZE) != 0) {
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "NvKmsKapiMemory 0x%p size should be in a multiple of page size to "
            "create a gem object",
            pMemory);
        return -EINVAL;
    }

    mutex_init(&nv_nvkms_memory->map_lock);
    nv_nvkms_memory->pPhysicalAddress = NULL;
    nv_nvkms_memory->pWriteCombinedIORemapAddress = NULL;
    nv_nvkms_memory->physically_mapped = false;
    nv_nvkms_memory->was_mmapped = false;

    if (!nvKms->isVidmem(pMemory) &&
        !nvKms->getMemoryPages(nv_dev->pDevice,
                               pMemory,
                               &pages,
                               &numPages)) {
        /* GetMemoryPages will fail for vidmem allocations,
         * but it should not fail for sysmem allocations. */
        NV_DRM_DEV_LOG_ERR(nv_dev,
            "Failed to get memory pages for NvKmsKapiMemory 0x%p",
            pMemory);
        return -ENOMEM;
    }
    nv_nvkms_memory->pages_count = numPages;
    nv_nvkms_memory->pages = (struct page **)pages;

    nv_drm_gem_object_init(nv_dev,
                           &nv_nvkms_memory->base,
                           &nv_gem_nvkms_memory_ops,
                           size,
                           pMemory);

    return 0;
}

int nv_drm_dumb_create(
    struct drm_file *file_priv,
    struct drm_device *dev, struct drm_mode_create_dumb *args)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory;
    uint8_t compressible = 0;
    struct NvKmsKapiMemory *pMemory;
    struct NvKmsKapiAllocateMemoryParams allocParams = { };
    int ret = 0;

    args->pitch = roundup(args->width * ((args->bpp + 7) >> 3),
                          nv_dev->pitchAlignment);

    args->size = args->height * args->pitch;

    /* Core DRM requires gem object size to be aligned with PAGE_SIZE */

    args->size = roundup(args->size, PAGE_SIZE);

    if ((nv_nvkms_memory =
            nv_drm_calloc(1, sizeof(*nv_nvkms_memory))) == NULL) {
        ret = -ENOMEM;
        goto fail;
    }

    allocParams.layout = NvKmsSurfaceMemoryLayoutPitch;
    allocParams.type = NVKMS_KAPI_ALLOCATION_TYPE_SCANOUT;
    allocParams.size = args->size;
    allocParams.noDisplayCaching = true;
    allocParams.useVideoMemory = nv_dev->hasVideoMemory;
    allocParams.compressible = &compressible;

    pMemory = nvKms->allocateMemory(nv_dev->pDevice, &allocParams);
    if (pMemory == NULL) {
        ret = -ENOMEM;
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to allocate NvKmsKapiMemory for dumb object of size %" NvU64_fmtu,
            args->size);
        goto nvkms_alloc_memory_failed;
    }

    ret = __nv_drm_nvkms_gem_obj_init(nv_dev, nv_nvkms_memory, pMemory, args->size);
    if (ret) {
        goto nvkms_gem_obj_init_failed;
    }

    /* Always map dumb buffer memory up front.  Clients are only expected
     * to use dumb buffers for software rendering, so they're not much use
     * without a CPU mapping.
     */
    ret = __nv_drm_gem_nvkms_map(nv_nvkms_memory);
    if (ret) {
        nv_drm_gem_object_unreference_unlocked(&nv_nvkms_memory->base);
        goto fail;
    }

    return nv_drm_gem_handle_create_drop_reference(file_priv,
                                                   &nv_nvkms_memory->base,
                                                   &args->handle);

nvkms_gem_obj_init_failed:
    nvKms->freeMemory(nv_dev->pDevice, pMemory);

nvkms_alloc_memory_failed:
    nv_drm_free(nv_nvkms_memory);

fail:
    return ret;
}

int nv_drm_gem_import_nvkms_memory_ioctl(struct drm_device *dev,
                                         void *data, struct drm_file *filep)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct drm_nvidia_gem_import_nvkms_memory_params *p = data;
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory;
    struct NvKmsKapiMemory *pMemory;
    int ret;

    if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
        ret = -EOPNOTSUPP;
        goto failed;
    }

    if ((nv_nvkms_memory =
            nv_drm_calloc(1, sizeof(*nv_nvkms_memory))) == NULL) {
        ret = -ENOMEM;
        goto failed;
    }

    pMemory = nvKms->importMemory(nv_dev->pDevice,
                                  p->mem_size,
                                  p->nvkms_params_ptr,
                                  p->nvkms_params_size);

    if (pMemory == NULL) {
        ret = -EINVAL;
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to import NVKMS memory to GEM object");
        goto nvkms_import_memory_failed;
    }

    ret = __nv_drm_nvkms_gem_obj_init(nv_dev, nv_nvkms_memory, pMemory, p->mem_size);
    if (ret) {
        goto nvkms_gem_obj_init_failed;
    }

    return nv_drm_gem_handle_create_drop_reference(filep,
                                                   &nv_nvkms_memory->base,
                                                   &p->handle);
nvkms_gem_obj_init_failed:
    nvKms->freeMemory(nv_dev->pDevice, pMemory);

nvkms_import_memory_failed:
    nv_drm_free(nv_nvkms_memory);

failed:
    return ret;
}

int nv_drm_gem_export_nvkms_memory_ioctl(struct drm_device *dev,
                                         void *data, struct drm_file *filep)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct drm_nvidia_gem_export_nvkms_memory_params *p = data;
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory = NULL;
    int ret = 0;

    if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
        ret = -EOPNOTSUPP;
        goto done;
    }

    if (p->__pad != 0) {
        ret = -EINVAL;
        NV_DRM_DEV_LOG_ERR(nv_dev, "Padding fields must be zeroed");
        goto done;
    }

    if ((nv_nvkms_memory = nv_drm_gem_object_nvkms_memory_lookup(
                    filep,
                    p->handle)) == NULL) {
        ret = -EINVAL;
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to lookup NVKMS gem object for export: 0x%08x",
            p->handle);
        goto done;
    }

    if (!nvKms->exportMemory(nv_dev->pDevice,
                             nv_nvkms_memory->base.pMemory,
                             p->nvkms_params_ptr,
                             p->nvkms_params_size)) {
        ret = -EINVAL;
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to export memory from NVKMS GEM object: 0x%08x", p->handle);
        goto done;
    }

done:
    if (nv_nvkms_memory != NULL) {
        nv_drm_gem_object_unreference_unlocked(&nv_nvkms_memory->base);
    }

    return ret;
}

int nv_drm_gem_alloc_nvkms_memory_ioctl(struct drm_device *dev,
                                        void *data, struct drm_file *filep)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct drm_nvidia_gem_alloc_nvkms_memory_params *p = data;
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory = NULL;
    struct NvKmsKapiMemory *pMemory;
    struct NvKmsKapiAllocateMemoryParams allocParams = { };
    int ret = 0;

    if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
        ret = -EOPNOTSUPP;
        goto failed;
    }

    if ((p->__pad0 != 0) || (p->__pad1 != 0)) {
        ret = -EINVAL;
        NV_DRM_DEV_LOG_ERR(nv_dev, "non-zero value in padding field");
        goto failed;
    }

    if ((nv_nvkms_memory =
            nv_drm_calloc(1, sizeof(*nv_nvkms_memory))) == NULL) {
        ret = -ENOMEM;
        goto failed;
    }

    allocParams.layout = p->block_linear ?
        NvKmsSurfaceMemoryLayoutBlockLinear : NvKmsSurfaceMemoryLayoutPitch;
    allocParams.type = (p->flags & NV_GEM_ALLOC_NO_SCANOUT) ?
        NVKMS_KAPI_ALLOCATION_TYPE_OFFSCREEN : NVKMS_KAPI_ALLOCATION_TYPE_SCANOUT;
    allocParams.size = p->memory_size;
    allocParams.useVideoMemory = nv_dev->hasVideoMemory;
    allocParams.compressible = &p->compressible;

    pMemory = nvKms->allocateMemory(nv_dev->pDevice, &allocParams);
    if (pMemory == NULL && (p->flags & NV_GEM_ALLOC_NO_SCANOUT)) {
        NV_DRM_DEV_LOG_INFO(
                nv_dev,
                "Failed to allocate NVKMS video memory for GEM object, trying to fall back to sysmem");
        allocParams.useVideoMemory = false;
        pMemory = nvKms->allocateMemory(nv_dev->pDevice, &allocParams);
    }

    if (pMemory == NULL) {
        ret = -EINVAL;
        NV_DRM_DEV_LOG_ERR(nv_dev,
                           "Failed to allocate NVKMS memory for GEM object");
        goto nvkms_alloc_memory_failed;
    }

    ret = __nv_drm_nvkms_gem_obj_init(nv_dev, nv_nvkms_memory, pMemory,
                                      p->memory_size);
    if (ret) {
        goto nvkms_gem_obj_init_failed;
    }

    return nv_drm_gem_handle_create_drop_reference(filep,
                                                   &nv_nvkms_memory->base,
                                                   &p->handle);

nvkms_gem_obj_init_failed:
    nvKms->freeMemory(nv_dev->pDevice, pMemory);

nvkms_alloc_memory_failed:
    nv_drm_free(nv_nvkms_memory);

failed:
    return ret;
}

static struct drm_gem_object *__nv_drm_gem_nvkms_prime_dup(
    struct drm_device *dev,
    const struct nv_drm_gem_object *nv_gem_src)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    const struct nv_drm_device *nv_dev_src;
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory;
    struct NvKmsKapiMemory *pMemory;

    BUG_ON(nv_gem_src == NULL || nv_gem_src->ops != &nv_gem_nvkms_memory_ops);

    nv_dev_src = to_nv_device(nv_gem_src->base.dev);

    if ((nv_nvkms_memory =
            nv_drm_calloc(1, sizeof(*nv_nvkms_memory))) == NULL) {
        return NULL;
    }

    pMemory = nvKms->dupMemory(nv_dev->pDevice,
                               nv_dev_src->pDevice, nv_gem_src->pMemory);
    if (pMemory == NULL) {
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to import NVKMS memory to GEM object");
        goto nvkms_dup_memory_failed;
    }

    if (__nv_drm_nvkms_gem_obj_init(nv_dev,
                                    nv_nvkms_memory,
                                    pMemory,
                                    nv_gem_src->base.size)) {
        goto nvkms_gem_obj_init_failed;
    }

    return &nv_nvkms_memory->base.base;

nvkms_gem_obj_init_failed:
    nvKms->freeMemory(nv_dev->pDevice, pMemory);

nvkms_dup_memory_failed:
    nv_drm_free(nv_nvkms_memory);

    return NULL;
}

int nv_drm_dumb_map_offset(struct drm_file *file,
                           struct drm_device *dev, uint32_t handle,
                           uint64_t *offset)
{
    struct nv_drm_device *nv_dev = to_nv_device(dev);
    struct nv_drm_gem_nvkms_memory *nv_nvkms_memory;
    int ret = -EINVAL;

    if ((nv_nvkms_memory = nv_drm_gem_object_nvkms_memory_lookup(
                    file,
                    handle)) == NULL) {
        NV_DRM_DEV_LOG_ERR(
            nv_dev,
            "Failed to lookup gem object for mapping: 0x%08x",
            handle);
        return ret;
    }

    ret = __nv_drm_gem_map_nvkms_memory_offset(nv_dev,
                                               &nv_nvkms_memory->base, offset);

    nv_drm_gem_object_unreference_unlocked(&nv_nvkms_memory->base);

    return ret;
}

#if defined(NV_DRM_DRIVER_HAS_DUMB_DESTROY)
int nv_drm_dumb_destroy(struct drm_file *file,
                        struct drm_device *dev,
                        uint32_t handle)
{
    return drm_gem_handle_delete(file, handle);
}
#endif /* NV_DRM_DRIVER_HAS_DUMB_DESTROY */

#endif
