gentoo-overlays
/
smoverlay
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
smoverlay/x11-drivers/nvidia-drivers/files/nvidia-drivers-337.25-3.18....

3614 lines
102 KiB
Diff
Raw Normal View History

Add x11-drivers/nvidia-drivers-337.25
2015-01-18 00:11:03 +03:00
diff -urN kernel.orig/nv.c kernel/nv.c
--- kernel.orig/nv.c 2014-05-27 22:59:51.000000000 +0400
+++ kernel/nv.c 2015-01-17 23:33:57.949450431 +0300
@@ -1827,7 +1827,7 @@
unsigned long i_arg
)
{
- return nvidia_ioctl(file->f_dentry->d_inode, file, cmd, i_arg);
+ return nvidia_ioctl(file->f_path.dentry->d_inode, file, cmd, i_arg);
}
/*
diff -urN kernel.orig/nv.c.orig kernel/nv.c.orig
--- kernel.orig/nv.c.orig 1970-01-01 03:00:00.000000000 +0300
+++ kernel/nv.c.orig 2014-05-27 22:59:51.000000000 +0400
@@ -0,0 +1,3319 @@
+/* _NVRM_COPYRIGHT_BEGIN_
+ *
+ * Copyright 1999-2013 by NVIDIA Corporation. All rights reserved. All
+ * information contained herein is proprietary and confidential to NVIDIA
+ * Corporation. Any use, reproduction, or disclosure without the written
+ * permission of NVIDIA Corporation is prohibited.
+ *
+ * _NVRM_COPYRIGHT_END_
+ */
+
+#include "nv-misc.h"
+#include "os-interface.h"
+#include "nv-linux.h"
+#include "nv-p2p.h"
+#include "nv-reg.h"
+#include "rmil.h"
+
+#if defined(NV_UVM_ENABLE) || defined(NV_UVM_NEXT_ENABLE)
+#include "nv_uvm_interface.h"
+#endif
+
+#if !defined(NV_VMWARE)
+#include "nv-frontend.h"
+#endif
+
+/*
+ * The module information macros for Linux single-module builds
+ * are present in nv-frontend.c.
+ */
+
+#if defined(NV_VMWARE) || (NV_BUILD_MODULE_INSTANCES != 0)
+#if defined(MODULE_LICENSE)
+MODULE_LICENSE("NVIDIA");
+#endif
+#if defined(MODULE_INFO)
+MODULE_INFO(supported, "external");
+#endif
+#if defined(MODULE_VERSION)
+MODULE_VERSION(NV_VERSION_STRING);
+#endif
+#ifdef MODULE_ALIAS_CHARDEV_MAJOR
+MODULE_ALIAS_CHARDEV_MAJOR(NV_MAJOR_DEVICE_NUMBER);
+#endif
+#endif
+
+#include "conftest/patches.h"
+
+/*
+ * our global state; one per device
+ */
+
+static NvU32 num_nv_devices = 0;
+NvU32 num_probed_nv_devices = 0;
+
+NvU32 nv_assign_gpu_count = 0;
+nv_pci_info_t nv_assign_gpu_pci_info[NV_MAX_DEVICES];
+
+nv_linux_state_t *nv_linux_devices;
+static nv_smu_state_t nv_smu_device;
+
+#define NV_STACK_CACHE_STR (NV_DEV_NAME"_stack_t")
+#define NV_PTE_CACHE_STR (NV_DEV_NAME"_pte_t")
+
+#if defined(NVCPU_X86) || defined(NVCPU_X86_64)
+NvU64 __nv_supported_pte_mask = ~_PAGE_NX;
+#endif
+
+/*
+ * And one for the control device
+ */
+
+nv_linux_state_t nv_ctl_device = { { 0 } };
+wait_queue_head_t nv_ctl_waitqueue;
+
+#if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT)
+static const char *__cpgattr_warning = \
+ "Your Linux kernel has known problems in its implementation of\n"
+ "the change_page_attr() kernel interface.\n\n"
+ "The NVIDIA graphics driver will attempt to work around these\n"
+ "problems, but system stability may be adversely affected.\n"
+ "It is recommended that you update to Linux 2.6.11 (or a newer\n"
+ "Linux kernel release).\n";
+
+static const char *__cpgattr_warning_2 = \
+ "Your Linux kernel's version and architecture indicate that it\n"
+ "may have an implementation of the change_page_attr() kernel\n"
+ "kernel interface known to have problems. The NVIDIA graphics\n"
+ "driver made an attempt to determine whether your kernel is\n"
+ "affected, but could not. It will assume the interface does not\n"
+ "work correctly and attempt to employ workarounds.\n"
+ "This may adversely affect system stability.\n"
+ "It is recommended that you update to Linux 2.6.11 (or a newer\n"
+ "Linux kernel release).\n";
+#endif
+
+static int nv_mmconfig_failure_detected = 0;
+static const char *__mmconfig_warning = \
+ "Your current system configuration has known problems when\n"
+ "accessing PCI Configuration Space that can lead to accesses\n"
+ "to the PCI Configuration Space of the wrong PCI device. This\n"
+ "is known to cause instabilities with the NVIDIA graphics driver.\n\n"
+ "Please see the MMConfig section in the readme for more information\n"
+ "on how to work around this problem.\n";
+
+#if !defined(NV_VMWARE) && \
+ (defined(NVCPU_X86) || defined(NVCPU_X86_64))
+static int nv_fbdev_failure_detected = 0;
+static const char *__fbdev_warning = \
+ "Your system is not currently configured to drive a VGA console\n"
+ "on the primary VGA device. The NVIDIA Linux graphics driver\n"
+ "requires the use of a text-mode VGA console. Use of other console\n"
+ "drivers including, but not limited to, vesafb, may result in\n"
+ "corruption and stability problems, and is not supported.\n";
+#endif
+
+#if defined(NV_SG_MAP_BUFFERS) && defined(NV_NEED_REMAP_CHECK)
+unsigned int nv_remap_count;
+unsigned int nv_remap_limit;
+#endif
+
+#if defined(NV_CONFIG_PREEMPT_RT)
+#define NV_UPDATE_MEMORY_TYPES_DEFAULT 0
+#else
+#define NV_UPDATE_MEMORY_TYPES_DEFAULT 1
+#endif
+
+int nv_update_memory_types = NV_UPDATE_MEMORY_TYPES_DEFAULT;
+
+void *nvidia_p2p_page_t_cache;
+static void *nv_pte_t_cache;
+void *nv_stack_t_cache;
+static nv_stack_t *__nv_init_sp;
+
+/*
+ * vGPU specific macro to lock/unlock nv_linux_devices list
+ * These macros are enabled only for vGPU module
+ * Lock acquisition order while using the nv_linux_devices list
+ * 1. LOCK_NV_LINUX_DEVICES()
+ * 2. Traverse the list
+ * If the list is traversed to search for an element say nvl,
+ * acquire the nvl->ldata_lock before step 3
+ * 3. UNLOCK_NV_LINUX_DEVICES()
+ * 4. Release nvl->ldata_lock after any read/write access to the
+ * nvl element is complete
+ */
+#if defined(NV_VGX_HYPER)
+struct semaphore nv_linux_devices_lock;
+#define LOCK_NV_LINUX_DEVICES() down(&nv_linux_devices_lock)
+#define UNLOCK_NV_LINUX_DEVICES() up(&nv_linux_devices_lock)
+#else
+#define LOCK_NV_LINUX_DEVICES()
+#define UNLOCK_NV_LINUX_DEVICES()
+#endif
+
+// allow an easy way to convert all debug printfs related to events
+// back and forth between 'info' and 'errors'
+#if defined(NV_DBG_EVENTS)
+#define NV_DBG_EVENTINFO NV_DBG_ERRORS
+#else
+#define NV_DBG_EVENTINFO NV_DBG_INFO
+#endif
+
+//
+// Attempt to determine if we are running into the MMCONFIG coherency
+// issue and, if so, warn the user and stop attempting to verify
+// and correct the BAR values (see NV_CHECK_PCI_CONFIG_SPACE()), so
+// that we do not do more harm than good.
+//
+#define NV_CHECK_MMCONFIG_FAILURE(nv,bar,value) \
+ { \
+ nv_linux_state_t *nvl; \
+ for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next) \
+ { \
+ nv_state_t *nv_tmp = NV_STATE_PTR(nvl); \
+ if (((nv) != nv_tmp) && \
+ (nv_tmp->bars[(bar)].address == (value))) \
+ { \
+ nv_prints(NV_DBG_ERRORS, __mmconfig_warning); \
+ nv_procfs_add_warning("mmconfig", __mmconfig_warning); \
+ nv_mmconfig_failure_detected = 1; \
+ return; \
+ } \
+ } \
+ }
+
+static void
+verify_pci_bars(
+ nv_state_t *nv,
+ void *dev_handle
+)
+{
+ NvU32 bar, bar_hi, bar_lo;
+
+ //
+ // If an MMCONFIG specific failure was detected, skip the
+ // PCI BAR verification to avoid overwriting the BAR(s)
+ // of a given device with those of this GPU. See above for
+ // more information.
+ //
+ if (nv_mmconfig_failure_detected)
+ return;
+
+ for (bar = 0; bar < NV_GPU_NUM_BARS; bar++)
+ {
+ nv_aperture_t *tmp = &nv->bars[bar];
+
+ bar_lo = bar_hi = 0;
+ if (tmp->offset == 0)
+ continue;
+
+ os_pci_read_dword(dev_handle, tmp->offset, &bar_lo);
+
+ if ((bar_lo & NVRM_PCICFG_BAR_ADDR_MASK)
+ != (tmp->address & 0xffffffff))
+ {
+ nv_printf(NV_DBG_USERERRORS,
+ "NVRM: BAR%u(L) is 0x%08x, will restore to 0x%08llx.\n",
+ bar, bar_lo, (tmp->address & 0xffffffff));
+
+ NV_CHECK_MMCONFIG_FAILURE(nv, bar,
+ (bar_lo & NVRM_PCICFG_BAR_ADDR_MASK));
+
+ os_pci_write_dword(dev_handle, tmp->offset, tmp->address);
+ }
+
+ if ((bar_lo & NVRM_PCICFG_BAR_MEMTYPE_MASK)
+ != NVRM_PCICFG_BAR_MEMTYPE_64BIT)
+ continue;
+
+ os_pci_read_dword(dev_handle, (tmp->offset + 4), &bar_hi);
+
+ if (bar_hi != (tmp->address >> 32))
+ {
+ nv_printf(NV_DBG_USERERRORS,
+ "NVRM: BAR%u(H) is 0x%08x, will restore to 0x%08llx.\n",
+ bar, bar_hi, (tmp->address >> 32));
+
+ os_pci_write_dword(dev_handle, (tmp->offset + 4),
+ (tmp->address >> 32));
+ }
+ }
+}
+
+void nv_check_pci_config_space(nv_state_t *nv, BOOL check_the_bars)
+{
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ unsigned short cmd, flag = 0;
+
+ pci_read_config_word(nvl->dev, PCI_COMMAND, &cmd);
+ if (!(cmd & PCI_COMMAND_MASTER))
+ {
+ nv_printf(NV_DBG_USERERRORS, "NVRM: restoring bus mastering!\n");
+ cmd |= PCI_COMMAND_MASTER;
+ flag = 1;
+ }
+
+ if (!(cmd & PCI_COMMAND_MEMORY))
+ {
+ nv_printf(NV_DBG_USERERRORS, "NVRM: restoring MEM access!\n");
+ cmd |= PCI_COMMAND_MEMORY;
+ flag = 1;
+ }
+
+ if (cmd & PCI_COMMAND_SERR)
+ {
+ nv_printf(NV_DBG_USERERRORS, "NVRM: clearing SERR enable bit!\n");
+ cmd &= ~PCI_COMMAND_SERR;
+ flag = 1;
+ }
+
+ if (cmd & PCI_COMMAND_INTX_DISABLE)
+ {
+ nv_printf(NV_DBG_USERERRORS, "NVRM: clearing INTx disable bit!\n");
+ cmd &= ~PCI_COMMAND_INTX_DISABLE;
+ flag = 1;
+ }
+
+ if (flag)
+ pci_write_config_word(nvl->dev, PCI_COMMAND, cmd);
+
+ if (check_the_bars && NV_MAY_SLEEP() && !(nv->flags & NV_FLAG_PASSTHRU))
+ verify_pci_bars(nv, nvl->dev);
+}
+
+void NV_API_CALL nv_verify_pci_config(
+ nv_state_t *nv,
+ BOOL check_the_bars
+)
+{
+ nv_linux_state_t *nvl;
+ nv_stack_t *sp;
+
+ if ((nv)->flags & NV_FLAG_USE_BAR0_CFG)
+ {
+ nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ sp = nvl->pci_cfgchk_sp;
+
+ rm_check_pci_config_space(sp, nv,
+ check_the_bars, FALSE, NV_MAY_SLEEP());
+ }
+ else
+ nv_check_pci_config_space(nv, NV_MAY_SLEEP());
+}
+
+/***
+ *** STATIC functions, only in this file
+ ***/
+
+/* nvos_ functions.. do not take a state device parameter */
+static int nvos_count_devices(nv_stack_t *);
+
+static nv_alloc_t *nvos_create_alloc(struct pci_dev *, int);
+static int nvos_free_alloc(nv_alloc_t *);
+
+/* lock-related functions that should only be called from this file */
+static void nv_lock_init_locks(nv_state_t *nv);
+
+
+/***
+ *** EXPORTS to Linux Kernel
+ ***/
+
+static int nvidia_open (struct inode *, struct file *);
+static int nvidia_close (struct inode *, struct file *);
+static unsigned int nvidia_poll (struct file *, poll_table *);
+static int nvidia_ioctl (struct inode *, struct file *, unsigned int, unsigned long);
+static long nvidia_unlocked_ioctl (struct file *, unsigned int, unsigned long);
+static void nvidia_isr_bh (unsigned long);
+#if !defined(NV_IRQ_HANDLER_T_PRESENT) || (NV_IRQ_HANDLER_T_ARGUMENT_COUNT == 3)
+static irqreturn_t nvidia_isr (int, void *, struct pt_regs *);
+#else
+static irqreturn_t nvidia_isr (int, void *);
+#endif
+static void nvidia_rc_timer (unsigned long);
+
+static int nvidia_ctl_open (struct inode *, struct file *);
+static int nvidia_ctl_close (struct inode *, struct file *);
+
+static int nvidia_probe (struct pci_dev *, const struct pci_device_id *);
+static void nvidia_remove (struct pci_dev *);
+static int nvidia_smu_probe (struct pci_dev *);
+
+#if defined(NV_PM_SUPPORT_DEVICE_DRIVER_MODEL)
+static int nvidia_suspend (struct pci_dev *, pm_message_t);
+static int nvidia_smu_suspend (void);
+static int nvidia_resume (struct pci_dev *);
+static int nvidia_smu_resume (void);
+#endif
+
+/***
+ *** see nv.h for functions exported to other parts of resman
+ ***/
+
+static struct pci_device_id nv_pci_table[] = {
+ {
+ .vendor = PCI_VENDOR_ID_NVIDIA,
+ .device = PCI_ANY_ID,
+ .subvendor = PCI_ANY_ID,
+ .subdevice = PCI_ANY_ID,
+ .class = (PCI_CLASS_DISPLAY_VGA << 8),
+ .class_mask = ~0
+ },
+ {
+ .vendor = PCI_VENDOR_ID_NVIDIA,
+ .device = PCI_ANY_ID,
+ .subvendor = PCI_ANY_ID,
+ .subdevice = PCI_ANY_ID,
+ .class = (PCI_CLASS_DISPLAY_3D << 8),
+ .class_mask = ~0
+ },
+ {
+ .vendor = PCI_VENDOR_ID_NVIDIA,
+ .device = NV_PCI_DEVICE_ID_SMU,
+ .subvendor = PCI_ANY_ID,
+ .subdevice = PCI_ANY_ID,
+ .class = (PCI_CLASS_PROCESSOR_CO << 8), /* SMU device class */
+ .class_mask = ~0
+ },
+ {
+ .vendor = PCI_VENDOR_ID_NVIDIA,
+ .device = 0x0e00,
+ .subvendor = PCI_ANY_ID,
+ .subdevice = PCI_ANY_ID,
+ .class = (PCI_CLASS_MULTIMEDIA_OTHER << 8),
+ .class_mask = ~0
+ },
+ { }
+};
+
+MODULE_DEVICE_TABLE(pci, nv_pci_table);
+
+static struct pci_driver nv_pci_driver = {
+ .name = NV_DEV_NAME,
+ .id_table = nv_pci_table,
+ .probe = nvidia_probe,
+#if defined(NV_VGX_HYPER)
+ .remove = nvidia_remove,
+#endif
+#if defined(NV_PM_SUPPORT_DEVICE_DRIVER_MODEL)
+ .suspend = nvidia_suspend,
+ .resume = nvidia_resume,
+#endif
+};
+
+#if defined(NV_VMWARE)
+/* character driver entry points */
+
+static struct file_operations nv_fops = {
+ .owner = THIS_MODULE,
+ .poll = nvidia_poll,
+#if defined(NV_FILE_OPERATIONS_HAS_IOCTL)
+ .ioctl = nvidia_ioctl,
+#endif
+#if defined(NV_FILE_OPERATIONS_HAS_UNLOCKED_IOCTL)
+ .unlocked_ioctl = nvidia_unlocked_ioctl,
+#endif
+ .open = nvidia_open,
+ .release = nvidia_close,
+};
+#else
+static nvidia_module_t nv_fops = {
+ .owner = THIS_MODULE,
+ .module_name = NV_DEV_NAME,
+ .open = nvidia_open,
+ .close = nvidia_close,
+ .ioctl = nvidia_ioctl,
+ .mmap = nvidia_mmap,
+ .poll = nvidia_poll,
+};
+#endif
+
+#if defined(VM_CHECKER)
+/* kernel virtual memory usage/allocation information */
+NvU32 vm_usage = 0;
+struct mem_track_t *vm_list = NULL;
+nv_spinlock_t vm_lock;
+#endif
+
+#if defined(KM_CHECKER)
+/* kernel logical memory usage/allocation information */
+NvU32 km_usage = 0;
+struct mem_track_t *km_list = NULL;
+nv_spinlock_t km_lock;
+#endif
+
+
+/***
+ *** STATIC functions
+ ***/
+
+static
+nv_alloc_t *nvos_create_alloc(
+ struct pci_dev *dev,
+ int num_pages
+)
+{
+ nv_alloc_t *at;
+ unsigned int pt_size, i;
+
+ NV_KMALLOC(at, sizeof(nv_alloc_t));
+ if (at == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate alloc info\n");
+ return NULL;
+ }
+
+ memset(at, 0, sizeof(nv_alloc_t));
+
+ pt_size = num_pages * sizeof(nv_pte_t *);
+ if (os_alloc_mem((void **)&at->page_table, pt_size) != RM_OK)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate page table\n");
+ NV_KFREE(at, sizeof(nv_alloc_t));
+ return NULL;
+ }
+
+ memset(at->page_table, 0, pt_size);
+ at->num_pages = num_pages;
+ NV_ATOMIC_SET(at->usage_count, 0);
+
+ for (i = 0; i < at->num_pages; i++)
+ {
+ NV_KMEM_CACHE_ALLOC(at->page_table[i], nv_pte_t_cache, nv_pte_t);
+ if (at->page_table[i] == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to allocate page table entry\n");
+ nvos_free_alloc(at);
+ return NULL;
+ }
+ memset(at->page_table[i], 0, sizeof(nv_pte_t));
+ }
+
+ at->pid = os_get_current_process();
+
+ return at;
+}
+
+static
+int nvos_free_alloc(
+ nv_alloc_t *at
+)
+{
+ unsigned int i;
+
+ if (at == NULL)
+ return -1;
+
+ if (NV_ATOMIC_READ(at->usage_count))
+ return 1;
+
+ for (i = 0; i < at->num_pages; i++)
+ {
+ if (at->page_table[i] != NULL)
+ NV_KMEM_CACHE_FREE(at->page_table[i], nv_pte_t, nv_pte_t_cache);
+ }
+ os_free_mem(at->page_table);
+
+ NV_KFREE(at, sizeof(nv_alloc_t));
+
+ return 0;
+}
+
+NvU8 nv_find_pci_capability(struct pci_dev *dev, NvU8 capability)
+{
+ u16 status;
+ u8 cap_ptr, cap_id;
+
+ pci_read_config_word(dev, PCI_STATUS, &status);
+ status &= PCI_STATUS_CAP_LIST;
+ if (!status)
+ return 0;
+
+ switch (dev->hdr_type) {
+ case PCI_HEADER_TYPE_NORMAL:
+ case PCI_HEADER_TYPE_BRIDGE:
+ pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &cap_ptr);
+ break;
+ default:
+ return 0;
+ }
+
+ do {
+ cap_ptr &= 0xfc;
+ pci_read_config_byte(dev, cap_ptr + PCI_CAP_LIST_ID, &cap_id);
+ if (cap_id == capability)
+ return cap_ptr;
+ pci_read_config_byte(dev, cap_ptr + PCI_CAP_LIST_NEXT, &cap_ptr);
+ } while (cap_ptr && cap_id != 0xff);
+
+ return 0;
+}
+
+#if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT)
+/*
+ * nv_verify_cpa_interface() - determine if the change_page_attr() large page
+ * management accounting bug known to exist in early Linux/x86-64 kernels
+ * is present in this kernel.
+ *
+ * There's really no good way to determine if change_page_attr() is working
+ * correctly. We can't reliably use change_page_attr() on Linux/x86-64 2.6
+ * kernels < 2.6.11: if we run into the accounting bug, the Linux kernel will
+ * trigger a BUG() if we attempt to restore the WB memory type of a page
+ * originally part of a large page.
+ *
+ * So if we can successfully allocate such a page, change its memory type to
+ * UC and check if the accounting was done correctly, we can determine if
+ * the change_page_attr() interface can be used safely.
+ *
+ * Return values:
+ * 0 - test passed, the change_page_attr() interface works
+ * 1 - test failed, the status is unclear
+ * -1 - test failed, the change_page_attr() interface is broken
+ */
+
+static inline pte_t *check_large_page(unsigned long vaddr)
+{
+ pgd_t *pgd = NULL;
+ pmd_t *pmd = NULL;
+
+ pgd = NV_PGD_OFFSET(vaddr, 1, NULL);
+ if (!NV_PGD_PRESENT(pgd))
+ return NULL;
+
+ pmd = NV_PMD_OFFSET(vaddr, pgd);
+ if (!pmd || pmd_none(*pmd))
+ return NULL;
+
+ if (!pmd_large(*pmd))
+ return NULL;
+
+ return (pte_t *) pmd;
+}
+
+#define CPA_FIXED_MAX_ALLOCS 500
+
+int nv_verify_cpa_interface(void)
+{
+ unsigned int i, size;
+ unsigned long large_page = 0;
+ unsigned long *vaddr_list;
+ size = sizeof(unsigned long) * CPA_FIXED_MAX_ALLOCS;
+
+ NV_KMALLOC(vaddr_list, size);
+ if (!vaddr_list)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: nv_verify_cpa_interface: failed to allocate "
+ "page table\n");
+ return 1;
+ }
+
+ memset(vaddr_list, 0, size);
+
+ /* try to track down an allocation from a 2M page. */
+ for (i = 0; i < CPA_FIXED_MAX_ALLOCS; i++)
+ {
+ vaddr_list[i] = __get_free_page(GFP_KERNEL);
+ if (!vaddr_list[i])
+ continue;
+
+#if defined(_PAGE_NX)
+ if ((pgprot_val(PAGE_KERNEL) & _PAGE_NX) &&
+ virt_to_phys((void *)vaddr_list[i]) < 0x400000)
+ continue;
+#endif
+
+ if (check_large_page(vaddr_list[i]) != NULL)
+ {
+ large_page = vaddr_list[i];
+ vaddr_list[i] = 0;
+ break;
+ }
+ }
+
+ for (i = 0; i < CPA_FIXED_MAX_ALLOCS; i++)
+ {
+ if (vaddr_list[i])
+ free_page(vaddr_list[i]);
+ }
+ NV_KFREE(vaddr_list, size);
+
+ if (large_page)
+ {
+ struct page *page = virt_to_page(large_page);
+ struct page *kpte_page;
+ pte_t *kpte;
+ unsigned long kpte_val;
+ pgprot_t prot;
+
+ // lookup a pointer to our pte
+ kpte = check_large_page(large_page);
+ kpte_val = pte_val(*kpte);
+ kpte_page = virt_to_page(((unsigned long)kpte) & PAGE_MASK);
+
+ prot = PAGE_KERNEL_NOCACHE;
+ pgprot_val(prot) &= __nv_supported_pte_mask;
+
+ // this should split the large page
+ change_page_attr(page, 1, prot);
+
+ // broken kernels may get confused after splitting the page and
+ // restore the page before returning to us. detect that case.
+ if (((pte_val(*kpte) & ~_PAGE_NX) == kpte_val) &&
+ (pte_val(*kpte) & _PAGE_PSE))
+ {
+ if ((pte_val(*kpte) & _PAGE_NX) &&
+ (__nv_supported_pte_mask & _PAGE_NX) == 0)
+ clear_bit(_PAGE_BIT_NX, kpte);
+ // don't change the page back, as it's already been reverted
+ put_page(kpte_page);
+ free_page(large_page);
+ return -1; // yep, we're broken
+ }
+
+ // ok, now see if our bookkeeping is broken
+ if (page_count(kpte_page) != 0)
+ return -1; // yep, we're broken
+
+ prot = PAGE_KERNEL;
+ pgprot_val(prot) &= __nv_supported_pte_mask;
+
+ // everything's ok!
+ change_page_attr(page, 1, prot);
+ free_page(large_page);
+ return 0;
+ }
+
+ return 1;
+}
+#endif /* defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) */
+
+int __init nvidia_init_module(void)
+{
+ RM_STATUS status;
+ int rc;
+ NvU32 count, data, i;
+ nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device);
+ nv_stack_t *sp = NULL;
+ nv_linux_state_t *nvl;
+ nv_smu_state_t *nv_smu = &nv_smu_device;
+
+ if (NV_BUILD_MODULE_INSTANCES != 0)
+ {
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia module instance %d\n",
+ NV_MODULE_INSTANCE);
+ }
+
+ nv_user_map_init();
+
+ rc = nv_heap_create();
+ if (rc < 0)
+ {
+ goto failed4;
+ }
+
+ rc = nv_mem_pool_create();
+ if (rc < 0)
+ {
+ goto failed4;
+ }
+
+#if defined(NV_LINUX_NVMAP_H_PRESENT) && defined(HAVE_NV_ANDROID)
+ status = nv_nvmap_create_client();
+ if (RM_OK != status)
+ {
+ rc = -EIO;
+ goto failed4;
+ }
+#endif
+
+#if defined(VM_CHECKER)
+ NV_SPIN_LOCK_INIT(&vm_lock);
+#endif
+#if defined(KM_CHECKER)
+ NV_SPIN_LOCK_INIT(&km_lock);
+#endif
+
+ NV_KMEM_CACHE_CREATE(nv_stack_t_cache, NV_STACK_CACHE_STR, nv_stack_t);
+ if (nv_stack_t_cache == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: stack cache allocation failed!\n");
+ rc = -ENOMEM;
+ goto failed4;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ NV_KMEM_CACHE_DESTROY(nv_stack_t_cache);
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ rc = -ENOMEM;
+ goto failed4;
+ }
+
+ if (!rm_init_rm(sp))
+ {
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ NV_KMEM_CACHE_DESTROY(nv_stack_t_cache);
+ nv_printf(NV_DBG_ERRORS, "NVRM: rm_init_rm() failed!\n");
+ return -EIO;
+ }
+
+ count = nvos_count_devices(sp);
+ if (count == 0)
+ {
+ if (NV_IS_ASSIGN_GPU_PCI_INFO_SPECIFIED())
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The requested GPU assignments are invalid. Please ensure\n"
+ "NVRM: that the GPUs you wish to assign to this kernel module\n"
+ "NVRM: are present and available.\n");
+ }
+ else
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: No NVIDIA graphics adapter found!\n");
+ }
+ rc = -ENODEV;
+ goto failed5;
+ }
+
+ nv_linux_devices = NULL;
+#if defined(NV_VGX_HYPER)
+ NV_INIT_MUTEX(&nv_linux_devices_lock);
+#endif
+
+ memset(&nv_smu_device, 0, sizeof(nv_smu_state_t));
+
+ rc = nv_register_chrdev((void *)&nv_fops);
+ if (rc < 0)
+ goto failed5;
+
+ /* create /proc/driver/nvidia/... */
+ rc = nv_register_procfs();
+ if (rc < 0)
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to register procfs!\n");
+
+ if (pci_register_driver(&nv_pci_driver) < 0)
+ {
+ rc = -ENODEV;
+ nv_printf(NV_DBG_ERRORS, "NVRM: No NVIDIA graphics adapter found!\n");
+ goto failed4;
+ }
+
+ if (nv_drm_init(&nv_pci_driver) < 0)
+ {
+ rc = -ENODEV;
+ nv_printf(NV_DBG_ERRORS, "NVRM: DRM init failed\n");
+ goto failed3;
+ }
+
+ if (nv_smu->handle != NULL)
+ {
+ /* init SMU functionality */
+ rm_init_smu(sp, nv_smu);
+ }
+
+ if (num_probed_nv_devices != count)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The NVIDIA probe routine was not called for %d device(s).\n",
+ count - num_probed_nv_devices);
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This can occur when a driver such as: \n"
+ "NVRM: nouveau, rivafb, nvidiafb or rivatv "
+#if (NV_BUILD_MODULE_INSTANCES != 0)
+ "NVRM: or another NVIDIA kernel module "
+#endif
+ "\nNVRM: was loaded and obtained ownership of the NVIDIA device(s).\n");
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: Try unloading the conflicting kernel module (and/or\n"
+ "NVRM: reconfigure your kernel without the conflicting\n"
+ "NVRM: driver(s)), then try loading the NVIDIA kernel module\n"
+ "NVRM: again.\n");
+ }
+
+ if (num_probed_nv_devices == 0)
+ {
+ rc = -ENODEV;
+ nv_printf(NV_DBG_ERRORS, "NVRM: No NVIDIA graphics adapter probed!\n");
+ goto failed2;
+ }
+
+ if (num_probed_nv_devices != num_nv_devices)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The NVIDIA probe routine failed for %d device(s).\n",
+ num_probed_nv_devices - num_nv_devices);
+ }
+
+ if (num_nv_devices == 0)
+ {
+ rc = -ENODEV;
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: None of the NVIDIA graphics adapters were initialized!\n");
+ goto failed2;
+ }
+
+ nv_printf(NV_DBG_ERRORS, "NVRM: loading %s", pNVRM_ID);
+ if (__nv_patches[0].short_description != NULL)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ " (applied patches: %s", __nv_patches[0].short_description);
+ for (i = 1; __nv_patches[i].short_description; i++)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ ",%s", __nv_patches[i].short_description);
+ }
+ nv_printf(NV_DBG_ERRORS, ")");
+ }
+ nv_printf(NV_DBG_ERRORS, "\n");
+
+ // init the nvidia control device
+ nv->os_state = (void *) &nv_ctl_device;
+ nv_lock_init_locks(nv);
+
+ NV_KMEM_CACHE_CREATE(nv_pte_t_cache, NV_PTE_CACHE_STR, nv_pte_t);
+ if (nv_pte_t_cache == NULL)
+ {
+ rc = -ENOMEM;
+ nv_printf(NV_DBG_ERRORS, "NVRM: pte cache allocation failed\n");
+ goto failed;
+ }
+
+ if (NV_BUILD_MODULE_INSTANCES == 0)
+ {
+ NV_KMEM_CACHE_CREATE(nvidia_p2p_page_t_cache, "nvidia_p2p_page_t",
+ nvidia_p2p_page_t);
+ if (nvidia_p2p_page_t_cache == NULL)
+ {
+ rc = -ENOMEM;
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: p2p page cache allocation failed\n");
+ goto failed;
+ }
+ }
+
+#if defined(NV_SG_MAP_BUFFERS) && defined(NV_NEED_REMAP_CHECK)
+ rm_read_registry_dword(sp, nv, "NVreg", "RemapLimit", &nv_remap_limit);
+
+ // allow an override, but use default if no override
+ if (nv_remap_limit == 0)
+ nv_remap_limit = NV_REMAP_LIMIT_DEFAULT;
+
+ nv_remap_count = 0;
+#endif
+
+#if !defined(NV_VMWARE) && \
+ (defined(NVCPU_X86_64) || (defined(NVCPU_X86) && defined(CONFIG_X86_PAE)))
+ if (boot_cpu_has(X86_FEATURE_NX))
+ {
+ NvU32 __eax, __edx;
+ rdmsr(MSR_EFER, __eax, __edx);
+ if ((__eax & EFER_NX) != 0)
+ __nv_supported_pte_mask |= _PAGE_NX;
+ }
+ if (_PAGE_NX != ((NvU64)1<<63))
+ {
+ /*
+ * Make sure we don't strip software no-execute
+ * bits from PAGE_KERNEL(_NOCACHE) before calling
+ * change_page_attr().
+ */
+ __nv_supported_pte_mask |= _PAGE_NX;
+ }
+#endif
+
+ /*
+ * Give users an opportunity to disable the driver's use of
+ * the change_page_attr(), set_pages_{uc,wb}() and set_memory_{uc,wb}() kernel
+ * interfaces.
+ */
+ status = rm_read_registry_dword(sp, nv,
+ "NVreg", NV_REG_UPDATE_MEMORY_TYPES, &data);
+ if ((status == RM_OK) && ((int)data != ~0))
+ {
+ nv_update_memory_types = data;
+ }
+
+#if defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT)
+ /*
+ * Unless we explicitely detect that the change_page_attr()
+ * inteface is fixed, disable usage of the interface on
+ * this kernel. Notify the user of this problem using the
+ * driver's /proc warnings interface (read by the installer
+ * and the bug report script).
+ */
+ else
+ {
+ rc = nv_verify_cpa_interface();
+ if (rc < 0)
+ {
+ nv_prints(NV_DBG_ERRORS, __cpgattr_warning);
+ nv_procfs_add_warning("change_page_attr", __cpgattr_warning);
+ nv_update_memory_types = 0;
+ }
+ else if (rc != 0)
+ {
+ nv_prints(NV_DBG_ERRORS, __cpgattr_warning_2);
+ nv_procfs_add_warning("change_page_attr", __cpgattr_warning_2);
+ nv_update_memory_types = 0;
+ }
+ }
+#endif /* defined(NV_CHANGE_PAGE_ATTR_BUG_PRESENT) */
+
+#if defined(NVCPU_X86_64) && defined(CONFIG_IA32_EMULATION) && \
+ !defined(NV_FILE_OPERATIONS_HAS_COMPAT_IOCTL)
+ rm_register_compatible_ioctls(sp);
+#endif
+
+ rc = nv_init_pat_support(sp);
+ if (rc < 0)
+ goto failed;
+
+ __nv_init_sp = sp;
+
+ for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
+ {
+#if defined(NV_PM_MESSAGE_T_HAS_EVENT)
+ nvl->nv_state.flags |= NV_FLAG_S4_AVAILABLE;
+#else
+ nvl->nv_state.flags &= ~NV_FLAG_S4_AVAILABLE;
+#endif
+#if defined(NV_PM_VT_SWITCH_REQUIRED_PRESENT)
+ pm_vt_switch_required(&nvl->dev->dev, NV_TRUE);
+#endif
+ }
+ return 0;
+
+failed:
+ if (nvidia_p2p_page_t_cache != NULL)
+ NV_KMEM_CACHE_DESTROY(nvidia_p2p_page_t_cache);
+
+ if (nv_pte_t_cache != NULL)
+ NV_KMEM_CACHE_DESTROY(nv_pte_t_cache);
+
+ nv_unregister_chrdev((void *)&nv_fops);
+
+failed2:
+ while (nv_linux_devices != NULL)
+ {
+ nv_linux_state_t *tmp;
+ if (nv_linux_devices->dev)
+ {
+ struct pci_dev *dev = nv_linux_devices->dev;
+ release_mem_region(NV_PCI_RESOURCE_START(dev, NV_GPU_BAR_INDEX_REGS),
+ NV_PCI_RESOURCE_SIZE(dev, NV_GPU_BAR_INDEX_REGS));
+ NV_PCI_DISABLE_DEVICE(dev);
+ pci_set_drvdata(dev, NULL);
+ }
+ tmp = nv_linux_devices;
+ nv_linux_devices = nv_linux_devices->next;
+ NV_KFREE(tmp, sizeof(nv_linux_state_t));
+ }
+
+ nv_drm_exit(&nv_pci_driver);
+
+failed3:
+ if (nv_smu->handle != NULL)
+ {
+ struct pci_dev *dev = nv_smu->handle;
+ rm_shutdown_smu(sp, nv_smu);
+ release_mem_region(NV_PCI_RESOURCE_START(dev, 0),
+ NV_PCI_RESOURCE_SIZE(dev, 0));
+ pci_disable_device(dev);
+ pci_set_drvdata(dev, NULL);
+ memset(&nv_smu_device, 0, sizeof(nv_smu_state_t));
+ }
+
+ pci_unregister_driver(&nv_pci_driver);
+
+failed5:
+ rm_shutdown_rm(sp);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ NV_KMEM_CACHE_DESTROY(nv_stack_t_cache);
+
+failed4:
+ nv_mem_pool_destroy();
+ nv_heap_destroy();
+
+ return rc;
+}
+
+void __exit nvidia_exit_module(void)
+{
+ nv_smu_state_t *nv_smu = &nv_smu_device;
+ nv_stack_t *sp = __nv_init_sp;
+ struct pci_dev *dev;
+
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia_exit_module\n");
+
+ nv_drm_exit(&nv_pci_driver);
+
+ if ((dev = (struct pci_dev *)(nv_smu->handle)) != NULL)
+ {
+ rm_shutdown_smu(sp, nv_smu);
+ release_mem_region(NV_PCI_RESOURCE_START(dev, 0),
+ NV_PCI_RESOURCE_SIZE(dev, 0));
+ pci_disable_device(dev);
+ pci_set_drvdata(dev, NULL);
+ memset(&nv_smu_device, 0, sizeof(nv_smu_state_t));
+ }
+
+ while (nv_linux_devices != NULL)
+ {
+ nv_linux_state_t *next = nv_linux_devices->next;
+
+ if ((dev = nv_linux_devices->dev) != NULL)
+ {
+#if defined(NV_PM_VT_SWITCH_REQUIRED_PRESENT)
+ pm_vt_switch_unregister(&dev->dev);
+#endif
+ nvidia_remove(dev);
+ }
+ nv_linux_devices = next;
+ }
+
+ pci_unregister_driver(&nv_pci_driver);
+
+ /* remove /proc/driver/nvidia/... */
+ nv_unregister_procfs();
+
+ nv_unregister_chrdev((void *)&nv_fops);
+
+#if defined(NV_LINUX_NVMAP_H_PRESENT) && defined(HAVE_NV_ANDROID)
+ nv_nvmap_destroy_client();
+#endif
+ // Shutdown the resource manager
+ rm_shutdown_rm(sp);
+
+#if defined(NVCPU_X86_64) && defined(CONFIG_IA32_EMULATION) && \
+ !defined(NV_FILE_OPERATIONS_HAS_COMPAT_IOCTL)
+ rm_unregister_compatible_ioctls(sp);
+#endif
+
+ nv_teardown_pat_support();
+
+#if defined(NV_ENABLE_MEM_TRACKING)
+#if defined(VM_CHECKER)
+ if (vm_usage != 0)
+ {
+ nv_list_mem("VM", vm_list);
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: final VM memory usage: 0x%x bytes\n", vm_usage);
+ }
+#endif
+#if defined(KM_CHECKER)
+ if (km_usage != 0)
+ {
+ nv_list_mem("KM", km_list);
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: final KM memory usage: 0x%x bytes\n", km_usage);
+ }
+#endif
+#if defined(NV_SG_MAP_BUFFERS) && defined(NV_NEED_REMAP_CHECK)
+ if (nv_remap_count != 0)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: final SG memory usage: 0x%x bytes\n", nv_remap_count);
+ }
+#endif
+#endif /* NV_ENABLE_MEM_TRACKING */
+
+ if (NV_BUILD_MODULE_INSTANCES == 0)
+ {
+ NV_KMEM_CACHE_DESTROY(nvidia_p2p_page_t_cache);
+ }
+ NV_KMEM_CACHE_DESTROY(nv_pte_t_cache);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ NV_KMEM_CACHE_DESTROY(nv_stack_t_cache);
+
+ nv_mem_pool_destroy();
+ nv_heap_destroy();
+}
+
+
+/*
+ * Module entry and exit functions for Linux single-module builds
+ * are present in nv-frontend.c.
+ */
+
+#if defined(NV_VMWARE) || (NV_BUILD_MODULE_INSTANCES != 0)
+module_init(nvidia_init_module);
+module_exit(nvidia_exit_module);
+#endif
+
+void *nv_alloc_file_private(void)
+{
+ nv_file_private_t *nvfp;
+ unsigned int i;
+
+ NV_KMALLOC(nvfp, sizeof(nv_file_private_t));
+ if (!nvfp)
+ return NULL;
+
+ memset(nvfp, 0, sizeof(nv_file_private_t));
+
+ for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
+ {
+ NV_INIT_MUTEX(&nvfp->fops_sp_lock[i]);
+ }
+ init_waitqueue_head(&nvfp->waitqueue);
+ NV_SPIN_LOCK_INIT(&nvfp->fp_lock);
+
+ return nvfp;
+}
+
+void nv_free_file_private(nv_file_private_t *nvfp)
+{
+ nvidia_event_t *nvet;
+
+ if (nvfp == NULL)
+ return;
+
+ for (nvet = nvfp->event_head; nvet != NULL; nvet = nvfp->event_head)
+ {
+ nvfp->event_head = nvfp->event_head->next;
+ NV_KFREE(nvet, sizeof(nvidia_event_t));
+ }
+ NV_KFREE(nvfp, sizeof(nv_file_private_t));
+}
+
+
+/*
+** nvidia_open
+**
+** nv driver open entry point. Sessions are created here.
+*/
+static int
+nvidia_open(
+ struct inode *inode,
+ struct file *file
+)
+{
+ nv_state_t *nv = NULL;
+ nv_linux_state_t *nvl = NULL;
+ NvU32 minor_num;
+ int rc = 0;
+ nv_file_private_t *nvfp = NULL;
+ nv_stack_t *sp = NULL;
+#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
+ NvU32 msi_config = 0;
+#endif
+#if defined(NV_UVM_ENABLE) || defined(NV_UVM_NEXT_ENABLE)
+ NvU8 *uuid;
+#endif
+ unsigned int i;
+ unsigned int k;
+
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia_open...\n");
+
+ nvfp = nv_alloc_file_private();
+ if (nvfp == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate file private!\n");
+ return -ENOMEM;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ nv_free_file_private(nvfp);
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
+ {
+ NV_KMEM_CACHE_ALLOC_STACK(nvfp->fops_sp[i]);
+ if (nvfp->fops_sp[i] == NULL)
+ {
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ for (k = 0; k < i; ++k)
+ {
+ NV_KMEM_CACHE_FREE_STACK(nvfp->fops_sp[k]);
+ }
+ nv_free_file_private(nvfp);
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack[%d]\n", i);
+ return -ENOMEM;
+ }
+ }
+
+ /* what device are we talking about? */
+ minor_num = NV_DEVICE_MINOR_NUMBER(inode);
+
+ nvfp->minor_num = minor_num;
+
+ NV_SET_FILE_PRIVATE(file, nvfp);
+ nvfp->sp = sp;
+
+ /* for control device, just jump to its open routine */
+ /* after setting up the private data */
+ if (NV_IS_CONTROL_DEVICE(inode))
+ {
+ rc = nvidia_ctl_open(inode, file);
+ if (rc != 0)
+ goto failed2;
+ return rc;
+ }
+
+ LOCK_NV_LINUX_DEVICES();
+ nvl = nv_linux_devices;
+ while (nvl != NULL)
+ {
+ if (nvl->minor_num == minor_num)
+ break;
+ nvl = nvl->next;
+ }
+
+ if (nvl == NULL)
+ {
+ UNLOCK_NV_LINUX_DEVICES();
+ rc = -ENODEV;
+ goto failed2;
+ }
+
+ nv = NV_STATE_PTR(nvl);
+
+ down(&nvl->ldata_lock);
+
+ UNLOCK_NV_LINUX_DEVICES();
+
+ if (IS_VGX_HYPER())
+ {
+ /* fail open if GPU is being unbound */
+ if (nv->flags & NV_FLAG_UNBIND_LOCK)
+ {
+ rc = -ENODEV;
+ nv_printf(NV_DBG_ERRORS, "NVRM: nvidia_open on device %04x:%02x:%02x.0"
+ " failed as GPU is locked for unbind operation\n",
+ nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot);
+ goto failed;
+ }
+ }
+
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia_open on device "
+ "bearing minor number %d\n", minor_num);
+
+ NV_CHECK_PCI_CONFIG_SPACE(sp, nv, TRUE, TRUE, NV_MAY_SLEEP());
+
+ nvfp->nvptr = nvl;
+
+ /*
+ * map the memory and allocate isr on first open
+ */
+
+ if ( ! (nv->flags & NV_FLAG_OPEN))
+ {
+ if (nv->pci_info.device_id == 0)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: open of nonexistent "
+ "device bearing minor number %d\n", minor_num);
+ rc = -ENXIO;
+ goto failed;
+ }
+
+ if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
+ {
+ NV_KMEM_CACHE_ALLOC_STACK(nvl->isr_sp);
+ if (nvl->isr_sp == NULL)
+ {
+ rc = -ENOMEM;
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ goto failed;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(nvl->pci_cfgchk_sp);
+ if (nvl->pci_cfgchk_sp == NULL)
+ {
+ rc = -ENOMEM;
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ goto failed;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(nvl->isr_bh_sp);
+ if (nvl->isr_bh_sp == NULL)
+ {
+ rc = -ENOMEM;
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ goto failed;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(nvl->timer_sp);
+ if (nvl->timer_sp == NULL)
+ {
+ rc = -ENOMEM;
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ goto failed;
+ }
+ }
+
+#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
+ if (!NV_IS_GVI_DEVICE(nv))
+ {
+ if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
+ {
+ rm_read_registry_dword(sp, nv, "NVreg", NV_REG_ENABLE_MSI,
+ &msi_config);
+ if ((msi_config == 1) &&
+ (nv_find_pci_capability(nvl->dev, PCI_CAP_ID_MSI)))
+ {
+ rc = pci_enable_msi(nvl->dev);
+ if (rc == 0)
+ {
+ nv->interrupt_line = nvl->dev->irq;
+ nv->flags |= NV_FLAG_USES_MSI;
+ }
+ else
+ {
+ nv->flags &= ~NV_FLAG_USES_MSI;
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to enable MSI, \n"
+ "using PCIe virtual-wire interrupts.\n");
+ }
+ }
+ }
+ }
+#endif
+
+ if (NV_IS_GVI_DEVICE(nv))
+ {
+ rc = request_irq(nv->interrupt_line, nv_gvi_kern_isr,
+ IRQF_SHARED, NV_DEV_NAME, (void *)nvl);
+ if (rc == 0)
+ {
+ nvl->work.data = (void *)nvl;
+ NV_TASKQUEUE_INIT(&nvl->work.task, nv_gvi_kern_bh,
+ (void *)&nvl->work);
+ rm_init_gvi_device(sp, nv);
+ goto done;
+ }
+ }
+ else
+ {
+ rc = 0;
+ if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
+ {
+ rc = request_irq(nv->interrupt_line, nvidia_isr, IRQF_SHARED,
+ NV_DEV_NAME, (void *)nvl);
+ }
+ }
+ if (rc != 0)
+ {
+ if ((nv->interrupt_line != 0) && (rc == -EBUSY))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: Tried to get IRQ %d, but another driver\n",
+ (unsigned int) nv->interrupt_line);
+ nv_printf(NV_DBG_ERRORS, "NVRM: has it and is not sharing it.\n");
+ nv_printf(NV_DBG_ERRORS, "NVRM: You may want to verify that no audio driver");
+ nv_printf(NV_DBG_ERRORS, " is using the IRQ.\n");
+ }
+ nv_printf(NV_DBG_ERRORS, "NVRM: request_irq() failed (%d)\n", rc);
+ goto failed;
+ }
+
+ if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
+ {
+ tasklet_init(&nvl->tasklet, nvidia_isr_bh, (NvUPtr)NV_STATE_PTR(nvl));
+ }
+
+ if (!rm_init_adapter(sp, nv))
+ {
+ if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
+ {
+ tasklet_kill(&nvl->tasklet);
+ }
+ free_irq(nv->interrupt_line, (void *) nvl);
+ nv_printf(NV_DBG_ERRORS, "NVRM: rm_init_adapter failed "
+ "for device bearing minor number %d\n", minor_num);
+ rc = -EIO;
+ goto failed;
+ }
+
+#if defined(NV_UVM_ENABLE) || defined(NV_UVM_NEXT_ENABLE)
+ if (!NV_IS_GVI_DEVICE(nv))
+ {
+ if (rm_get_gpu_uuid_raw(sp, nv, &uuid, NULL) == RM_OK)
+ {
+ nv_uvm_notify_start_device(uuid);
+ os_free_mem(uuid);
+ }
+ }
+#endif
+#if !defined(NV_VMWARE) && \
+ (defined(NVCPU_X86) || defined(NVCPU_X86_64))
+ if (nv->primary_vga && (screen_info.orig_video_isVGA != 0x01))
+ {
+ if (!nv_fbdev_failure_detected)
+ {
+ nv_prints(NV_DBG_ERRORS, __fbdev_warning);
+ nv_procfs_add_warning("fbdev", __fbdev_warning);
+ }
+ nv_fbdev_failure_detected = 1;
+ }
+#endif
+
+done:
+ nv->flags |= NV_FLAG_OPEN;
+ }
+
+ NV_ATOMIC_INC(nvl->usage_count);
+
+failed:
+ if (rc != 0)
+ {
+#if defined(NV_LINUX_PCIE_MSI_SUPPORTED)
+ if (nv->flags & NV_FLAG_USES_MSI)
+ {
+ nv->flags &= ~NV_FLAG_USES_MSI;
+ NV_PCI_DISABLE_MSI(nvl->dev);
+ }
+#endif
+ if (nvl->timer_sp != NULL)
+ NV_KMEM_CACHE_FREE_STACK(nvl->timer_sp);
+ if (nvl->isr_bh_sp != NULL)
+ NV_KMEM_CACHE_FREE_STACK(nvl->isr_bh_sp);
+ if (nvl->pci_cfgchk_sp != NULL)
+ NV_KMEM_CACHE_FREE_STACK(nvl->pci_cfgchk_sp);
+ if (nvl->isr_sp != NULL)
+ NV_KMEM_CACHE_FREE_STACK(nvl->isr_sp);
+ }
+ up(&nvl->ldata_lock);
+failed2:
+ if (rc != 0)
+ {
+ if (nvfp != NULL)
+ {
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
+ {
+ NV_KMEM_CACHE_FREE_STACK(nvfp->fops_sp[i]);
+ }
+ nv_free_file_private(nvfp);
+ NV_SET_FILE_PRIVATE(file, NULL);
+ }
+ }
+
+ return rc;
+}
+
+/*
+** nvidia_close
+**
+** Master driver close entry point.
+*/
+
+static int
+nvidia_close(
+ struct inode *inode,
+ struct file *file
+)
+{
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_FILEP(file);
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+ nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(file);
+ nv_stack_t *sp = nvfp->sp;
+ unsigned int i;
+
+ NV_CHECK_PCI_CONFIG_SPACE(sp, nv, TRUE, TRUE, NV_MAY_SLEEP());
+
+ /* for control device, just jump to its open routine */
+ /* after setting up the private data */
+ if (NV_IS_CONTROL_DEVICE(inode))
+ return nvidia_ctl_close(inode, file);
+
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia_close on device "
+ "bearing minor number %d\n", NV_DEVICE_MINOR_NUMBER(inode));
+
+ rm_free_unused_clients(sp, nv, nvfp);
+
+ down(&nvl->ldata_lock);
+ if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
+ {
+ if (NV_IS_GVI_DEVICE(nv))
+ {
+ rm_shutdown_gvi_device(sp, nv);
+ NV_TASKQUEUE_FLUSH();
+ free_irq(nv->interrupt_line, (void *)nvl);
+ }
+ else
+ {
+#if defined(NV_UVM_ENABLE) || defined(NV_UVM_NEXT_ENABLE)
+ {
+ NvU8 *uuid;
+ // Inform UVM before disabling adapter
+ if(rm_get_gpu_uuid_raw(sp, nv, &uuid, NULL) == RM_OK)
+ {
+ // this function cannot fail
+ nv_uvm_notify_stop_device(uuid);
+ // get_uuid allocates memory for this call free it here
+ os_free_mem(uuid);
+ }
+ }
+#endif
+ if (nv->flags & NV_FLAG_PERSISTENT_SW_STATE)
+ {
+ rm_disable_adapter(sp, nv);
+ }
+ else
+ {
+ NV_SHUTDOWN_ADAPTER(sp, nv, nvl);
+ }
+ }
+
+ if (!(nv->flags & NV_FLAG_PERSISTENT_SW_STATE))
+ {
+ NV_KMEM_CACHE_FREE_STACK(nvl->timer_sp);
+ NV_KMEM_CACHE_FREE_STACK(nvl->isr_bh_sp);
+ NV_KMEM_CACHE_FREE_STACK(nvl->pci_cfgchk_sp);
+ NV_KMEM_CACHE_FREE_STACK(nvl->isr_sp);
+ }
+
+ /* leave INIT flag alone so we don't reinit every time */
+ nv->flags &= ~NV_FLAG_OPEN;
+ }
+ up(&nvl->ldata_lock);
+
+ for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
+ {
+ NV_KMEM_CACHE_FREE_STACK(nvfp->fops_sp[i]);
+ }
+
+ nv_free_file_private(nvfp);
+ NV_SET_FILE_PRIVATE(file, NULL);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+
+ return 0;
+}
+
+static unsigned int
+nvidia_poll(
+ struct file *file,
+ poll_table *wait
+)
+{
+ unsigned int mask = 0;
+ nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(file);
+ unsigned long eflags;
+
+ if ((file->f_flags & O_NONBLOCK) == 0)
+ poll_wait(file, &nvfp->waitqueue, wait);
+
+ NV_SPIN_LOCK_IRQSAVE(&nvfp->fp_lock, eflags);
+
+ if ((nvfp->event_head != NULL) || nvfp->event_pending)
+ {
+ mask = (POLLPRI | POLLIN);
+ nvfp->event_pending = FALSE;
+ }
+
+ NV_SPIN_UNLOCK_IRQRESTORE(&nvfp->fp_lock, eflags);
+
+ return mask;
+}
+
+#define NV_CTL_DEVICE_ONLY(nv) \
+{ \
+ if (((nv)->flags & NV_FLAG_CONTROL) == 0) \
+ { \
+ status = -EINVAL; \
+ goto done; \
+ } \
+}
+
+static int
+nvidia_ioctl(
+ struct inode *inode,
+ struct file *file,
+ unsigned int cmd,
+ unsigned long i_arg)
+{
+ RM_STATUS rmStatus;
+ int status = 0;
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_FILEP(file);
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+ nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(file);
+ nv_stack_t *sp = NULL;
+ nv_ioctl_xfer_t ioc_xfer;
+ void *arg_ptr = (void *) i_arg;
+ void *arg_copy = NULL;
+ size_t arg_size;
+ int arg_cmd;
+
+ nv_printf(NV_DBG_INFO, "NVRM: ioctl(0x%x, 0x%x, 0x%x)\n",
+ _IOC_NR(cmd), (unsigned int) i_arg, _IOC_SIZE(cmd));
+
+ down(&nvfp->fops_sp_lock[NV_FOPS_STACK_INDEX_IOCTL]);
+ sp = nvfp->fops_sp[NV_FOPS_STACK_INDEX_IOCTL];
+
+ NV_CHECK_PCI_CONFIG_SPACE(sp, nv, TRUE, TRUE, NV_MAY_SLEEP());
+
+ arg_size = _IOC_SIZE(cmd);
+ arg_cmd = _IOC_NR(cmd);
+
+ if (arg_cmd == NV_ESC_IOCTL_XFER_CMD)
+ {
+ if (arg_size != sizeof(nv_ioctl_xfer_t))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: invalid ioctl XFER structure size!\n");
+ status = -EINVAL;
+ goto done;
+ }
+
+ if (NV_COPY_FROM_USER(&ioc_xfer, arg_ptr, sizeof(ioc_xfer)))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to copy in ioctl XFER data!\n");
+ status = -EFAULT;
+ goto done;
+ }
+
+ arg_cmd = ioc_xfer.cmd;
+ arg_size = ioc_xfer.size;
+ arg_ptr = NvP64_VALUE(ioc_xfer.ptr);
+
+ if (arg_size > NV_ABSOLUTE_MAX_IOCTL_SIZE)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: invalid ioctl XFER size!\n");
+ status = -EINVAL;
+ goto done;
+ }
+ }
+
+ NV_KMALLOC(arg_copy, arg_size);
+ if (arg_copy == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate ioctl memory\n");
+ status = -ENOMEM;
+ goto done;
+ }
+
+ if (NV_COPY_FROM_USER(arg_copy, arg_ptr, arg_size))
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to copy in ioctl data!\n");
+ status = -EFAULT;
+ goto done;
+ }
+
+ switch (arg_cmd)
+ {
+ /* pass out info about the card */
+ case NV_ESC_CARD_INFO:
+ {
+ nv_ioctl_card_info_t *ci;
+ nv_linux_state_t *tnvl;
+ nv_ioctl_rm_api_old_version_t *rm_api;
+
+ NV_CTL_DEVICE_ONLY(nv);
+
+ if (arg_size < (sizeof(*ci) * num_nv_devices))
+ {
+ status = -EINVAL;
+ goto done;
+ }
+
+ /* the first element of card info passed from the client will have
+ * the rm_api_version_magic value to show that the client is new
+ * enough to support versioning. If the client is too old to
+ * support versioning, our mmap interfaces are probably different
+ * enough to cause serious damage.
+ * just copy in the one dword to check.
+ */
+ rm_api = arg_copy;
+ switch (rm_api->magic)
+ {
+ case NV_RM_API_OLD_VERSION_MAGIC_REQ:
+ case NV_RM_API_OLD_VERSION_MAGIC_LAX_REQ:
+ case NV_RM_API_OLD_VERSION_MAGIC_OVERRIDE_REQ:
+ /* the client is using the old major-minor-patch
+ * API version check; reject it.
+ */
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: API mismatch: the client has the version %d.%d-%d, but\n"
+ "NVRM: this kernel module has the version %s. Please\n"
+ "NVRM: make sure that this kernel module and all NVIDIA driver\n"
+ "NVRM: components have the same version.\n",
+ rm_api->major, rm_api->minor, rm_api->patch,
+ NV_VERSION_STRING);
+ status = -EINVAL;
+ goto done;
+
+ case NV_RM_API_OLD_VERSION_MAGIC_IGNORE:
+ /* the client is telling us to ignore the old
+ * version scheme; it will do a version check via
+ * NV_ESC_CHECK_VERSION_STR
+ */
+ break;
+ default:
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: client does not support versioning!!\n");
+ status = -EINVAL;
+ goto done;
+ }
+
+ ci = arg_copy;
+ memset(ci, 0, arg_size);
+ LOCK_NV_LINUX_DEVICES();
+ for (tnvl = nv_linux_devices; tnvl != NULL; tnvl = tnvl->next)
+ {
+ nv_state_t *tnv;
+ tnv = NV_STATE_PTR(tnvl);
+ if (tnv->pci_info.device_id)
+ {
+ ci->flags = NV_IOCTL_CARD_INFO_FLAG_PRESENT;
+ ci->pci_info.domain = tnv->pci_info.domain;
+ ci->pci_info.bus = tnv->pci_info.bus;
+ ci->pci_info.slot = tnv->pci_info.slot;
+ ci->pci_info.vendor_id = tnv->pci_info.vendor_id;
+ ci->pci_info.device_id = tnv->pci_info.device_id;
+ ci->gpu_id = tnv->gpu_id;
+ ci->interrupt_line = tnv->interrupt_line;
+ ci->reg_address = tnv->regs->address;
+ ci->reg_size = tnv->regs->size;
+ ci->fb_address = tnv->fb->address;
+ ci->fb_size = tnv->fb->size;
+ ci->minor_number = tnvl->minor_num;
+ ci++;
+ }
+ }
+ UNLOCK_NV_LINUX_DEVICES();
+ break;
+ }
+
+ case NV_ESC_CHECK_VERSION_STR:
+ {
+ NV_CTL_DEVICE_ONLY(nv);
+
+ rmStatus = rm_perform_version_check(sp, arg_copy, arg_size);
+ status = ((rmStatus == RM_OK) ? 0 : -EINVAL);
+ break;
+ }
+
+ default:
+ rmStatus = rm_ioctl(sp, nv, nvfp, arg_cmd, arg_copy, arg_size);
+ status = ((rmStatus == RM_OK) ? 0 : -EINVAL);
+ break;
+ }
+
+done:
+ up(&nvfp->fops_sp_lock[NV_FOPS_STACK_INDEX_IOCTL]);
+
+ if (arg_copy != NULL)
+ {
+ if (status != -EFAULT)
+ {
+ if (NV_COPY_TO_USER(arg_ptr, arg_copy, arg_size))
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to copy out ioctl data\n");
+ status = -EFAULT;
+ }
+ }
+ NV_KFREE(arg_copy, arg_size);
+ }
+
+ return status;
+}
+
+static long
+nvidia_unlocked_ioctl(
+ struct file *file,
+ unsigned int cmd,
+ unsigned long i_arg
+)
+{
+ return nvidia_ioctl(file->f_dentry->d_inode, file, cmd, i_arg);
+}
+
+/*
+ * driver receives an interrupt
+ * if someone waiting, then hand it off.
+ */
+static irqreturn_t
+nvidia_isr(
+ int irq,
+ void *arg
+#if !defined(NV_IRQ_HANDLER_T_PRESENT) || (NV_IRQ_HANDLER_T_ARGUMENT_COUNT == 3)
+ ,struct pt_regs *regs
+#endif
+)
+{
+ nv_linux_state_t *nvl = (void *) arg;
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+ NvU32 need_to_run_bottom_half = 0;
+ BOOL rm_handled = FALSE, uvm_handled = FALSE;
+
+#if defined (NV_UVM_NEXT_ENABLE)
+ //
+ // Returns RM_OK if the UVM driver handled the interrupt
+ // Returns RM_ERR_NO_INTR_PENDING if the interrupt is not for the UVM driver
+ //
+ if (nv_uvm_event_interrupt() == RM_OK)
+ uvm_handled = TRUE;
+#endif
+
+ rm_handled = rm_isr(nvl->isr_sp, nv, &need_to_run_bottom_half);
+ if (need_to_run_bottom_half)
+ {
+ tasklet_schedule(&nvl->tasklet);
+ }
+
+ return IRQ_RETVAL(rm_handled || uvm_handled);
+}
+
+static void
+nvidia_isr_bh(
+ unsigned long data
+)
+{
+ nv_state_t *nv = (nv_state_t *) data;
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+
+ NV_CHECK_PCI_CONFIG_SPACE(nvl->isr_bh_sp, nv, TRUE, FALSE, FALSE);
+ rm_isr_bh(nvl->isr_bh_sp, nv);
+}
+
+static void
+nvidia_rc_timer(
+ unsigned long data
+)
+{
+ nv_linux_state_t *nvl = (nv_linux_state_t *) data;
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+
+ NV_CHECK_PCI_CONFIG_SPACE(nvl->timer_sp, nv, TRUE, TRUE, FALSE);
+
+ if (rm_run_rc_callback(nvl->timer_sp, nv) == RM_OK)
+ mod_timer(&nvl->rc_timer, jiffies + HZ); /* set another timeout in 1 second */
+}
+
+/*
+** nvidia_ctl_open
+**
+** nv control driver open entry point. Sessions are created here.
+*/
+static int
+nvidia_ctl_open(
+ struct inode *inode,
+ struct file *file
+)
+{
+ nv_linux_state_t *nvl = &nv_ctl_device;
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+ nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(file);
+ static int count = 0;
+
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_open\n");
+
+ down(&nvl->ldata_lock);
+
+ /* save the nv away in file->private_data */
+ nvfp->nvptr = nvl;
+
+ if (NV_ATOMIC_READ(nvl->usage_count) == 0)
+ {
+ init_waitqueue_head(&nv_ctl_waitqueue);
+
+ nv->flags |= (NV_FLAG_OPEN | NV_FLAG_CONTROL);
+
+ if ((nv_acpi_init() < 0) &&
+ (count++ < NV_MAX_RECURRING_WARNING_MESSAGES))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to register with the ACPI subsystem!\n");
+ }
+ }
+
+ NV_ATOMIC_INC(nvl->usage_count);
+ up(&nvl->ldata_lock);
+
+ return 0;
+}
+
+
+/*
+** nvidia_ctl_close
+*/
+static int
+nvidia_ctl_close(
+ struct inode *inode,
+ struct file *file
+)
+{
+ nv_alloc_t *at, *next;
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_FILEP(file);
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+ nv_file_private_t *nvfp = NV_GET_FILE_PRIVATE(file);
+ nv_stack_t *sp = nvfp->sp;
+ static int count = 0;
+ unsigned int i;
+
+ nv_printf(NV_DBG_INFO, "NVRM: nvidia_ctl_close\n");
+
+ down(&nvl->ldata_lock);
+ if (NV_ATOMIC_DEC_AND_TEST(nvl->usage_count))
+ {
+ nv->flags &= ~NV_FLAG_OPEN;
+
+ if ((nv_acpi_uninit() < 0) &&
+ (count++ < NV_MAX_RECURRING_WARNING_MESSAGES))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to unregister from the ACPI subsystem!\n");
+ }
+ }
+ up(&nvl->ldata_lock);
+
+ rm_free_unused_clients(sp, nv, nvfp);
+
+ if (nvfp->free_list != NULL)
+ {
+ at = nvfp->free_list;
+ while (at != NULL)
+ {
+ next = at->next;
+ if (at->pid == os_get_current_process())
+ NV_PRINT_AT(NV_DBG_MEMINFO, at);
+ nv_free_pages(nv, at->num_pages,
+ NV_ALLOC_MAPPING_CONTIG(at->flags),
+ NV_ALLOC_MAPPING(at->flags),
+ (void *)at);
+ at = next;
+ }
+ }
+
+ for (i = 0; i < NV_FOPS_STACK_INDEX_COUNT; ++i)
+ {
+ NV_KMEM_CACHE_FREE_STACK(nvfp->fops_sp[i]);
+ }
+
+ nv_free_file_private(nvfp);
+ NV_SET_FILE_PRIVATE(file, NULL);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+
+ return 0;
+}
+
+
+void NV_API_CALL nv_set_dma_address_size(
+ nv_state_t *nv,
+ NvU32 phys_addr_bits
+)
+{
+ nv_linux_state_t *nvl;
+
+ nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ nvl->dev->dma_mask = (((u64)1) << phys_addr_bits) - 1;
+}
+
+#if defined(NV_VMAP_PRESENT)
+static unsigned long
+nv_map_guest_pages(nv_alloc_t *at,
+ NvU64 address,
+ NvU32 page_count,
+ NvU32 page_idx)
+{
+ struct page **pages;
+ NvU32 j;
+ unsigned long virt_addr;
+
+ NV_KMALLOC(pages, sizeof(struct page *) * page_count);
+ if (pages == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to allocate vmap() page descriptor table!\n");
+ return 0;
+ }
+
+ for (j = 0; j < page_count; j++)
+ {
+ pages[j] = NV_GET_PAGE_STRUCT(at->page_table[page_idx+j]->phys_addr);
+ }
+
+ NV_VMAP(virt_addr, pages, page_count, NV_ALLOC_MAPPING_CACHED(at->flags));
+ NV_KFREE(pages, sizeof(struct page *) * page_count);
+
+ return virt_addr;
+}
+#endif
+
+RM_STATUS NV_API_CALL
+nv_alias_pages(
+ nv_state_t *nv,
+ NvU32 page_cnt,
+ NvU32 contiguous,
+ NvU32 cache_type,
+ NvU64 guest_id,
+ NvU64 *pte_array,
+ void **priv_data
+)
+{
+ nv_alloc_t *at;
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ NvU32 i=0;
+ nv_pte_t *page_ptr = NULL;
+
+ page_cnt = RM_PAGES_TO_OS_PAGES(page_cnt);
+ at = nvos_create_alloc(nvl->dev, page_cnt);
+
+ if (at == NULL)
+ {
+ return RM_ERR_NO_FREE_MEM;
+ }
+
+ at->flags = nv_alloc_init_flags(cache_type, contiguous, 0);
+ at->flags |= NV_ALLOC_TYPE_GUEST;
+
+ at->order = nv_calc_order(at->num_pages * PAGE_SIZE);
+
+ for (i=0; i < at->num_pages; ++i)
+ {
+ page_ptr = at->page_table[i];
+
+ if (contiguous && i>0)
+ {
+ page_ptr->dma_addr = pte_array[0] + (i << PAGE_SHIFT);
+ }
+ else
+ {
+ page_ptr->dma_addr = pte_array[i];
+ }
+
+ page_ptr->phys_addr = page_ptr->dma_addr;
+
+ /* aliased pages will be mapped on demand. */
+ page_ptr->virt_addr = 0x0;
+ }
+
+ at->guest_id = guest_id;
+ *priv_data = at;
+ NV_ATOMIC_INC(at->usage_count);
+
+ NV_PRINT_AT(NV_DBG_MEMINFO, at);
+
+ return RM_OK;
+}
+
+void* NV_API_CALL nv_alloc_kernel_mapping(
+ nv_state_t *nv,
+ void *pAllocPrivate,
+ NvU64 pageIndex,
+ NvU32 pageOffset,
+ NvU64 size,
+ void **pPrivate
+)
+{
+ nv_alloc_t *at = pAllocPrivate;
+#if defined(NV_VMAP_PRESENT)
+ NvU32 j, page_count;
+ unsigned long virt_addr;
+ struct page **pages;
+#endif
+
+ if (((size + pageOffset) <= PAGE_SIZE) &&
+ !NV_ALLOC_MAPPING_GUEST(at->flags) && !NV_ALLOC_MAPPING_ALIASED(at->flags))
+ {
+ *pPrivate = NULL;
+ return (void *)(at->page_table[pageIndex]->virt_addr + pageOffset);
+ }
+ else
+ {
+#if defined(NV_VMAP_PRESENT)
+ size += pageOffset;
+ page_count = (size >> PAGE_SHIFT) + ((size & ~NV_PAGE_MASK) ? 1 : 0);
+
+ if (NV_ALLOC_MAPPING_GUEST(at->flags))
+ {
+ virt_addr = nv_map_guest_pages(at,
+ nv->bars[NV_GPU_BAR_INDEX_REGS].address,
+ page_count, pageIndex);
+ }
+ else
+ {
+ NV_KMALLOC(pages, sizeof(struct page *) * page_count);
+ if (pages == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to allocate vmap() page descriptor table!\n");
+ return NULL;
+ }
+
+ for (j = 0; j < page_count; j++)
+ pages[j] = NV_GET_PAGE_STRUCT(at->page_table[pageIndex+j]->phys_addr);
+
+ NV_VMAP(virt_addr, pages, page_count, NV_ALLOC_MAPPING_CACHED(at->flags));
+ NV_KFREE(pages, sizeof(struct page *) * page_count);
+ }
+
+ if (virt_addr == 0)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to map pages!\n");
+ return NULL;
+ }
+
+ *pPrivate = (void *)(NvUPtr)page_count;
+ return (void *)(virt_addr + pageOffset);
+#else
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This version of the Linux kernel does not provide the vmap()\n"
+ "NVRM: kernel interface. If you see this message, please update\n"
+ "NVRM: your kernel to Linux 2.4.22 or install a distribution kernel\n"
+ "NVRM: that supports the vmap() kernel interface.\n");
+#endif
+ }
+
+ return NULL;
+}
+
+RM_STATUS NV_API_CALL nv_free_kernel_mapping(
+ nv_state_t *nv,
+ void *pAllocPrivate,
+ void *address,
+ void *pPrivate
+)
+{
+#if defined(NV_VMAP_PRESENT)
+ nv_alloc_t *at = pAllocPrivate;
+ unsigned long virt_addr;
+ NvU32 page_count;
+
+ virt_addr = ((NvUPtr)address & NV_PAGE_MASK);
+ page_count = (NvUPtr)pPrivate;
+
+ if (NV_ALLOC_MAPPING_GUEST(at->flags))
+ {
+ NV_IOUNMAP((void *)virt_addr, (page_count * PAGE_SIZE));
+ }
+ else if (pPrivate != NULL)
+ {
+ NV_VUNMAP(virt_addr, page_count);
+ }
+#endif
+ return RM_OK;
+}
+
+RM_STATUS NV_API_CALL nv_alloc_pages(
+ nv_state_t *nv,
+ NvU32 page_count,
+ NvBool contiguous,
+ NvU32 cache_type,
+ NvBool zeroed,
+ NvU64 *pte_array,
+ void **priv_data
+)
+{
+ nv_alloc_t *at;
+ RM_STATUS status = RM_ERR_NO_FREE_MEM;
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ NvU32 i, memory_type;
+
+ nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_alloc_pages: %d pages\n", page_count);
+ nv_printf(NV_DBG_MEMINFO, "NVRM: VM: contig %d cache_type %d\n",
+ contiguous, cache_type);
+
+ memory_type = NV_MEMORY_TYPE_SYSTEM;
+
+#if !defined(NV_VMWARE)
+ if (nv_encode_caching(NULL, cache_type, memory_type))
+ return RM_ERR_NOT_SUPPORTED;
+#endif
+
+ page_count = RM_PAGES_TO_OS_PAGES(page_count);
+ at = nvos_create_alloc(nvl->dev, page_count);
+ if (at == NULL)
+ return RM_ERR_NO_FREE_MEM;
+
+ at->flags = nv_alloc_init_flags(cache_type, contiguous, zeroed);
+
+ if (NV_ALLOC_MAPPING_CONTIG(at->flags))
+ status = nv_alloc_contig_pages(nv, at);
+ else
+ status = nv_alloc_system_pages(nv, at);
+
+ if (status != RM_OK)
+ goto failed;
+
+ for (i = 0; i < ((contiguous) ? 1 : page_count); i++)
+ pte_array[i] = at->page_table[i]->dma_addr;
+
+ *priv_data = at;
+ NV_ATOMIC_INC(at->usage_count);
+
+ NV_PRINT_AT(NV_DBG_MEMINFO, at);
+
+ return RM_OK;
+
+failed:
+ nvos_free_alloc(at);
+
+ return status;
+}
+
+RM_STATUS NV_API_CALL nv_free_pages(
+ nv_state_t *nv,
+ NvU32 page_count,
+ NvBool contiguous,
+ NvU32 cache_type,
+ void *priv_data
+)
+{
+ RM_STATUS rmStatus = RM_OK;
+ nv_alloc_t *at = priv_data;
+
+ page_count = RM_PAGES_TO_OS_PAGES(page_count);
+ nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_free_pages: 0x%x\n", page_count);
+
+ NV_PRINT_AT(NV_DBG_MEMINFO, at);
+
+ /*
+ * If the 'at' usage count doesn't drop to zero here, not all of
+ * the user mappings have been torn down in time - we can't
+ * safely free the memory. We report success back to the RM, but
+ * defer the actual free operation until later.
+ *
+ * This is described in greater detail in the comments above the
+ * nvidia_vma_(open|release)() callbacks in nv-mmap.c.
+ */
+ if (!NV_ATOMIC_DEC_AND_TEST(at->usage_count))
+ return RM_OK;
+
+ if (!NV_ALLOC_MAPPING_GUEST(at->flags))
+ {
+ if (NV_ALLOC_MAPPING_CONTIG(at->flags))
+ nv_free_contig_pages(nv, at);
+ else
+ nv_free_system_pages(nv, at);
+ }
+
+ nvos_free_alloc(at);
+
+ return rmStatus;
+}
+
+static void nv_lock_init_locks
+(
+ nv_state_t *nv
+)
+{
+ nv_linux_state_t *nvl;
+ nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+
+ NV_INIT_MUTEX(&nvl->ldata_lock);
+
+ NV_ATOMIC_SET(nvl->usage_count, 0);
+}
+
+void NV_API_CALL nv_post_event(
+ nv_state_t *nv,
+ nv_event_t *event,
+ NvU32 handle,
+ NvU32 index,
+ NvBool data_valid
+)
+{
+ nv_file_private_t *nvfp = event->file;
+ unsigned long eflags;
+ nvidia_event_t *nvet;
+
+ NV_SPIN_LOCK_IRQSAVE(&nvfp->fp_lock, eflags);
+
+ if (data_valid)
+ {
+ NV_KMALLOC_ATOMIC(nvet, sizeof(nvidia_event_t));
+ if (nvet == NULL)
+ {
+ NV_SPIN_UNLOCK_IRQRESTORE(&nvfp->fp_lock, eflags);
+ return;
+ }
+
+ if (nvfp->event_tail != NULL)
+ nvfp->event_tail->next = nvet;
+ if (nvfp->event_head == NULL)
+ nvfp->event_head = nvet;
+ nvfp->event_tail = nvet;
+ nvet->next = NULL;
+
+ nvet->event = *event;
+ nvet->event.hObject = handle;
+ nvet->event.index = index;
+ }
+
+ nvfp->event_pending = TRUE;
+ wake_up_interruptible(&nvfp->waitqueue);
+
+ NV_SPIN_UNLOCK_IRQRESTORE(&nvfp->fp_lock, eflags);
+}
+
+int NV_API_CALL nv_get_event(
+ nv_state_t *nv,
+ void *file,
+ nv_event_t *event,
+ NvU32 *pending
+)
+{
+ nv_file_private_t *nvfp = file;
+ nvidia_event_t *nvet;
+ unsigned long eflags;
+
+ NV_SPIN_LOCK_IRQSAVE(&nvfp->fp_lock, eflags);
+
+ nvet = nvfp->event_head;
+ if (nvet == NULL)
+ {
+ NV_SPIN_UNLOCK_IRQRESTORE(&nvfp->fp_lock, eflags);
+ return RM_ERROR;
+ }
+
+ *event = nvet->event;
+
+ if (nvfp->event_tail == nvet)
+ nvfp->event_tail = NULL;
+ nvfp->event_head = nvet->next;
+
+ *pending = (nvfp->event_head != NULL);
+
+ NV_SPIN_UNLOCK_IRQRESTORE(&nvfp->fp_lock, eflags);
+
+ NV_KFREE(nvet, sizeof(nvidia_event_t));
+
+ return RM_OK;
+}
+
+int NV_API_CALL nv_start_rc_timer(
+ nv_state_t *nv
+)
+{
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+
+ if (nv->rc_timer_enabled)
+ return -1;
+
+ nv_printf(NV_DBG_INFO, "NVRM: initializing rc timer\n");
+ init_timer(&nvl->rc_timer);
+ nvl->rc_timer.function = nvidia_rc_timer;
+ nvl->rc_timer.data = (unsigned long) nvl;
+ nv->rc_timer_enabled = 1;
+ mod_timer(&nvl->rc_timer, jiffies + HZ); /* set our timeout for 1 second */
+ nv_printf(NV_DBG_INFO, "NVRM: rc timer initialized\n");
+
+ return 0;
+}
+
+int NV_API_CALL nv_stop_rc_timer(
+ nv_state_t *nv
+)
+{
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+
+ if (!nv->rc_timer_enabled)
+ return -1;
+
+ nv_printf(NV_DBG_INFO, "NVRM: stopping rc timer\n");
+ nv->rc_timer_enabled = 0;
+ del_timer_sync(&nvl->rc_timer);
+ nv_printf(NV_DBG_INFO, "NVRM: rc timer stopped\n");
+
+ return 0;
+}
+
+static void
+nvos_validate_assigned_gpus(struct pci_dev *dev)
+{
+ NvU32 i;
+
+ if (NV_IS_ASSIGN_GPU_PCI_INFO_SPECIFIED())
+ {
+ for (i = 0; i < nv_assign_gpu_count; i++)
+ {
+ if ((nv_assign_gpu_pci_info[i].domain == NV_PCI_DOMAIN_NUMBER(dev)) &&
+ (nv_assign_gpu_pci_info[i].bus == NV_PCI_BUS_NUMBER(dev)) &&
+ (nv_assign_gpu_pci_info[i].slot == NV_PCI_SLOT_NUMBER(dev)))
+ {
+ nv_assign_gpu_pci_info[i].valid = NV_TRUE;
+ return;
+ }
+ }
+ }
+}
+
+/* make sure the pci_driver called probe for all of our devices.
+ * we've seen cases where rivafb claims the device first and our driver
+ * doesn't get called.
+ */
+static int
+nvos_count_devices(nv_stack_t *sp)
+{
+ struct pci_dev *dev;
+ int count = 0;
+
+ dev = NV_PCI_GET_CLASS(PCI_CLASS_DISPLAY_VGA << 8, NULL);
+ while (dev)
+ {
+ if ((dev->vendor == 0x10de) && (dev->device >= 0x20) &&
+ !rm_is_legacy_device(sp, dev->device, TRUE))
+ {
+ count++;
+ nvos_validate_assigned_gpus(dev);
+ }
+ dev = NV_PCI_GET_CLASS(PCI_CLASS_DISPLAY_VGA << 8, dev);
+ }
+
+ dev = NV_PCI_GET_CLASS(PCI_CLASS_DISPLAY_3D << 8, NULL);
+ while (dev)
+ {
+ if ((dev->vendor == 0x10de) && (dev->device >= 0x20) &&
+ !rm_is_legacy_device(sp, dev->device, TRUE))
+ {
+ count++;
+ nvos_validate_assigned_gpus(dev);
+ }
+ dev = NV_PCI_GET_CLASS(PCI_CLASS_DISPLAY_3D << 8, dev);
+ }
+
+ dev = NV_PCI_GET_CLASS(PCI_CLASS_MULTIMEDIA_OTHER << 8, NULL);
+ while (dev)
+ {
+ if ((dev->vendor == 0x10de) && (dev->device == 0x0e00))
+ count++;
+ dev = NV_PCI_GET_CLASS(PCI_CLASS_MULTIMEDIA_OTHER << 8, dev);
+ }
+
+ if (NV_IS_ASSIGN_GPU_PCI_INFO_SPECIFIED())
+ {
+ NvU32 i;
+
+ for (i = 0; i < nv_assign_gpu_count; i++)
+ {
+ if (nv_assign_gpu_pci_info[i].valid == NV_TRUE)
+ count++;
+ }
+ }
+
+ return count;
+}
+
+/* find nvidia devices and set initial state */
+static int
+nvidia_probe
+(
+ struct pci_dev *dev,
+ const struct pci_device_id *id_table
+)
+{
+ nv_state_t *nv;
+ nv_linux_state_t *nvl = NULL;
+ unsigned int i, j;
+ int flags = 0;
+ nv_stack_t *sp = NULL;
+
+ if (NV_IS_SMU_DEVICE(dev))
+ {
+ return nvidia_smu_probe(dev);
+ }
+
+ nv_printf(NV_DBG_SETUP, "NVRM: probing 0x%x 0x%x, class 0x%x\n",
+ dev->vendor, dev->device, dev->class);
+
+ if ((dev->class == (PCI_CLASS_MULTIMEDIA_OTHER << 8)) &&
+ (dev->device == 0x0e00))
+ {
+ flags = NV_FLAG_GVI;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ return -1;
+ }
+
+ if (!(flags & NV_FLAG_GVI))
+ {
+ if ((dev->vendor != 0x10de) || (dev->device < 0x20) ||
+ ((dev->class != (PCI_CLASS_DISPLAY_VGA << 8)) &&
+ (dev->class != (PCI_CLASS_DISPLAY_3D << 8))) ||
+ rm_is_legacy_device(sp, dev->device, FALSE))
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: ignoring the legacy GPU %04x:%02x:%02x.%x\n",
+ NV_PCI_DOMAIN_NUMBER(dev), NV_PCI_BUS_NUMBER(dev), NV_PCI_SLOT_NUMBER(dev),
+ PCI_FUNC(dev->devfn));
+ goto failed;
+ }
+ }
+
+ if (NV_IS_ASSIGN_GPU_PCI_INFO_SPECIFIED())
+ {
+ for (i = 0; i < nv_assign_gpu_count; i++)
+ {
+ if (((nv_assign_gpu_pci_info[i].domain == NV_PCI_DOMAIN_NUMBER(dev)) &&
+ (nv_assign_gpu_pci_info[i].bus == NV_PCI_BUS_NUMBER(dev)) &&
+ (nv_assign_gpu_pci_info[i].slot == NV_PCI_SLOT_NUMBER(dev))) &&
+ (nv_assign_gpu_pci_info[i].valid))
+ break;
+ }
+
+ if (i == nv_assign_gpu_count)
+ {
+ goto failed;
+ }
+ }
+
+ num_probed_nv_devices++;
+
+ if (pci_enable_device(dev) != 0)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: pci_enable_device failed, aborting\n");
+ goto failed;
+ }
+
+ if (dev->irq == 0)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: Can't find an IRQ for your NVIDIA card!\n");
+ nv_printf(NV_DBG_ERRORS, "NVRM: Please check your BIOS settings.\n");
+ nv_printf(NV_DBG_ERRORS, "NVRM: [Plug & Play OS] should be set to NO\n");
+ nv_printf(NV_DBG_ERRORS, "NVRM: [Assign IRQ to VGA] should be set to YES \n");
+ goto failed;
+ }
+
+ for (i = 0; i < (NV_GPU_NUM_BARS - 1); i++)
+ {
+ if (NV_PCI_RESOURCE_VALID(dev, i))
+ {
+#if defined(NV_PCI_MAX_MMIO_BITS_SUPPORTED)
+ if ((NV_PCI_RESOURCE_FLAGS(dev, i) & PCI_BASE_ADDRESS_MEM_TYPE_64) &&
+ ((NV_PCI_RESOURCE_START(dev, i) >> NV_PCI_MAX_MMIO_BITS_SUPPORTED)))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This is a 64-bit BAR mapped above %dGB by the system\n"
+ "NVRM: BIOS or the %s kernel. This PCI I/O region assigned\n"
+ "NVRM: to your NVIDIA device is not supported by the kernel.\n"
+ "NVRM: BAR%d is %dM @ 0x%llx (PCI:%04x:%02x:%02x.%x)\n",
+ (1 << (NV_PCI_MAX_MMIO_BITS_SUPPORTED - 30)),
+ NV_KERNEL_NAME, i,
+ (NV_PCI_RESOURCE_SIZE(dev, i) >> 20),
+ (NvU64)NV_PCI_RESOURCE_START(dev, i),
+ NV_PCI_DOMAIN_NUMBER(dev),
+ NV_PCI_BUS_NUMBER(dev), NV_PCI_SLOT_NUMBER(dev),
+ PCI_FUNC(dev->devfn));
+ goto failed;
+ }
+#endif
+#if !defined(NV_VMWARE)
+ if ((NV_PCI_RESOURCE_FLAGS(dev, i) & PCI_BASE_ADDRESS_MEM_TYPE_64) &&
+ ((NV_PCI_RESOURCE_START(dev, i) >> PAGE_SHIFT) > 0xfffffULL))
+ {
+ struct pci_dev *bridge = dev->bus->self;
+ NvU32 base_upper, limit_upper;
+
+ pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32,
+ &base_upper);
+ pci_read_config_dword(bridge, PCI_PREF_LIMIT_UPPER32,
+ &limit_upper);
+
+ if ((base_upper != 0) && (limit_upper != 0))
+ continue;
+
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This is a 64-bit BAR mapped above 4GB by the system\n"
+ "NVRM: BIOS or the %s kernel, but the PCI bridge\n"
+ "NVRM: immediately upstream of this GPU does not define\n"
+ "NVRM: a matching prefetchable memory window.\n",
+ NV_KERNEL_NAME);
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This may be due to a known Linux kernel bug. Please\n"
+ "NVRM: see the README section on 64-bit BARs for additional\n"
+ "NVRM: information.\n");
+ goto failed;
+ }
+#endif
+ continue;
+ }
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This PCI I/O region assigned to your NVIDIA device is invalid:\n"
+ "NVRM: BAR%d is %dM @ 0x%llx (PCI:%04x:%02x:%02x.%x)\n", i,
+ (NV_PCI_RESOURCE_SIZE(dev, i) >> 20),
+ (NvU64)NV_PCI_RESOURCE_START(dev, i),
+ NV_PCI_DOMAIN_NUMBER(dev),
+ NV_PCI_BUS_NUMBER(dev), NV_PCI_SLOT_NUMBER(dev), PCI_FUNC(dev->devfn));
+#if defined(NVCPU_X86)
+ if ((NV_PCI_RESOURCE_FLAGS(dev, i) & PCI_BASE_ADDRESS_MEM_TYPE_64) &&
+ ((NV_PCI_RESOURCE_START(dev, i) >> PAGE_SHIFT) > 0xfffffULL))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This is a 64-bit BAR mapped above 4GB by the system\n"
+ "NVRM: BIOS or the Linux kernel. The NVIDIA Linux/x86\n"
+ "NVRM: graphics driver and other system software components\n"
+ "NVRM: do not support this configuration.\n");
+ }
+ else
+#endif
+ if (NV_PCI_RESOURCE_FLAGS(dev, i) & PCI_BASE_ADDRESS_MEM_TYPE_64)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: This is a 64-bit BAR, which some operating system\n"
+ "NVRM: kernels and other system software components are known\n"
+ "NVRM: to handle incorrectly. Please see the README section\n"
+ "NVRM: on 64-bit BARs for more information.\n");
+ }
+ else
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The system BIOS may have misconfigured your GPU.\n");
+ }
+ goto failed;
+ }
+
+ if (!request_mem_region(NV_PCI_RESOURCE_START(dev, NV_GPU_BAR_INDEX_REGS),
+ NV_PCI_RESOURCE_SIZE(dev, NV_GPU_BAR_INDEX_REGS), NV_DEV_NAME))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: request_mem_region failed for %dM @ 0x%llx. This can\n"
+ "NVRM: occur when a driver such as rivatv is loaded and claims\n"
+ "NVRM: ownership of the device's registers.\n",
+ (NV_PCI_RESOURCE_SIZE(dev, NV_GPU_BAR_INDEX_REGS) >> 20),
+ (NvU64)NV_PCI_RESOURCE_START(dev, NV_GPU_BAR_INDEX_REGS));
+ goto failed;
+ }
+
+ NV_KMALLOC(nvl, sizeof(nv_linux_state_t));
+ if (nvl == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate memory\n");
+ goto err_not_supported;
+ }
+
+ os_mem_set(nvl, 0, sizeof(nv_linux_state_t));
+
+ nv = NV_STATE_PTR(nvl);
+
+ pci_set_drvdata(dev, (void *)nvl);
+
+ /* default to 32-bit PCI bus address space */
+ dev->dma_mask = 0xffffffffULL;
+
+ nvl->dev = dev;
+ nv->pci_info.vendor_id = dev->vendor;
+ nv->pci_info.device_id = dev->device;
+ nv->subsystem_id = dev->subsystem_device;
+ nv->os_state = (void *) nvl;
+ nv->pci_info.domain = NV_PCI_DOMAIN_NUMBER(dev);
+ nv->pci_info.bus = NV_PCI_BUS_NUMBER(dev);
+ nv->pci_info.slot = NV_PCI_SLOT_NUMBER(dev);
+ nv->handle = dev;
+ nv->flags |= flags;
+
+ if (NVOS_IS_VMWARE)
+ {
+ nvl->minor_num = num_nv_devices;
+ }
+
+ nv_lock_init_locks(nv);
+
+ for (i = 0, j = 0; i < NVRM_PCICFG_NUM_BARS && j < NV_GPU_NUM_BARS; i++)
+ {
+ if ((NV_PCI_RESOURCE_VALID(dev, i)) &&
+ (NV_PCI_RESOURCE_FLAGS(dev, i) & PCI_BASE_ADDRESS_SPACE)
+ == PCI_BASE_ADDRESS_SPACE_MEMORY)
+ {
+ nv->bars[j].address = NV_PCI_RESOURCE_START(dev, i);
+ nv->bars[j].strapped_size = NV_PCI_RESOURCE_SIZE(dev, i);
+ nv->bars[j].size = nv->bars[j].strapped_size;
+ nv->bars[j].offset = NVRM_PCICFG_BAR_OFFSET(i);
+ j++;
+ }
+ }
+ nv->regs = &nv->bars[NV_GPU_BAR_INDEX_REGS];
+ nv->fb = &nv->bars[NV_GPU_BAR_INDEX_FB];
+
+ nv->interrupt_line = dev->irq;
+
+ pci_set_master(dev);
+
+#if defined(CONFIG_VGA_ARB)
+#if defined(VGA_DEFAULT_DEVICE)
+ vga_tryget(VGA_DEFAULT_DEVICE, VGA_RSRC_LEGACY_MASK);
+#endif
+ vga_set_legacy_decoding(dev, VGA_RSRC_NONE);
+#endif
+
+ if (NV_IS_GVI_DEVICE(nv))
+ {
+ if (!rm_gvi_init_private_state(sp, nv))
+ {
+ nv_printf(NV_DBG_ERRORS, "NVGVI: rm_init_gvi_private_state() failed!\n");
+ goto err_not_supported;
+ }
+
+ if (rm_gvi_attach_device(sp, nv) != RM_OK)
+ {
+ rm_gvi_free_private_state(sp, nv);
+ goto err_not_supported;
+ }
+ }
+ else
+ {
+ NvU32 pmc_boot_0, pmc_boot_42;
+ RM_STATUS status;
+
+ NV_CHECK_PCI_CONFIG_SPACE(sp, nv, FALSE, TRUE, NV_MAY_SLEEP());
+
+ if ((status = rm_is_supported_device(sp, nv, &pmc_boot_0, &pmc_boot_42)) != RM_OK)
+ {
+ if ((status != RM_ERR_NOT_SUPPORTED) ||
+ !rm_is_legacy_arch(pmc_boot_0, pmc_boot_42))
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: The NVIDIA GPU %04x:%02x:%02x.%x (PCI ID: %04x:%04x)\n"
+ "NVRM: installed in this system is not supported by the %s\n"
+ "NVRM: NVIDIA %s driver release. Please see 'Appendix\n"
+ "NVRM: A - Supported NVIDIA GPU Products' in this release's\n"
+ "NVRM: README, available on the %s driver download page\n"
+ "NVRM: at www.nvidia.com.\n",
+ nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot,
+ PCI_FUNC(dev->devfn), nv->pci_info.vendor_id,
+ nv->pci_info.device_id, NV_VERSION_STRING, NV_KERNEL_NAME,
+ NV_KERNEL_NAME);
+ }
+ goto err_not_supported;
+ }
+
+ if (!rm_init_private_state(sp, nv))
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: rm_init_private_state() failed!\n");
+ goto err_zero_dev;
+ }
+ }
+
+ nv_printf(NV_DBG_INFO,
+ "NVRM: PCI:%04x:%02x:%02x.%x (%04x:%04x): BAR0 @ 0x%llx (%lluMB)\n",
+ nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot,
+ PCI_FUNC(dev->devfn), nv->pci_info.vendor_id, nv->pci_info.device_id,
+ nv->regs->address, (nv->regs->size >> 20));
+ nv_printf(NV_DBG_INFO,
+ "NVRM: PCI:%04x:%02x:%02x.%x (%04x:%04x): BAR1 @ 0x%llx (%lluMB)\n",
+ nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot,
+ PCI_FUNC(dev->devfn), nv->pci_info.vendor_id, nv->pci_info.device_id,
+ nv->fb->address, (nv->fb->size >> 20));
+
+ num_nv_devices++;
+
+ for (i = 0; i < NV_GPU_NUM_BARS; i++)
+ {
+ if (nv->bars[i].size != 0)
+ {
+ if (nv_user_map_register(nv->bars[i].address,
+ nv->bars[i].strapped_size) != 0)
+ {
+ nv_printf(NV_DBG_ERRORS,
+ "NVRM: failed to register usermap for BAR %u!\n", i);
+ for (j = 0; j < i; j++)
+ {
+ nv_user_map_unregister(nv->bars[j].address,
+ nv->bars[j].strapped_size);
+ }
+ goto err_zero_dev;
+ }
+ }
+ }
+
+ /*
+ * The newly created nvl object is added to the nv_linux_devices global list
+ * only after all the initialization operations for that nvl object are
+ * completed, so as to protect against simultaneous lookup operations which
+ * may discover a partially initialized nvl object in the list
+ */
+ LOCK_NV_LINUX_DEVICES();
+ if (nv_linux_devices == NULL)
+ nv_linux_devices = nvl;
+ else
+ {
+ nv_linux_state_t *tnvl;
+ for (tnvl = nv_linux_devices; tnvl->next != NULL; tnvl = tnvl->next);
+ tnvl->next = nvl;
+ }
+ UNLOCK_NV_LINUX_DEVICES();
+#if !defined(NV_VMWARE)
+ if (nvidia_frontend_add_device((void *)&nv_fops, nvl) != 0)
+ goto err_zero_dev;
+#endif
+ nv_procfs_add_gpu(nvl);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+
+ return 0;
+
+err_zero_dev:
+ rm_free_private_state(sp, nv);
+err_not_supported:
+ if (nvl != NULL)
+ {
+ NV_KFREE(nvl, sizeof(nv_linux_state_t));
+ }
+ release_mem_region(NV_PCI_RESOURCE_START(dev, NV_GPU_BAR_INDEX_REGS),
+ NV_PCI_RESOURCE_SIZE(dev, NV_GPU_BAR_INDEX_REGS));
+ NV_PCI_DISABLE_DEVICE(dev);
+ pci_set_drvdata(dev, NULL);
+failed:
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ return -1;
+}
+
+void nvidia_remove(struct pci_dev *dev)
+{
+ nv_linux_state_t *nvl = NULL;
+ nv_state_t *nv;
+ nv_stack_t *sp = NULL;
+ NvU32 i;
+
+ nv_printf(NV_DBG_SETUP, "NVRM: removing GPU %04x:%02x:%02x.%x\n",
+ NV_PCI_DOMAIN_NUMBER(dev), NV_PCI_BUS_NUMBER(dev),
+ NV_PCI_SLOT_NUMBER(dev), PCI_FUNC(dev->devfn));
+
+ if (NV_IS_SMU_DEVICE(dev))
+ {
+ return;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: %s failed to allocate stack!\n", __FUNCTION__);
+ return;
+ }
+
+ LOCK_NV_LINUX_DEVICES();
+ nvl = pci_get_drvdata(dev);
+ if (!nvl || (nvl->dev != dev))
+ {
+ goto done;
+ }
+
+ nv = NV_STATE_PTR(nvl);
+ if (nvl == nv_linux_devices)
+ nv_linux_devices = nvl->next;
+ else
+ {
+ nv_linux_state_t *tnvl;
+ for (tnvl = nv_linux_devices; tnvl->next != nvl; tnvl = tnvl->next);
+ tnvl->next = nvl->next;
+ }
+
+ /* Remove proc entry for this GPU */
+ nv_procfs_remove_gpu(nvl);
+
+ down(&nvl->ldata_lock);
+ UNLOCK_NV_LINUX_DEVICES();
+
+#if !defined(NV_VMWARE)
+ /* Update the frontend data structures */
+ nvidia_frontend_remove_device((void *)&nv_fops, nvl);
+#endif
+
+#if defined(NV_UVM_ENABLE) || defined(NV_UVM_NEXT_ENABLE)
+ if (!NV_IS_GVI_DEVICE(nv))
+ {
+ NvU8 *uuid;
+ // Inform UVM before disabling adapter
+ if(rm_get_gpu_uuid_raw(sp, nv, &uuid, NULL) == RM_OK)
+ {
+ // this function cannot fail
+ nv_uvm_notify_stop_device(uuid);
+ // get_uuid allocates memory for this call free it here
+ os_free_mem(uuid);
+ }
+ }
+#endif
+
+ if ((nv->flags & NV_FLAG_PERSISTENT_SW_STATE) || (nv->flags & NV_FLAG_OPEN))
+ {
+ if (nv->flags & NV_FLAG_PERSISTENT_SW_STATE)
+ {
+ rm_disable_gpu_state_persistence(sp, nv);
+ }
+ NV_SHUTDOWN_ADAPTER(sp, nv, nvl);
+ NV_KMEM_CACHE_FREE_STACK(nvl->timer_sp);
+ NV_KMEM_CACHE_FREE_STACK(nvl->isr_bh_sp);
+ NV_KMEM_CACHE_FREE_STACK(nvl->pci_cfgchk_sp);
+ NV_KMEM_CACHE_FREE_STACK(nvl->isr_sp);
+ }
+
+ num_probed_nv_devices--;
+
+ pci_set_drvdata(dev, NULL);
+
+ if (NV_IS_GVI_DEVICE(nv))
+ {
+ NV_TASKQUEUE_FLUSH();
+ rm_gvi_detach_device(sp, nv);
+ rm_gvi_free_private_state(sp, nv);
+ }
+ else
+ {
+ for (i = 0; i < NV_GPU_NUM_BARS; i++)
+ {
+ if (nv->bars[i].size != 0)
+ {
+ nv_user_map_unregister(nv->bars[i].address,
+ nv->bars[i].strapped_size);
+ }
+ }
+ rm_i2c_remove_adapters(sp, nv);
+ rm_free_private_state(sp, nv);
+ }
+ release_mem_region(NV_PCI_RESOURCE_START(dev, NV_GPU_BAR_INDEX_REGS),
+ NV_PCI_RESOURCE_SIZE(dev, NV_GPU_BAR_INDEX_REGS));
+ NV_PCI_DISABLE_DEVICE(dev);
+ num_nv_devices--;
+
+ NV_KFREE(nvl, sizeof(nv_linux_state_t));
+ NV_KMEM_CACHE_FREE_STACK(sp);
+ return;
+
+done:
+ UNLOCK_NV_LINUX_DEVICES();
+ NV_KMEM_CACHE_FREE_STACK(sp);
+}
+
+static int
+nvidia_smu_probe
+(
+ struct pci_dev *dev
+)
+{
+ nv_smu_state_t *nv = &nv_smu_device;
+
+ nv_printf(NV_DBG_SETUP, "NVRM: probing 0x%x 0x%x, class 0x%x\n",
+ dev->vendor, dev->device, dev->class);
+
+ if ((dev->vendor != 0x10de) ||
+ (dev->class != (PCI_CLASS_PROCESSOR_CO << 8)) ||
+ (dev->device != NV_PCI_DEVICE_ID_SMU))
+ {
+ nv_printf(NV_DBG_INFO, "NVRM: ignoring the SMU %04x:%02x:%02x.%x\n",
+ NV_PCI_DOMAIN_NUMBER(dev), NV_PCI_BUS_NUMBER(dev),
+ NV_PCI_SLOT_NUMBER(dev), PCI_FUNC(dev->devfn));
+ goto failed;
+ }
+
+ if (nv->handle != NULL)
+ {
+ nv_printf(NV_DBG_INFO,
+ "NVRM: More than one SMU device? Driver not yet designed to handle this case\n");
+ goto failed;
+ }
+
+ if (pci_enable_device(dev) != 0)
+ {
+ nv_printf(NV_DBG_INFO,
+ "NVRM: pci_enable_device for SMU device failed, aborting\n");
+ goto failed;
+ }
+
+ // validate BAR0
+ if (!NV_PCI_RESOURCE_VALID(dev, 0))
+ {
+ nv_printf(NV_DBG_INFO,
+ "NVRM: This PCI I/O region assigned to the SMU device is invalid:\n"
+ "NVRM: BAR0 is %dM @ 0x%08x (PCI:%04x:%02x:%02x.%x)\n",
+ NV_PCI_RESOURCE_SIZE(dev, 0) >> 20, NV_PCI_RESOURCE_START(dev, 0),
+ NV_PCI_DOMAIN_NUMBER(dev), NV_PCI_BUS_NUMBER(dev),
+ NV_PCI_SLOT_NUMBER(dev), PCI_FUNC(dev->devfn));
+ goto failed;
+ }
+
+ if (!request_mem_region(NV_PCI_RESOURCE_START(dev, 0),
+ NV_PCI_RESOURCE_SIZE(dev, 0), NV_DEV_NAME))
+ {
+ nv_printf(NV_DBG_INFO,
+ "NVRM: request_mem_region failed for %dM @ 0x%08x.\n",
+ NV_PCI_RESOURCE_SIZE(dev, 0) >> 20,
+ NV_PCI_RESOURCE_START(dev, 0));
+
+ goto failed;
+ }
+
+ pci_set_drvdata(dev, (void *)nv);
+
+ /* default to 32-bit PCI bus address space */
+ dev->dma_mask = 0xffffffffULL;
+
+ nv->pci_info.vendor_id = dev->vendor;
+ nv->pci_info.device_id = dev->device;
+ nv->pci_info.domain = NV_PCI_DOMAIN_NUMBER(dev);
+ nv->pci_info.bus = NV_PCI_BUS_NUMBER(dev);
+ nv->pci_info.slot = NV_PCI_SLOT_NUMBER(dev);
+ nv->handle = dev;
+
+ if ((NV_PCI_RESOURCE_VALID(dev, 0)) &&
+ (NV_PCI_RESOURCE_FLAGS(dev, 0) & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_MEMORY)
+ {
+ nv->bar0.address = NV_PCI_RESOURCE_START(dev, 0);
+ nv->bar0.size = NV_PCI_RESOURCE_SIZE(dev, 0);
+ nv->bar0.offset = NVRM_PCICFG_BAR_OFFSET(0);
+ }
+
+ nv->regs = &nv->bar0;
+
+ nv_printf(NV_DBG_INFO, "NVRM: %04x:%02x:%02x.%x %04x:%04x - 0x%08x [size=%dK]\n",
+ nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot,
+ PCI_FUNC(dev->devfn), nv->pci_info.vendor_id, nv->pci_info.device_id,
+ nv->regs->address, nv->regs->size / (1024));
+
+ return 0;
+
+failed:
+ return -1;
+}
+
+int NV_API_CALL nv_no_incoherent_mappings(void)
+{
+ return (nv_update_memory_types);
+}
+
+#if defined(NV_PM_SUPPORT_DEVICE_DRIVER_MODEL)
+
+static int
+nv_power_management(
+ struct pci_dev *dev,
+ u32 pci_state,
+ u32 power_state
+)
+{
+ nv_state_t *nv;
+ nv_linux_state_t *lnv = NULL;
+ int status = RM_OK;
+ nv_stack_t *sp = NULL;
+
+ nv_printf(NV_DBG_INFO, "NVRM: nv_power_management: %d\n", pci_state);
+ lnv = pci_get_drvdata(dev);
+
+ if (!lnv || (lnv->dev != dev))
+ {
+ nv_printf(NV_DBG_WARNINGS, "NVRM: PM: invalid device!\n");
+ return -1;
+ }
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ return -1;
+ }
+
+ nv = NV_STATE_PTR(lnv);
+ NV_CHECK_PCI_CONFIG_SPACE(sp, nv, TRUE, TRUE, NV_MAY_SLEEP());
+
+ switch (pci_state)
+ {
+ case PCI_D3hot:
+ nv_printf(NV_DBG_INFO, "NVRM: ACPI: received suspend event\n");
+ status = rm_power_management(sp, nv, 0, power_state);
+ tasklet_kill(&lnv->tasklet);
+ nv_disable_pat_support();
+ break;
+
+ case PCI_D0:
+ nv_printf(NV_DBG_INFO, "NVRM: ACPI: received resume event\n");
+ nv_enable_pat_support();
+ tasklet_init(&lnv->tasklet, nvidia_isr_bh, (NvUPtr)NV_STATE_PTR(lnv));
+ status = rm_power_management(sp, nv, 0, power_state);
+ break;
+
+ default:
+ nv_printf(NV_DBG_WARNINGS, "NVRM: PM: unsupported event: %d\n", pci_state);
+ status = -1;
+ }
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+
+ if (status != RM_OK)
+ nv_printf(NV_DBG_ERRORS, "NVRM: PM: failed event: %d\n", pci_state);
+
+ return status;
+}
+
+static int
+nvidia_suspend(
+ struct pci_dev *dev,
+ pm_message_t state
+)
+{
+ int pci_state = -1;
+ u32 power_state;
+ nv_state_t *nv;
+ nv_linux_state_t *lnv = NULL;
+
+ if (NV_IS_SMU_DEVICE(dev))
+ {
+ return nvidia_smu_suspend();
+ }
+
+ lnv = pci_get_drvdata(dev);
+
+ if (!lnv || (lnv->dev != dev))
+ {
+ nv_printf(NV_DBG_WARNINGS, "NVRM: PM: invalid device!\n");
+ return -1;
+ }
+
+ nv = NV_STATE_PTR(lnv);
+
+ if (NV_IS_GVI_DEVICE(nv))
+ {
+ return nv_gvi_kern_suspend(dev, state);
+ }
+
+#if !defined(NV_PM_MESSAGE_T_PRESENT)
+ pci_state = state;
+#elif defined(NV_PCI_CHOOSE_STATE_PRESENT)
+ pci_state = PCI_D3hot;
+#endif
+
+ power_state = NV_PM_ACPI_STANDBY;
+
+#if defined(NV_PM_MESSAGE_T_HAS_EVENT)
+ if (state.event == PM_EVENT_FREEZE) /* for hibernate */
+ power_state = NV_PM_ACPI_HIBERNATE;
+#endif
+
+ return nv_power_management(dev, pci_state, power_state);
+}
+
+static int
+nvidia_smu_suspend(void)
+{
+ nv_stack_t *sp = NULL;
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ return RM_ERR_NO_FREE_MEM;
+ }
+
+ rm_suspend_smu(sp);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+
+ return 0;
+}
+
+static int
+nvidia_resume(
+ struct pci_dev *dev
+)
+{
+ nv_state_t *nv;
+ nv_linux_state_t *lnv = NULL;
+
+ if (NV_IS_SMU_DEVICE(dev))
+ {
+ return nvidia_smu_resume();
+ }
+
+ lnv = pci_get_drvdata(dev);
+
+ if (!lnv || (lnv->dev != dev))
+ {
+ nv_printf(NV_DBG_WARNINGS, "NVRM: PM: invalid device!\n");
+ return -1;
+ }
+
+ nv = NV_STATE_PTR(lnv);
+
+ if (NV_IS_GVI_DEVICE(nv))
+ {
+ return nv_gvi_kern_resume(dev);
+ }
+
+ return nv_power_management(dev, PCI_D0, NV_PM_ACPI_RESUME);
+}
+
+static int
+nvidia_smu_resume(void)
+{
+ nv_stack_t *sp = NULL;
+
+ NV_KMEM_CACHE_ALLOC_STACK(sp);
+ if (sp == NULL)
+ {
+ nv_printf(NV_DBG_ERRORS, "NVRM: failed to allocate stack!\n");
+ return RM_ERR_NO_FREE_MEM;
+ }
+
+ rm_resume_smu(sp);
+
+ NV_KMEM_CACHE_FREE_STACK(sp);
+
+ return 0;
+}
+
+#endif /* defined(NV_PM_SUPPORT_DEVICE_DRIVER_MODEL) */
+
+void* NV_API_CALL nv_get_adapter_state(
+ NvU32 domain,
+ NvU8 bus,
+ NvU8 slot
+)
+{
+ nv_linux_state_t *nvl;
+
+ LOCK_NV_LINUX_DEVICES();
+ for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
+ {
+ nv_state_t *nv = NV_STATE_PTR(nvl);
+ if (nv->pci_info.domain == domain && nv->pci_info.bus == bus
+ && nv->pci_info.slot == slot)
+ {
+ UNLOCK_NV_LINUX_DEVICES();
+ return (void *) nv;
+ }
+ }
+ UNLOCK_NV_LINUX_DEVICES();
+
+ if (NV_PCI_MATCH_CTL_DEVICE(domain, bus, slot))
+ {
+ nv_state_t *nv = NV_STATE_PTR(&nv_ctl_device);
+ return (void *) nv;
+ }
+
+ return NULL;
+}
+
+void* NV_API_CALL nv_get_smu_state(void)
+{
+ nv_smu_state_t *nv_smu = &nv_smu_device;
+
+ if (nv_smu->handle == NULL)
+ {
+ return NULL;
+ }
+
+ return nv_smu;
+}
+
+RM_STATUS NV_API_CALL nv_log_error(
+ nv_state_t *nv,
+ NvU32 error_number,
+ const char *format,
+ va_list ap
+)
+{
+ RM_STATUS status = RM_OK;
+#if defined(CONFIG_CRAY_XT)
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ if (ap != NULL)
+ {
+ status = nvos_forward_error_to_cray(nvl->dev, error_number,
+ format, ap);
+ }
+#endif
+ return status;
+}
diff -urN kernel.orig/nv-drm.c kernel/nv-drm.c
--- kernel.orig/nv-drm.c 2014-05-27 22:59:51.000000000 +0400
+++ kernel/nv-drm.c 2015-01-17 23:34:40.419585549 +0300
@@ -18,6 +18,11 @@
#include <drm/drmP.h>
+/* 3.18-rc0+ */
+#ifndef drm_gem_object
+#include <drm/drm_gem.h>
+#endif
+
extern nv_linux_state_t *nv_linux_devices;
struct nv_gem_object {
@@ -124,6 +129,10 @@
.gem_prime_vmap = nv_gem_prime_vmap,
.gem_prime_vunmap = nv_gem_prime_vunmap,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0)
+ .set_busid = drm_pci_set_busid,
+#endif
+
.name = "nvidia-drm",
.desc = "NVIDIA DRM driver",
.date = "20130102",
diff -urN kernel.orig/nv-drm.c.orig kernel/nv-drm.c.orig
--- kernel.orig/nv-drm.c.orig 1970-01-01 03:00:00.000000000 +0300
+++ kernel/nv-drm.c.orig 2014-05-27 22:59:51.000000000 +0400
@@ -0,0 +1,227 @@
+/* _NVRM_COPYRIGHT_BEGIN_
+ *
+ * Copyright 2013 by NVIDIA Corporation. All rights reserved. All
+ * information contained herein is proprietary and confidential to NVIDIA
+ * Corporation. Any use, reproduction, or disclosure without the written
+ * permission of NVIDIA Corporation is prohibited.
+ *
+ * _NVRM_COPYRIGHT_END_
+ */
+
+#define __NO_VERSION__
+
+#include "nv-misc.h"
+#include "os-interface.h"
+#include "nv-linux.h"
+
+#if defined(NV_DRM_AVAILABLE)
+
+#include <drm/drmP.h>
+
+extern nv_linux_state_t *nv_linux_devices;
+
+struct nv_gem_object {
+ struct drm_gem_object base;
+ struct page **pages;
+};
+
+static int nv_drm_load(
+ struct drm_device *dev,
+ unsigned long flags
+)
+{
+ nv_linux_state_t *nvl;
+
+ for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
+ {
+ if (nvl->dev == dev->pdev)
+ {
+ nvl->drm = dev;
+ return 0;
+ }
+ }
+
+ return -ENODEV;
+}
+
+static int nv_drm_unload(
+ struct drm_device *dev
+)
+{
+ nv_linux_state_t *nvl;
+
+ for (nvl = nv_linux_devices; nvl != NULL; nvl = nvl->next)
+ {
+ if (nvl->dev == dev->pdev)
+ {
+ BUG_ON(nvl->drm != dev);
+ nvl->drm = NULL;
+ return 0;
+ }
+ }
+
+ return -ENODEV;
+}
+
+static void nv_gem_free(
+ struct drm_gem_object *obj
+)
+{
+ struct nv_gem_object *nv_obj = container_of(obj, struct nv_gem_object, base);
+ NV_KFREE(nv_obj, sizeof(*nv_obj));
+}
+
+static struct sg_table* nv_gem_prime_get_sg_table(
+ struct drm_gem_object *obj
+)
+{
+ struct nv_gem_object *nv_obj = container_of(obj, struct nv_gem_object, base);
+ int page_count = obj->size >> PAGE_SHIFT;
+
+ return drm_prime_pages_to_sg(nv_obj->pages, page_count);
+}
+
+static void* nv_gem_prime_vmap(
+ struct drm_gem_object *obj
+)
+{
+ struct nv_gem_object *nv_obj = container_of(obj, struct nv_gem_object, base);
+ int page_count = obj->size >> PAGE_SHIFT;
+
+ return vmap(nv_obj->pages, page_count, VM_USERMAP, PAGE_KERNEL);
+}
+
+static void nv_gem_prime_vunmap(
+ struct drm_gem_object *obj,
+ void *virtual
+)
+{
+ vunmap(virtual);
+}
+
+static const struct file_operations nv_drm_fops = {
+ .owner = THIS_MODULE,
+ .open = drm_open,
+ .release = drm_release,
+ .unlocked_ioctl = drm_ioctl,
+ .mmap = drm_gem_mmap,
+ .poll = drm_poll,
+ .read = drm_read,
+ .llseek = noop_llseek,
+};
+
+static struct drm_driver nv_drm_driver = {
+ .driver_features = DRIVER_GEM | DRIVER_PRIME,
+ .load = nv_drm_load,
+ .unload = nv_drm_unload,
+ .fops = &nv_drm_fops,
+
+ .gem_free_object = nv_gem_free,
+
+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+ .gem_prime_export = drm_gem_prime_export,
+ .gem_prime_get_sg_table = nv_gem_prime_get_sg_table,
+ .gem_prime_vmap = nv_gem_prime_vmap,
+ .gem_prime_vunmap = nv_gem_prime_vunmap,
+
+ .name = "nvidia-drm",
+ .desc = "NVIDIA DRM driver",
+ .date = "20130102",
+ .major = 0,
+ .minor = 0,
+ .patchlevel = 0,
+};
+#endif /* defined(NV_DRM_AVAILABLE) */
+
+int __init nv_drm_init(
+ struct pci_driver *pci_driver
+)
+{
+ int ret = 0;
+#if defined(NV_DRM_AVAILABLE)
+ ret = drm_pci_init(&nv_drm_driver, pci_driver);
+#endif
+ return ret;
+}
+
+void nv_drm_exit(
+ struct pci_driver *pci_driver
+)
+{
+#if defined(NV_DRM_AVAILABLE)
+ drm_pci_exit(&nv_drm_driver, pci_driver);
+#endif
+}
+
+RM_STATUS NV_API_CALL nv_alloc_os_descriptor_handle(
+ nv_state_t *nv,
+ NvS32 drm_fd,
+ void *private,
+ NvU64 page_count,
+ NvU32 *handle
+)
+{
+ RM_STATUS status = RM_ERR_NOT_SUPPORTED;
+
+#if defined(NV_DRM_AVAILABLE)
+ nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
+ struct page **pages = private;
+ struct file *drmf;
+ struct drm_file *file_priv;
+ struct nv_gem_object *nv_obj = NULL;
+ size_t size = page_count << PAGE_SHIFT;
+ int ret;
+
+ if (drm_fd < 0)
+ {
+ return RM_ERR_INVALID_ARGUMENT;
+ }
+
+ drmf = fget((unsigned int)drm_fd);
+ if (drmf == NULL)
+ {
+ return RM_ERR_INVALID_ARGUMENT;
+ }
+
+ if (drmf->f_op != &nv_drm_fops)
+ {
+ status = RM_ERR_INVALID_ARGUMENT;
+ goto done;
+ }
+
+ file_priv = drmf->private_data;
+
+ NV_KMALLOC(nv_obj, sizeof(*nv_obj));
+ if (!nv_obj)
+ {
+ status = RM_ERR_INSUFFICIENT_RESOURCES;
+ goto done;
+ }
+
+ memset(&nv_obj->base, 0, sizeof(nv_obj->base));
+ nv_obj->pages = pages;
+
+ drm_gem_private_object_init(nvl->drm, &nv_obj->base, size);
+
+ ret = drm_gem_handle_create(file_priv, &nv_obj->base, handle);
+ if (ret)
+ {
+ status = RM_ERR_OPERATING_SYSTEM;
+ goto done;
+ }
+
+ drm_gem_object_unreference_unlocked(&nv_obj->base);
+
+ status = RM_OK;
+
+done:
+ if (status != RM_OK)
+ {
+ NV_KFREE(nv_obj, sizeof(*nv_obj));
+ }
+
+ fput(drmf);
+#endif
+
+ return status;
+}
diff -urN kernel.orig/nv-frontend.c kernel/nv-frontend.c
--- kernel.orig/nv-frontend.c 2014-05-27 22:59:52.000000000 +0400
+++ kernel/nv-frontend.c 2015-01-17 23:33:57.946450351 +0300
@@ -327,7 +327,7 @@
unsigned long i_arg
)
{
- return nvidia_frontend_ioctl(file->f_dentry->d_inode, file, cmd, i_arg);
+ return nvidia_frontend_ioctl(file->f_path.dentry->d_inode, file, cmd, i_arg);
}
long nvidia_frontend_compat_ioctl(
@@ -336,7 +336,7 @@
unsigned long i_arg
)
{
- return nvidia_frontend_ioctl(file->f_dentry->d_inode, file, cmd, i_arg);
+ return nvidia_frontend_ioctl(file->f_path.dentry->d_inode, file, cmd, i_arg);
}
int nvidia_frontend_mmap(