[PATCH 1/5] drm/radeon: separate gart and vm functions (original) (raw)

Alex Deucher alexdeucher at gmail.com
Fri Feb 21 08:37:20 PST 2014

On Thu, Feb 20, 2014 at 3:20 PM, Christian König <deathsimple at vodafone.de> wrote:

From: Christian König <christian.koenig at amd.com>

Both are complex enough on their own. Signed-off-by: Christian König <christian.koenig at amd.com>

For the series:

Reviewed-by: Alex Deucher <alexander.deucher at amd.com>

--- drivers/gpu/drm/radeon/Makefile | 2 +- drivers/gpu/drm/radeon/radeongart.c | 958 ---------------------------------- drivers/gpu/drm/radeon/radeonvm.c | 981 +++++++++++++++++++++++++++++++++++ 3 files changed, 982 insertions(+), 959 deletions(-) create mode 100644 drivers/gpu/drm/radeon/radeonvm.c

diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile index ed60caa..0943353 100644 --- a/drivers/gpu/drm/radeon/Makefile +++ b/drivers/gpu/drm/radeon/Makefile _@@ -80,7 +80,7 @@ radeon-y += radeondevice.o radeonasic.o radeonkms.o _ _r600dpm.o rs780dpm.o rv6xxdpm.o rv770dpm.o rv730dpm.o rv740dpm.o _ _rv770smc.o cypressdpm.o btcdpm.o sumodpm.o sumosmc.o trinitydpm.o _ _trinitysmc.o nidpm.o sismc.o sidpm.o kvsmc.o kvdpm.o cismc.o _ - cidpm.o dce6afmt.o + cidpm.o dce6afmt.o radeonvm.o # add async DMA block _radeon-y += _ diff --git a/drivers/gpu/drm/radeon/radeongart.c b/drivers/gpu/drm/radeon/radeongart.c index a8f9b46..2e72365 100644 --- a/drivers/gpu/drm/radeon/radeongart.c +++ b/drivers/gpu/drm/radeon/radeongart.c @@ -28,8 +28,6 @@ #include <drm/drmP.h> #include <drm/radeondrm.h> #include "radeon.h" -#include "radeonreg.h" -#include "radeontrace.h" /* * GART @@ -394,959 +392,3 @@ void radeongartfini(struct radeondevice *rdev) radeondummypagefini(rdev); } - -/* - * GPUVM - * GPUVM is similar to the legacy gart on older asics, however - * rather than there being a single global gart table - * for the entire GPU, there are multiple VM page tables active - * at any given time. The VM page tables can contain a mix - * vram pages and system memory pages and system memory pages - * can be mapped as snooped (cached system pages) or unsnooped - * (uncached system pages). - * Each VM has an ID associated with it and there is a page table - * associated with each VMID. When execting a command buffer, - * the kernel tells the the ring what VMID to use for that command - * buffer. VMIDs are allocated dynamically as commands are submitted. - * The userspace drivers maintain their own address space and the kernel - * sets up their pages tables accordingly when they submit their - * command buffers and a VMID is assigned. - * Cayman/Trinity support up to 8 active VMs at any given time; - * SI supports 16. - */ - -/* - * vm helpers - * - * TODO bind a default page at vm initialization for default address - */ - -/** - * radeonvmnumpde - return the number of page directory entries - * - * @rdev: radeondevice pointer - * - * Calculate the number of page directory entries (cayman+). - */ -static unsigned radeonvmnumpdes(struct radeondevice *rdev) -{ - return rdev->vmmanager.maxpfn >> RADEONVMBLOCKSIZE; -} - -/** - * radeonvmdirectorysize - returns the size of the page directory in bytes - * - * @rdev: radeondevice pointer - * - * Calculate the size of the page directory in bytes (cayman+). - */ -static unsigned radeonvmdirectorysize(struct radeondevice *rdev) -{ - return RADEONGPUPAGEALIGN(radeonvmnumpdes(rdev) * 8); -} - -/** - * radeonvmmanagerinit - init the vm manager - * - * @rdev: radeondevice pointer - * - * Init the vm manager (cayman+). - * Returns 0 for success, error for failure. - */ -int radeonvmmanagerinit(struct radeondevice *rdev) -{ - struct radeonvm *vm; - struct radeonbova *bova; - int r; - unsigned size; - - if (!rdev->vmmanager.enabled) { - /* allocate enough for 2 full VM pts */ - size = radeonvmdirectorysize(rdev); - size += rdev->vmmanager.maxpfn * 8; - size *= 2; - r = radeonsabomanagerinit(rdev, &rdev->vmmanager.samanager, - RADEONGPUPAGEALIGN(size), - RADEONVMPTBALIGNSIZE, - RADEONGEMDOMAINVRAM); - if (r) { - deverr(rdev->dev, "failed to allocate vm bo (%dKB)\n", - (rdev->vmmanager.maxpfn * 8) >> 10); - return r; - } - - r = radeonasicvminit(rdev); - if (r) - return r; - - rdev->vmmanager.enabled = true; - - r = radeonsabomanagerstart(rdev, &rdev->vmmanager.samanager); - if (r) - return r; - } - - /* restore page table */ - listforeachentry(vm, &rdev->vmmanager.lruvm, list) { - if (vm->pagedirectory == NULL) - continue; - - listforeachentry(bova, &vm->va, vmlist) { - bova->valid = false; - } - } - return 0; -} - -/** - * radeonvmfreept - free the page table for a specific vm - * - * @rdev: radeondevice pointer - * @vm: vm to unbind - * - * Free the page table of a specific vm (cayman+). - * - * Global and local mutex must be lock! - */ -static void radeonvmfreept(struct radeondevice *rdev, - struct radeonvm *vm) -{ - struct radeonbova *bova; - int i; - - if (!vm->pagedirectory) - return; - - listdelinit(&vm->list); - radeonsabofree(rdev, &vm->pagedirectory, vm->fence); - - listforeachentry(bova, &vm->va, vmlist) { - bova->valid = false; - } - - if (vm->pagetables == NULL) - return; - - for (i = 0; i < radeonvmnumpdes(rdev); i++)_ _- radeonsabofree(rdev, &vm->pagetables[i], vm->fence); - - kfree(vm->pagetables); -} - -/** - * radeonvmmanagerfini - tear down the vm manager - * - * @rdev: radeondevice pointer - * - * Tear down the VM manager (cayman+). - */ -void radeonvmmanagerfini(struct radeondevice *rdev) -{ - struct radeonvm *vm, *tmp; - int i; - - if (!rdev->vmmanager.enabled) - return; - - mutexlock(&rdev->vmmanager.lock); - /* free all allocated page tables */ - listforeachentrysafe(vm, tmp, &rdev->vmmanager.lruvm, list) { - mutexlock(&vm->mutex); - radeonvmfreept(rdev, vm); - mutexunlock(&vm->mutex); - } - for (i = 0; i < RADEONNUMVM; ++i) {_ _- radeonfenceunref(&rdev->vmmanager.active[i]); - } - radeonasicvmfini(rdev); - mutexunlock(&rdev->vmmanager.lock); - - radeonsabomanagersuspend(rdev, &rdev->vmmanager.samanager); - radeonsabomanagerfini(rdev, &rdev->vmmanager.samanager); - rdev->vmmanager.enabled = false; -} - -/** - * radeonvmevict - evict page table to make room for new one - * - * @rdev: radeondevice pointer - * @vm: VM we want to allocate something for - * - * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). - * Returns 0 for success, -ENOMEM for failure. - * - * Global and local mutex must be locked! - */ -static int radeonvmevict(struct radeondevice *rdev, struct radeonvm *vm) -{ - struct radeonvm *vmevict; - - if (listempty(&rdev->vmmanager.lruvm)) - return -ENOMEM; - - vmevict = listfirstentry(&rdev->vmmanager.lruvm, - struct radeonvm, list); - if (vmevict == vm) - return -ENOMEM; - - mutexlock(&vmevict->mutex); - radeonvmfreept(rdev, vmevict); - mutexunlock(&vmevict->mutex); - return 0; -} - -/** - * radeonvmallocpt - allocates a page table for a VM - * - * @rdev: radeondevice pointer - * @vm: vm to bind - * - * Allocate a page table for the requested vm (cayman+). - * Returns 0 for success, error for failure. - * - * Global and local mutex must be locked! - */ -int radeonvmallocpt(struct radeondevice *rdev, struct radeonvm *vm) -{ - unsigned pdsize, pdentries, ptssize; - struct radeonib ib; - int r; - - if (vm == NULL) { - return -EINVAL; - } - - if (vm->pagedirectory != NULL) { - return 0; - } - - pdsize = radeonvmdirectorysize(rdev); - pdentries = radeonvmnumpdes(rdev); - -retry: - r = radeonsabonew(rdev, &rdev->vmmanager.samanager, - &vm->pagedirectory, pdsize, - RADEONVMPTBALIGNSIZE, false); - if (r == -ENOMEM) { - r = radeonvmevict(rdev, vm); - if (r) - return r; - goto retry; - - } else if (r) { - return r; - } - - vm->pdgpuaddr = radeonsabogpuaddr(vm->pagedirectory); - - /* Initially clear the page directory */ - r = radeonibget(rdev, R600RINGTYPEDMAINDEX, &ib, - NULL, pdentries * 2 + 64); - if (r) { - radeonsabofree(rdev, &vm->pagedirectory, vm->fence); - return r; - } - - ib.lengthdw = 0; - - radeonasicvmsetpage(rdev, &ib, vm->pdgpuaddr, - 0, pdentries, 0, 0); - - radeonsemaphoresyncto(ib.semaphore, vm->fence); - r = radeonibschedule(rdev, &ib, NULL); - if (r) { - radeonibfree(rdev, &ib); - radeonsabofree(rdev, &vm->pagedirectory, vm->fence); - return r; - } - radeonfenceunref(&vm->fence); - vm->fence = radeonfenceref(ib.fence); - radeonibfree(rdev, &ib); - radeonfenceunref(&vm->lastflush); - - /* allocate page table array */ - ptssize = radeonvmnumpdes(rdev) * sizeof(struct radeonsabo *); - vm->pagetables = kzalloc(ptssize, GFPKERNEL); - - if (vm->pagetables == NULL) { - DRMERROR("Cannot allocate memory for page table array\n"); - radeonsabofree(rdev, &vm->pagedirectory, vm->fence); - return -ENOMEM; - } - - return 0; -} - -/** - * radeonvmaddtolru - add VMs page table to LRU list - * - * @rdev: radeondevice pointer - * @vm: vm to add to LRU - * - * Add the allocated page table to the LRU list (cayman+). - * - * Global mutex must be locked! - */ -void radeonvmaddtolru(struct radeondevice *rdev, struct radeonvm *vm) -{ - listdelinit(&vm->list); - listaddtail(&vm->list, &rdev->vmmanager.lruvm); -} - -/** - * radeonvmgrabid - allocate the next free VMID - * - * @rdev: radeondevice pointer - * @vm: vm to allocate id for - * @ring: ring we want to submit job to - * - * Allocate an id for the vm (cayman+). - * Returns the fence we need to sync to (if any). - * - * Global and local mutex must be locked! - */ -struct radeonfence *radeonvmgrabid(struct radeondevice *rdev, - struct radeonvm *vm, int ring) -{ - struct radeonfence *best[RADEONNUMRINGS] = {}; - unsigned choices[2] = {}; - unsigned i; - - /* check if the id is still valid */ - if (vm->lastiduse && vm->lastiduse == rdev->vmmanager.active[vm->id]) - return NULL; - - /* we definately need to flush */ - radeonfenceunref(&vm->lastflush); - - /* skip over VMID 0, since it is the system VM */ - for (i = 1; i < rdev->vmmanager.nvm; ++i) { - struct radeonfence *fence = rdev->vmmanager.active[i]; - - if (fence == NULL) { - /* found a free one */ - vm->id = i; - traceradeonvmgrabid(vm->id, ring); - return NULL; - } - - if (radeonfenceisearlier(fence, best[fence->ring])) { - best[fence->ring] = fence; - choices[fence->ring == ring ? 0 : 1] = i; - } - } - - for (i = 0; i < 2; ++i) {_ _- if (choices[i]) {_ _- vm->id = choices[i]; - traceradeonvmgrabid(vm->id, ring); - return rdev->vmmanager.active[choices[i]]; - } - } - - /* should never happen */ - BUG(); - return NULL; -} - -/** - * radeonvmfence - remember fence for vm - * - * @rdev: radeondevice pointer - * @vm: vm we want to fence - * @fence: fence to remember - * - * Fence the vm (cayman+). - * Set the fence used to protect page table and id. - * - * Global and local mutex must be locked! - */ -void radeonvmfence(struct radeondevice *rdev, - struct radeonvm *vm, - struct radeonfence *fence) -{ - radeonfenceunref(&rdev->vmmanager.active[vm->id]); - rdev->vmmanager.active[vm->id] = radeonfenceref(fence); - - radeonfenceunref(&vm->fence); - vm->fence = radeonfenceref(fence); - - radeonfenceunref(&vm->lastiduse); - vm->lastiduse = radeonfenceref(fence); -} - -/** - * radeonvmbofind - find the bova for a specific vm & bo - * - * @vm: requested vm - * @bo: requested buffer object - * - * Find @bo inside the requested vm (cayman+). - * Search inside the @bos vm list for the requested vm - * Returns the found bova or NULL if none is found - * - * Object has to be reserved! - */ -struct radeonbova *radeonvmbofind(struct radeonvm *vm, - struct radeonbo *bo) -{ - struct radeonbova *bova; - - listforeachentry(bova, &bo->va, bolist) { - if (bova->vm == vm) { - return bova; - } - } - return NULL; -} - -/** - * radeonvmboadd - add a bo to a specific vm - * - * @rdev: radeondevice pointer - * @vm: requested vm - * @bo: radeon buffer object - * - * Add @bo into the requested vm (cayman+). - * Add @bo to the list of bos associated with the vm - * Returns newly added bova or NULL for failure - * - * Object has to be reserved! - */ -struct radeonbova *radeonvmboadd(struct radeondevice *rdev, - struct radeonvm *vm, - struct radeonbo *bo) -{ - struct radeonbova *bova; - - bova = kzalloc(sizeof(struct radeonbova), GFPKERNEL); - if (bova == NULL) { - return NULL; - } - bova->vm = vm; - bova->bo = bo; - bova->soffset = 0; - bova->eoffset = 0; - bova->flags = 0; - bova->valid = false; - bova->refcount = 1; - INITLISTHEAD(&bova->bolist); - INITLISTHEAD(&bova->vmlist); - - mutexlock(&vm->mutex); - listadd(&bova->vmlist, &vm->va); - listaddtail(&bova->bolist, &bo->va); - mutexunlock(&vm->mutex); - - return bova; -} - -/** - * radeonvmbosetaddr - set bos virtual address inside a vm - * - * @rdev: radeondevice pointer - * @bova: bova to store the address - * @soffset: requested offset of the buffer in the VM address space - * @flags: attributes of pages (read/write/valid/etc.) - * - * Set offset of @bova (cayman+). - * Validate and set the offset requested within the vm address space. - * Returns 0 for success, error for failure. - * - * Object has to be reserved! - */ -int radeonvmbosetaddr(struct radeondevice *rdev, - struct radeonbova *bova, - uint64t soffset, - uint32t flags) -{ - uint64t size = radeonbosize(bova->bo); - uint64t eoffset, lastoffset = 0; - struct radeonvm *vm = bova->vm; - struct radeonbova *tmp; - struct listhead *head; - unsigned lastpfn; - - if (soffset) { - /* make sure object fit at this offset */ - eoffset = soffset + size; - if (soffset >= eoffset) { - return -EINVAL; - } - - lastpfn = eoffset / RADEONGPUPAGESIZE; - if (lastpfn > rdev->vmmanager.maxpfn) { - deverr(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", - lastpfn, rdev->vmmanager.maxpfn); - return -EINVAL; - } - - } else { - eoffset = lastpfn = 0; - } - - mutexlock(&vm->mutex); - head = &vm->va; - lastoffset = 0; - listforeachentry(tmp, &vm->va, vmlist) { - if (bova == tmp) { - /* skip over currently modified bo */ - continue; - } - - if (soffset >= lastoffset && eoffset <= tmp->soffset) { - /* bo can be added before this one */ - break; - } - if (eoffset > tmp->soffset && soffset < tmp->eoffset) { - /* bo and tmp overlap, invalid offset */ - deverr(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", - bova->bo, (unsigned)bova->soffset, tmp->bo, - (unsigned)tmp->soffset, (unsigned)tmp->eoffset); - mutexunlock(&vm->mutex); - return -EINVAL; - } - lastoffset = tmp->eoffset; - head = &tmp->vmlist; - } - - bova->soffset = soffset; - bova->eoffset = eoffset; - bova->flags = flags; - bova->valid = false; - listmove(&bova->vmlist, head); - - mutexunlock(&vm->mutex); - return 0; -} - -/** - * radeonvmmapgart - get the physical address of a gart page - * - * @rdev: radeondevice pointer - * @addr: the unmapped addr - * - * Look up the physical address of the page that the pte resolves - * to (cayman+). - * Returns the physical address of the page. - */ -uint64t radeonvmmapgart(struct radeondevice *rdev, uint64t addr) -{ - uint64t result; - - /* page table offset */ - result = rdev->gart.pagesaddr[addr >> PAGESHIFT]; - - /* in case cpu page size != gpu page size*/ - result |= addr & (~PAGEMASK); - - return result; -} - -/** - * radeonvmpageflags - translate page flags to what the hw uses - * - * @flags: flags comming from userspace - * - * Translate the flags the userspace ABI uses to hw flags. - */ -static uint32t radeonvmpageflags(uint32t flags) -{ - uint32t hwflags = 0; - hwflags |= (flags & RADEONVMPAGEVALID) ? R600PTEVALID : 0; - hwflags |= (flags & RADEONVMPAGEREADABLE) ? R600PTEREADABLE : 0; - hwflags |= (flags & RADEONVMPAGEWRITEABLE) ? R600PTEWRITEABLE : 0; - if (flags & RADEONVMPAGESYSTEM) { - hwflags |= R600PTESYSTEM; - hwflags |= (flags & RADEONVMPAGESNOOPED) ? R600PTESNOOPED : 0; - } - return hwflags; -} - -/** - * radeonvmupdatepdes - make sure that page directory is valid - * - * @rdev: radeondevice pointer - * @vm: requested vm - * @start: start of GPU address range - * @end: end of GPU address range - * - * Allocates new page tables if necessary - * and updates the page directory (cayman+). - * Returns 0 for success, error for failure. - * - * Global and local mutex must be locked! - */ -static int radeonvmupdatepdes(struct radeondevice *rdev, - struct radeonvm *vm, - struct radeonib *ib, - uint64t start, uint64t end) -{ - static const uint32t incr = RADEONVMPTECOUNT * 8; - - uint64t lastpde = ~0, lastpt = ~0; - unsigned count = 0; - uint64t ptidx; - int r; - - start = (start / RADEONGPUPAGESIZE) >> RADEONVMBLOCKSIZE; - end = (end / RADEONGPUPAGESIZE) >> RADEONVMBLOCKSIZE; - - /* walk over the address space and update the page directory */ - for (ptidx = start; ptidx <= end; ++ptidx) {_ _- uint64t pde, pt;_ _-_ _- if (vm->pagetables[ptidx]) - continue; - -retry: - r = radeonsabonew(rdev, &rdev->vmmanager.samanager, - &vm->pagetables[ptidx], - RADEONVMPTECOUNT * 8, - RADEONGPUPAGESIZE, false); - - if (r == -ENOMEM) { - r = radeonvmevict(rdev, vm); - if (r) - return r; - goto retry; - } else if (r) { - return r; - } - - pde = vm->pdgpuaddr + ptidx * 8; - - pt = radeonsabogpuaddr(vm->pagetables[ptidx]); - - if (((lastpde + 8 * count) != pde) || - ((lastpt + incr * count) != pt)) { - - if (count) { - radeonasicvmsetpage(rdev, ib, lastpde, - lastpt, count, incr, - R600PTEVALID); - - count *= RADEONVMPTECOUNT; - radeonasicvmsetpage(rdev, ib, lastpt, 0, - count, 0, 0); - } - - count = 1; - lastpde = pde; - lastpt = pt; - } else { - ++count; - } - } - - if (count) { - radeonasicvmsetpage(rdev, ib, lastpde, lastpt, count, - incr, R600PTEVALID); - - count *= RADEONVMPTECOUNT; - radeonasicvmsetpage(rdev, ib, lastpt, 0, - count, 0, 0); - } - - return 0; -} - -/** - * radeonvmupdateptes - make sure that page tables are valid - * - * @rdev: radeondevice pointer - * @vm: requested vm - * @start: start of GPU address range - * @end: end of GPU address range - * @dst: destination address to map to - * @flags: mapping flags - * - * Update the page tables in the range @start - @end (cayman+). - * - * Global and local mutex must be locked! - */ -static void radeonvmupdateptes(struct radeondevice *rdev, - struct radeonvm *vm, - struct radeonib *ib, - uint64t start, uint64t end, - uint64t dst, uint32t flags) -{ - static const uint64t mask = RADEONVMPTECOUNT - 1; - - uint64t lastpte = ~0, lastdst = ~0; - unsigned count = 0; - uint64t addr; - - start = start / RADEONGPUPAGESIZE; - end = end / RADEONGPUPAGESIZE; - - /* walk over the address space and update the page tables */ - for (addr = start; addr < end; ) {_ _- uint64t ptidx = addr >> RADEONVMBLOCKSIZE; - unsigned nptes; - uint64t pte; - - if ((addr & ~mask) == (end & ~mask)) - nptes = end - addr; - else - nptes = RADEONVMPTECOUNT - (addr & mask); - - pte = radeonsabogpuaddr(vm->pagetables[ptidx]); - pte += (addr & mask) * 8; - - if ((lastpte + 8 * count) != pte) { - - if (count) { - radeonasicvmsetpage(rdev, ib, lastpte, - lastdst, count, - RADEONGPUPAGESIZE, - flags); - } - - count = nptes; - lastpte = pte; - lastdst = dst; - } else { - count += nptes; - } - - addr += nptes; - dst += nptes * RADEONGPUPAGESIZE; - } - - if (count) { - radeonasicvmsetpage(rdev, ib, lastpte, - lastdst, count, - RADEONGPUPAGESIZE, flags); - } -} - -/** - * radeonvmboupdate - map a bo into the vm page table - * - * @rdev: radeondevice pointer - * @vm: requested vm - * @bo: radeon buffer object - * @mem: ttm mem - * - * Fill in the page table entries for @bo (cayman+). - * Returns 0 for success, -EINVAL for failure. - * - * Object have to be reserved & global and local mutex must be locked! - */ -int radeonvmboupdate(struct radeondevice *rdev, - struct radeonvm *vm, - struct radeonbo *bo, - struct ttmmemreg *mem) -{ - struct radeonib ib; - struct radeonbova *bova; - unsigned nptes, npdes, ndw; - uint64t addr; - int r; - - /* nothing to do if vm isn't bound */ - if (vm->pagedirectory == NULL) - return 0; - - bova = radeonvmbofind(vm, bo); - if (bova == NULL) { - deverr(rdev->dev, "bo %p not in vm %p\n", bo, vm); - return -EINVAL; - } - - if (!bova->soffset) { - deverr(rdev->dev, "bo %p don't has a mapping in vm %p\n", - bo, vm); - return -EINVAL; - } - - if ((bova->valid && mem) || (!bova->valid && mem == NULL)) - return 0; - - bova->flags &= ~RADEONVMPAGEVALID; - bova->flags &= ~RADEONVMPAGESYSTEM; - if (mem) { - addr = mem->start << PAGESHIFT;_ _- if (mem->memtype != TTMPLSYSTEM) { - bova->flags |= RADEONVMPAGEVALID; - bova->valid = true; - } - if (mem->memtype == TTMPLTT) { - bova->flags |= RADEONVMPAGESYSTEM; - } else { - addr += rdev->vmmanager.vrambaseoffset; - } - } else { - addr = 0; - bova->valid = false; - } - - traceradeonvmboupdate(bova); - - nptes = radeonbongpupages(bo); - - /* assume two extra pdes in case the mapping overlaps the borders */ - npdes = (nptes >> RADEONVMBLOCKSIZE) + 2; - - /* padding, etc. */ - ndw = 64; - - if (RADEONVMBLOCKSIZE > 11) - /* reserve space for one header for every 2k dwords */ - ndw += (nptes >> 11) * 4; - else - /* reserve space for one header for - every (1 << BLOCKSIZE) entries */_ _- ndw += (nptes >> RADEONVMBLOCKSIZE) * 4; - - /* reserve space for pte addresses */ - ndw += nptes * 2; - - /* reserve space for one header for every 2k dwords */ - ndw += (npdes >> 11) * 4; - - /* reserve space for pde addresses */ - ndw += npdes * 2; - - /* reserve space for clearing new page tables */ - ndw += npdes * 2 * RADEONVMPTECOUNT; - - /* update too big for an IB */ - if (ndw > 0xfffff) - return -ENOMEM; - - r = radeonibget(rdev, R600RINGTYPEDMAINDEX, &ib, NULL, ndw * 4); - if (r) - return r; - ib.lengthdw = 0; - - r = radeonvmupdatepdes(rdev, vm, &ib, bova->soffset, bova->eoffset); - if (r) { - radeonibfree(rdev, &ib); - return r; - } - - radeonvmupdateptes(rdev, vm, &ib, bova->soffset, bova->eoffset, - addr, radeonvmpageflags(bova->flags)); - - radeonsemaphoresyncto(ib.semaphore, vm->fence); - r = radeonibschedule(rdev, &ib, NULL); - if (r) { - radeonibfree(rdev, &ib); - return r; - } - radeonfenceunref(&vm->fence); - vm->fence = radeonfenceref(ib.fence); - radeonibfree(rdev, &ib); - radeonfenceunref(&vm->lastflush); - - return 0; -} - -/** - * radeonvmbormv - remove a bo to a specific vm - * - * @rdev: radeondevice pointer - * @bova: requested bova - * - * Remove @bova->bo from the requested vm (cayman+). - * Remove @bova->bo from the list of bos associated with the bova->vm and - * remove the ptes for @bova in the page table. - * Returns 0 for success. - * - * Object have to be reserved! - */ -int radeonvmbormv(struct radeondevice *rdev, - struct radeonbova *bova) -{ - int r = 0; - - mutexlock(&rdev->vmmanager.lock); - mutexlock(&bova->vm->mutex); - if (bova->soffset) { - r = radeonvmboupdate(rdev, bova->vm, bova->bo, NULL); - } - mutexunlock(&rdev->vmmanager.lock); - listdel(&bova->vmlist); - mutexunlock(&bova->vm->mutex); - listdel(&bova->bolist); - - kfree(bova); - return r; -} - -/** - * radeonvmboinvalidate - mark the bo as invalid - * - * @rdev: radeondevice pointer - * @vm: requested vm - * @bo: radeon buffer object - * - * Mark @bo as invalid (cayman+). - */ -void radeonvmboinvalidate(struct radeondevice *rdev, - struct radeonbo *bo) -{ - struct radeonbova *bova; - - listforeachentry(bova, &bo->va, bolist) { - bova->valid = false; - } -} - -/** - * radeonvminit - initialize a vm instance - * - * @rdev: radeondevice pointer - * @vm: requested vm - * - * Init @vm fields (cayman+). - */ -void radeonvminit(struct radeondevice *rdev, struct radeonvm *vm) -{ - vm->id = 0; - vm->fence = NULL; - vm->lastflush = NULL; - vm->lastiduse = NULL; - mutexinit(&vm->mutex); - INITLISTHEAD(&vm->list); - INITLISTHEAD(&vm->va); -} - -/** - * radeonvmfini - tear down a vm instance - * - * @rdev: radeondevice pointer - * @vm: requested vm - * - * Tear down @vm (cayman+). - * Unbind the VM and remove all bos from the vm bo list - */ -void radeonvmfini(struct radeondevice *rdev, struct radeonvm *vm) -{ - struct radeonbova *bova, *tmp; - int r; - - mutexlock(&rdev->vmmanager.lock); - mutexlock(&vm->mutex); - radeonvmfreept(rdev, vm); - mutexunlock(&rdev->vmmanager.lock); - - if (!listempty(&vm->va)) { - deverr(rdev->dev, "still active bo inside vm\n"); - } - listforeachentrysafe(bova, tmp, &vm->va, vmlist) { - listdelinit(&bova->vmlist); - r = radeonboreserve(bova->bo, false); - if (!r) { - listdelinit(&bova->bolist); - radeonbounreserve(bova->bo); - kfree(bova); - } - } - radeonfenceunref(&vm->fence); - radeonfenceunref(&vm->lastflush); - radeonfenceunref(&vm->lastiduse); - mutexunlock(&vm->mutex); -} diff --git a/drivers/gpu/drm/radeon/radeonvm.c b/drivers/gpu/drm/radeon/radeonvm.c new file mode 100644 index 0000000..433b1eb --- /dev/null +++ b/drivers/gpu/drm/radeon/radeonvm.c @@ -0,0 +1,981 @@ +/* + * Copyright 2008 Advanced Micro Devices, Inc. + * Copyright 2008 Red Hat Inc. + * Copyright 2009 Jerome Glisse. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Alex Deucher + * Jerome Glisse + */ +#include <drm/drmP.h> +#include <drm/radeondrm.h> +#include "radeon.h" +#include "radeontrace.h" + +/* + * GPUVM + * GPUVM is similar to the legacy gart on older asics, however + * rather than there being a single global gart table + * for the entire GPU, there are multiple VM page tables active + * at any given time. The VM page tables can contain a mix + * vram pages and system memory pages and system memory pages + * can be mapped as snooped (cached system pages) or unsnooped + * (uncached system pages). + * Each VM has an ID associated with it and there is a page table + * associated with each VMID. When execting a command buffer, + * the kernel tells the the ring what VMID to use for that command + * buffer. VMIDs are allocated dynamically as commands are submitted. + * The userspace drivers maintain their own address space and the kernel + * sets up their pages tables accordingly when they submit their + * command buffers and a VMID is assigned. + * Cayman/Trinity support up to 8 active VMs at any given time; + * SI supports 16. + */ + +/** + * radeonvmnumpde - return the number of page directory entries + * + * @rdev: radeondevice pointer + * + * Calculate the number of page directory entries (cayman+). + */ +static unsigned radeonvmnumpdes(struct radeondevice *rdev) +{ + return rdev->vmmanager.maxpfn >> RADEONVMBLOCKSIZE; +} + +/** + * radeonvmdirectorysize - returns the size of the page directory in bytes + * + * @rdev: radeondevice pointer + * + * Calculate the size of the page directory in bytes (cayman+). + */ +static unsigned radeonvmdirectorysize(struct radeondevice *rdev) +{ + return RADEONGPUPAGEALIGN(radeonvmnumpdes(rdev) * 8); +} + +/** + * radeonvmmanagerinit - init the vm manager + * + * @rdev: radeondevice pointer + * + * Init the vm manager (cayman+). + * Returns 0 for success, error for failure. + */ +int radeonvmmanagerinit(struct radeondevice *rdev) +{ + struct radeonvm *vm; + struct radeonbova *bova; + int r; + unsigned size; + + if (!rdev->vmmanager.enabled) { + /* allocate enough for 2 full VM pts */ + size = radeonvmdirectorysize(rdev); + size += rdev->vmmanager.maxpfn * 8; + size *= 2; + r = radeonsabomanagerinit(rdev, &rdev->vmmanager.samanager, + RADEONGPUPAGEALIGN(size), + RADEONVMPTBALIGNSIZE, + RADEONGEMDOMAINVRAM); + if (r) { + deverr(rdev->dev, "failed to allocate vm bo (%dKB)\n", + (rdev->vmmanager.maxpfn * 8) >> 10); + return r; + } + + r = radeonasicvminit(rdev); + if (r) + return r; + + rdev->vmmanager.enabled = true; + + r = radeonsabomanagerstart(rdev, &rdev->vmmanager.samanager); + if (r) + return r; + } + + /* restore page table */ + listforeachentry(vm, &rdev->vmmanager.lruvm, list) { + if (vm->pagedirectory == NULL) + continue; + + listforeachentry(bova, &vm->va, vmlist) { + bova->valid = false; + } + } + return 0; +} + +/** + * radeonvmfreept - free the page table for a specific vm + * + * @rdev: radeondevice pointer + * @vm: vm to unbind + * + * Free the page table of a specific vm (cayman+). + * + * Global and local mutex must be lock! + */ +static void radeonvmfreept(struct radeondevice *rdev, + struct radeonvm *vm) +{ + struct radeonbova *bova; + int i; + + if (!vm->pagedirectory) + return; + + listdelinit(&vm->list); + radeonsabofree(rdev, &vm->pagedirectory, vm->fence); + + listforeachentry(bova, &vm->va, vmlist) { + bova->valid = false; + } + + if (vm->pagetables == NULL) + return; + + for (i = 0; i < radeonvmnumpdes(rdev); i++)_ _+ radeonsabofree(rdev, &vm->pagetables[i], vm->fence); + + kfree(vm->pagetables); +} + +/** + * radeonvmmanagerfini - tear down the vm manager + * + * @rdev: radeondevice pointer + * + * Tear down the VM manager (cayman+). + */ +void radeonvmmanagerfini(struct radeondevice *rdev) +{ + struct radeonvm *vm, *tmp; + int i; + + if (!rdev->vmmanager.enabled) + return; + + mutexlock(&rdev->vmmanager.lock); + /* free all allocated page tables */ + listforeachentrysafe(vm, tmp, &rdev->vmmanager.lruvm, list) { + mutexlock(&vm->mutex); + radeonvmfreept(rdev, vm); + mutexunlock(&vm->mutex); + } + for (i = 0; i < RADEONNUMVM; ++i) {_ _+ radeonfenceunref(&rdev->vmmanager.active[i]); + } + radeonasicvmfini(rdev); + mutexunlock(&rdev->vmmanager.lock); + + radeonsabomanagersuspend(rdev, &rdev->vmmanager.samanager); + radeonsabomanagerfini(rdev, &rdev->vmmanager.samanager); + rdev->vmmanager.enabled = false; +} + +/** + * radeonvmevict - evict page table to make room for new one + * + * @rdev: radeondevice pointer + * @vm: VM we want to allocate something for + * + * Evict a VM from the lru, making sure that it isn't @vm. (cayman+). + * Returns 0 for success, -ENOMEM for failure. + * + * Global and local mutex must be locked! + */ +static int radeonvmevict(struct radeondevice *rdev, struct radeonvm *vm) +{ + struct radeonvm *vmevict; + + if (listempty(&rdev->vmmanager.lruvm)) + return -ENOMEM; + + vmevict = listfirstentry(&rdev->vmmanager.lruvm, + struct radeonvm, list); + if (vmevict == vm) + return -ENOMEM; + + mutexlock(&vmevict->mutex); + radeonvmfreept(rdev, vmevict); + mutexunlock(&vmevict->mutex); + return 0; +} + +/** + * radeonvmallocpt - allocates a page table for a VM + * + * @rdev: radeondevice pointer + * @vm: vm to bind + * + * Allocate a page table for the requested vm (cayman+). + * Returns 0 for success, error for failure. + * + * Global and local mutex must be locked! + */ +int radeonvmallocpt(struct radeondevice *rdev, struct radeonvm *vm) +{ + unsigned pdsize, pdentries, ptssize; + struct radeonib ib; + int r; + + if (vm == NULL) { + return -EINVAL; + } + + if (vm->pagedirectory != NULL) { + return 0; + } + + pdsize = radeonvmdirectorysize(rdev); + pdentries = radeonvmnumpdes(rdev); + +retry: + r = radeonsabonew(rdev, &rdev->vmmanager.samanager, + &vm->pagedirectory, pdsize, + RADEONVMPTBALIGNSIZE, false); + if (r == -ENOMEM) { + r = radeonvmevict(rdev, vm); + if (r) + return r; + goto retry; + + } else if (r) { + return r; + } + + vm->pdgpuaddr = radeonsabogpuaddr(vm->pagedirectory); + + /* Initially clear the page directory */ + r = radeonibget(rdev, R600RINGTYPEDMAINDEX, &ib, + NULL, pdentries * 2 + 64); + if (r) { + radeonsabofree(rdev, &vm->pagedirectory, vm->fence); + return r; + } + + ib.lengthdw = 0; + + radeonasicvmsetpage(rdev, &ib, vm->pdgpuaddr, + 0, pdentries, 0, 0); + + radeonsemaphoresyncto(ib.semaphore, vm->fence); + r = radeonibschedule(rdev, &ib, NULL); + if (r) { + radeonibfree(rdev, &ib); + radeonsabofree(rdev, &vm->pagedirectory, vm->fence); + return r; + } + radeonfenceunref(&vm->fence); + vm->fence = radeonfenceref(ib.fence); + radeonibfree(rdev, &ib); + radeonfenceunref(&vm->lastflush); + + /* allocate page table array */ + ptssize = radeonvmnumpdes(rdev) * sizeof(struct radeonsabo *); + vm->pagetables = kzalloc(ptssize, GFPKERNEL); + + if (vm->pagetables == NULL) { + DRMERROR("Cannot allocate memory for page table array\n"); + radeonsabofree(rdev, &vm->pagedirectory, vm->fence); + return -ENOMEM; + } + + return 0; +} + +/** + * radeonvmaddtolru - add VMs page table to LRU list + * + * @rdev: radeondevice pointer + * @vm: vm to add to LRU + * + * Add the allocated page table to the LRU list (cayman+). + * + * Global mutex must be locked! + */ +void radeonvmaddtolru(struct radeondevice *rdev, struct radeonvm *vm) +{ + listdelinit(&vm->list); + listaddtail(&vm->list, &rdev->vmmanager.lruvm); +} + +/** + * radeonvmgrabid - allocate the next free VMID + * + * @rdev: radeondevice pointer + * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * + * Allocate an id for the vm (cayman+). + * Returns the fence we need to sync to (if any). + * + * Global and local mutex must be locked! + */ +struct radeonfence *radeonvmgrabid(struct radeondevice *rdev, + struct radeonvm *vm, int ring) +{ + struct radeonfence *best[RADEONNUMRINGS] = {}; + unsigned choices[2] = {}; + unsigned i; + + /* check if the id is still valid */ + if (vm->lastiduse && vm->lastiduse == rdev->vmmanager.active[vm->id]) + return NULL; + + /* we definately need to flush */ + radeonfenceunref(&vm->lastflush); + + /* skip over VMID 0, since it is the system VM */ + for (i = 1; i < rdev->vmmanager.nvm; ++i) { + struct radeonfence *fence = rdev->vmmanager.active[i]; + + if (fence == NULL) { + /* found a free one */ + vm->id = i; + traceradeonvmgrabid(vm->id, ring); + return NULL; + } + + if (radeonfenceisearlier(fence, best[fence->ring])) { + best[fence->ring] = fence; + choices[fence->ring == ring ? 0 : 1] = i; + } + } + + for (i = 0; i < 2; ++i) {_ _+ if (choices[i]) {_ _+ vm->id = choices[i]; + traceradeonvmgrabid(vm->id, ring); + return rdev->vmmanager.active[choices[i]]; + } + } + + /* should never happen */ + BUG(); + return NULL; +} + +/** + * radeonvmfence - remember fence for vm + * + * @rdev: radeondevice pointer + * @vm: vm we want to fence + * @fence: fence to remember + * + * Fence the vm (cayman+). + * Set the fence used to protect page table and id. + * + * Global and local mutex must be locked! + */ +void radeonvmfence(struct radeondevice *rdev, + struct radeonvm *vm, + struct radeonfence *fence) +{ + radeonfenceunref(&rdev->vmmanager.active[vm->id]); + rdev->vmmanager.active[vm->id] = radeonfenceref(fence); + + radeonfenceunref(&vm->fence); + vm->fence = radeonfenceref(fence); + + radeonfenceunref(&vm->lastiduse); + vm->lastiduse = radeonfenceref(fence); +} + +/** + * radeonvmbofind - find the bova for a specific vm & bo + * + * @vm: requested vm + * @bo: requested buffer object + * + * Find @bo inside the requested vm (cayman+). + * Search inside the @bos vm list for the requested vm + * Returns the found bova or NULL if none is found + * + * Object has to be reserved! + */ +struct radeonbova *radeonvmbofind(struct radeonvm *vm, + struct radeonbo *bo) +{ + struct radeonbova *bova; + + listforeachentry(bova, &bo->va, bolist) { + if (bova->vm == vm) { + return bova; + } + } + return NULL; +} + +/** + * radeonvmboadd - add a bo to a specific vm + * + * @rdev: radeondevice pointer + * @vm: requested vm + * @bo: radeon buffer object + * + * Add @bo into the requested vm (cayman+). + * Add @bo to the list of bos associated with the vm + * Returns newly added bova or NULL for failure + * + * Object has to be reserved! + */ +struct radeonbova *radeonvmboadd(struct radeondevice *rdev, + struct radeonvm *vm, + struct radeonbo *bo) +{ + struct radeonbova *bova; + + bova = kzalloc(sizeof(struct radeonbova), GFPKERNEL); + if (bova == NULL) { + return NULL; + } + bova->vm = vm; + bova->bo = bo; + bova->soffset = 0; + bova->eoffset = 0; + bova->flags = 0; + bova->valid = false; + bova->refcount = 1; + INITLISTHEAD(&bova->bolist); + INITLISTHEAD(&bova->vmlist); + + mutexlock(&vm->mutex); + listadd(&bova->vmlist, &vm->va); + listaddtail(&bova->bolist, &bo->va); + mutexunlock(&vm->mutex); + + return bova; +} + +/** + * radeonvmbosetaddr - set bos virtual address inside a vm + * + * @rdev: radeondevice pointer + * @bova: bova to store the address + * @soffset: requested offset of the buffer in the VM address space + * @flags: attributes of pages (read/write/valid/etc.) + * + * Set offset of @bova (cayman+). + * Validate and set the offset requested within the vm address space. + * Returns 0 for success, error for failure. + * + * Object has to be reserved! + */ +int radeonvmbosetaddr(struct radeondevice *rdev, + struct radeonbova *bova, + uint64t soffset, + uint32t flags) +{ + uint64t size = radeonbosize(bova->bo); + uint64t eoffset, lastoffset = 0; + struct radeonvm *vm = bova->vm; + struct radeonbova *tmp; + struct listhead *head; + unsigned lastpfn; + + if (soffset) { + /* make sure object fit at this offset */ + eoffset = soffset + size; + if (soffset >= eoffset) { + return -EINVAL; + } + + lastpfn = eoffset / RADEONGPUPAGESIZE; + if (lastpfn > rdev->vmmanager.maxpfn) { + deverr(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", + lastpfn, rdev->vmmanager.maxpfn); + return -EINVAL; + } + + } else { + eoffset = lastpfn = 0; + } + + mutexlock(&vm->mutex); + head = &vm->va; + lastoffset = 0; + listforeachentry(tmp, &vm->va, vmlist) { + if (bova == tmp) { + /* skip over currently modified bo */ + continue; + } + + if (soffset >= lastoffset && eoffset <= tmp->soffset) { + /* bo can be added before this one */ + break; + } + if (eoffset > tmp->soffset && soffset < tmp->eoffset) { + /* bo and tmp overlap, invalid offset */ + deverr(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", + bova->bo, (unsigned)bova->soffset, tmp->bo, + (unsigned)tmp->soffset, (unsigned)tmp->eoffset); + mutexunlock(&vm->mutex); + return -EINVAL; + } + lastoffset = tmp->eoffset; + head = &tmp->vmlist; + } + + bova->soffset = soffset; + bova->eoffset = eoffset; + bova->flags = flags; + bova->valid = false; + listmove(&bova->vmlist, head); + + mutexunlock(&vm->mutex); + return 0; +} + +/** + * radeonvmmapgart - get the physical address of a gart page + * + * @rdev: radeondevice pointer + * @addr: the unmapped addr + * + * Look up the physical address of the page that the pte resolves + * to (cayman+). + * Returns the physical address of the page. + */ +uint64t radeonvmmapgart(struct radeondevice *rdev, uint64t addr) +{ + uint64t result; + + /* page table offset */ + result = rdev->gart.pagesaddr[addr >> PAGESHIFT]; + + /* in case cpu page size != gpu page size*/ + result |= addr & (~PAGEMASK); + + return result; +} + +/** + * radeonvmpageflags - translate page flags to what the hw uses + * + * @flags: flags comming from userspace + * + * Translate the flags the userspace ABI uses to hw flags. + */ +static uint32t radeonvmpageflags(uint32t flags) +{ + uint32t hwflags = 0; + hwflags |= (flags & RADEONVMPAGEVALID) ? R600PTEVALID : 0; + hwflags |= (flags & RADEONVMPAGEREADABLE) ? R600PTEREADABLE : 0; + hwflags |= (flags & RADEONVMPAGEWRITEABLE) ? R600PTEWRITEABLE : 0; + if (flags & RADEONVMPAGESYSTEM) { + hwflags |= R600PTESYSTEM; + hwflags |= (flags & RADEONVMPAGESNOOPED) ? R600PTESNOOPED : 0; + } + return hwflags; +} + +/** + * radeonvmupdatepdes - make sure that page directory is valid + * + * @rdev: radeondevice pointer + * @vm: requested vm + * @start: start of GPU address range + * @end: end of GPU address range + * + * Allocates new page tables if necessary + * and updates the page directory (cayman+). + * Returns 0 for success, error for failure. + * + * Global and local mutex must be locked! + */ +static int radeonvmupdatepdes(struct radeondevice *rdev, + struct radeonvm *vm, + struct radeonib *ib, + uint64t start, uint64t end) +{ + static const uint32t incr = RADEONVMPTECOUNT * 8; + + uint64t lastpde = ~0, lastpt = ~0; + unsigned count = 0; + uint64t ptidx; + int r; + + start = (start / RADEONGPUPAGESIZE) >> RADEONVMBLOCKSIZE; + end = (end / RADEONGPUPAGESIZE) >> RADEONVMBLOCKSIZE; + + /* walk over the address space and update the page directory */ + for (ptidx = start; ptidx <= end; ++ptidx) {_ _+ uint64t pde, pt;_ _+_ _+ if (vm->pagetables[ptidx]) + continue; + +retry: + r = radeonsabonew(rdev, &rdev->vmmanager.samanager, + &vm->pagetables[ptidx], + RADEONVMPTECOUNT * 8, + RADEONGPUPAGESIZE, false); + + if (r == -ENOMEM) { + r = radeonvmevict(rdev, vm); + if (r) + return r; + goto retry; + } else if (r) { + return r; + } + + pde = vm->pdgpuaddr + ptidx * 8; + + pt = radeonsabogpuaddr(vm->pagetables[ptidx]); + + if (((lastpde + 8 * count) != pde) || + ((lastpt + incr * count) != pt)) { + + if (count) { + radeonasicvmsetpage(rdev, ib, lastpde, + lastpt, count, incr, + R600PTEVALID); + + count *= RADEONVMPTECOUNT; + radeonasicvmsetpage(rdev, ib, lastpt, 0, + count, 0, 0); + } + + count = 1; + lastpde = pde; + lastpt = pt; + } else { + ++count; + } + } + + if (count) { + radeonasicvmsetpage(rdev, ib, lastpde, lastpt, count, + incr, R600PTEVALID); + + count *= RADEONVMPTECOUNT; + radeonasicvmsetpage(rdev, ib, lastpt, 0, + count, 0, 0); + } + + return 0; +} + +/** + * radeonvmupdateptes - make sure that page tables are valid + * + * @rdev: radeondevice pointer + * @vm: requested vm + * @start: start of GPU address range + * @end: end of GPU address range + * @dst: destination address to map to + * @flags: mapping flags + * + * Update the page tables in the range @start - @end (cayman+). + * + * Global and local mutex must be locked! + */ +static void radeonvmupdateptes(struct radeondevice *rdev, + struct radeonvm *vm, + struct radeonib *ib, + uint64t start, uint64t end, + uint64t dst, uint32t flags) +{ + static const uint64t mask = RADEONVMPTECOUNT - 1; + + uint64t lastpte = ~0, lastdst = ~0; + unsigned count = 0; + uint64t addr; + + start = start / RADEONGPUPAGESIZE; + end = end / RADEONGPUPAGESIZE; + + /* walk over the address space and update the page tables */ + for (addr = start; addr < end; ) {_ _+ uint64t ptidx = addr >> RADEONVMBLOCKSIZE; + unsigned nptes; + uint64t pte; + + if ((addr & ~mask) == (end & ~mask)) + nptes = end - addr; + else + nptes = RADEONVMPTECOUNT - (addr & mask); + + pte = radeonsabogpuaddr(vm->pagetables[ptidx]); + pte += (addr & mask) * 8; + + if ((lastpte + 8 * count) != pte) { + + if (count) { + radeonasicvmsetpage(rdev, ib, lastpte, + lastdst, count, + RADEONGPUPAGESIZE, + flags); + } + + count = nptes; + lastpte = pte; + lastdst = dst; + } else { + count += nptes; + } + + addr += nptes; + dst += nptes * RADEONGPUPAGESIZE; + } + + if (count) { + radeonasicvmsetpage(rdev, ib, lastpte, + lastdst, count, + RADEONGPUPAGESIZE, flags); + } +} + +/** + * radeonvmboupdate - map a bo into the vm page table + * + * @rdev: radeondevice pointer + * @vm: requested vm + * @bo: radeon buffer object + * @mem: ttm mem + * + * Fill in the page table entries for @bo (cayman+). + * Returns 0 for success, -EINVAL for failure. + * + * Object have to be reserved & global and local mutex must be locked! + */ +int radeonvmboupdate(struct radeondevice *rdev, + struct radeonvm *vm, + struct radeonbo *bo, + struct ttmmemreg *mem) +{ + struct radeonib ib; + struct radeonbova *bova; + unsigned nptes, npdes, ndw; + uint64t addr; + int r; + + /* nothing to do if vm isn't bound */ + if (vm->pagedirectory == NULL) + return 0; + + bova = radeonvmbofind(vm, bo); + if (bova == NULL) { + deverr(rdev->dev, "bo %p not in vm %p\n", bo, vm); + return -EINVAL; + } + + if (!bova->soffset) { + deverr(rdev->dev, "bo %p don't has a mapping in vm %p\n", + bo, vm); + return -EINVAL; + } + + if ((bova->valid && mem) || (!bova->valid && mem == NULL)) + return 0; + + bova->flags &= ~RADEONVMPAGEVALID; + bova->flags &= ~RADEONVMPAGESYSTEM; + if (mem) { + addr = mem->start << PAGESHIFT;_ _+ if (mem->memtype != TTMPLSYSTEM) { + bova->flags |= RADEONVMPAGEVALID; + bova->valid = true; + } + if (mem->memtype == TTMPLTT) { + bova->flags |= RADEONVMPAGESYSTEM; + } else { + addr += rdev->vmmanager.vrambaseoffset; + } + } else { + addr = 0; + bova->valid = false; + } + + traceradeonvmboupdate(bova); + + nptes = radeonbongpupages(bo); + + /* assume two extra pdes in case the mapping overlaps the borders */ + npdes = (nptes >> RADEONVMBLOCKSIZE) + 2; + + /* padding, etc. */ + ndw = 64; + + if (RADEONVMBLOCKSIZE > 11) + /* reserve space for one header for every 2k dwords */ + ndw += (nptes >> 11) * 4; + else + /* reserve space for one header for + every (1 << BLOCKSIZE) entries */_ _+ ndw += (nptes >> RADEONVMBLOCKSIZE) * 4; + + /* reserve space for pte addresses */ + ndw += nptes * 2; + + /* reserve space for one header for every 2k dwords */ + ndw += (npdes >> 11) * 4; + + /* reserve space for pde addresses */ + ndw += npdes * 2; + + /* reserve space for clearing new page tables */ + ndw += npdes * 2 * RADEONVMPTECOUNT; + + /* update too big for an IB */ + if (ndw > 0xfffff) + return -ENOMEM; + + r = radeonibget(rdev, R600RINGTYPEDMAINDEX, &ib, NULL, ndw * 4); + if (r) + return r; + ib.lengthdw = 0; + + r = radeonvmupdatepdes(rdev, vm, &ib, bova->soffset, bova->eoffset); + if (r) { + radeonibfree(rdev, &ib); + return r; + } + + radeonvmupdateptes(rdev, vm, &ib, bova->soffset, bova->eoffset, + addr, radeonvmpageflags(bova->flags)); + + radeonsemaphoresyncto(ib.semaphore, vm->fence); + r = radeonibschedule(rdev, &ib, NULL); + if (r) { + radeonibfree(rdev, &ib); + return r; + } + radeonfenceunref(&vm->fence); + vm->fence = radeonfenceref(ib.fence); + radeonibfree(rdev, &ib); + radeonfenceunref(&vm->lastflush); + + return 0; +} + +/** + * radeonvmbormv - remove a bo to a specific vm + * + * @rdev: radeondevice pointer + * @bova: requested bova + * + * Remove @bova->bo from the requested vm (cayman+). + * Remove @bova->bo from the list of bos associated with the bova->vm and + * remove the ptes for @bova in the page table. + * Returns 0 for success. + * + * Object have to be reserved! + */ +int radeonvmbormv(struct radeondevice *rdev, + struct radeonbova *bova) +{ + int r = 0; + + mutexlock(&rdev->vmmanager.lock); + mutexlock(&bova->vm->mutex); + if (bova->soffset) { + r = radeonvmboupdate(rdev, bova->vm, bova->bo, NULL); + } + mutexunlock(&rdev->vmmanager.lock); + listdel(&bova->vmlist); + mutexunlock(&bova->vm->mutex); + listdel(&bova->bolist); + + kfree(bova); + return r; +} + +/** + * radeonvmboinvalidate - mark the bo as invalid + * + * @rdev: radeondevice pointer + * @vm: requested vm + * @bo: radeon buffer object + * + * Mark @bo as invalid (cayman+). + */ +void radeonvmboinvalidate(struct radeondevice *rdev, + struct radeonbo *bo) +{ + struct radeonbova *bova; + + listforeachentry(bova, &bo->va, bolist) { + bova->valid = false; + } +} + +/** + * radeonvminit - initialize a vm instance + * + * @rdev: radeondevice pointer + * @vm: requested vm + * + * Init @vm fields (cayman+). + */ +void radeonvminit(struct radeondevice *rdev, struct radeonvm *vm) +{ + vm->id = 0; + vm->fence = NULL; + vm->lastflush = NULL; + vm->lastiduse = NULL; + mutexinit(&vm->mutex); + INITLISTHEAD(&vm->list); + INITLISTHEAD(&vm->va); +} + +/** + * radeonvmfini - tear down a vm instance + * + * @rdev: radeondevice pointer + * @vm: requested vm + * + * Tear down @vm (cayman+). + * Unbind the VM and remove all bos from the vm bo list + */ +void radeonvmfini(struct radeondevice *rdev, struct radeonvm *vm) +{ + struct radeonbova *bova, *tmp; + int r; + + mutexlock(&rdev->vmmanager.lock); + mutexlock(&vm->mutex); + radeonvmfreept(rdev, vm); + mutexunlock(&rdev->vmmanager.lock); + + if (!listempty(&vm->va)) { + deverr(rdev->dev, "still active bo inside vm\n"); + } + listforeachentrysafe(bova, tmp, &vm->va, vmlist) { + listdelinit(&bova->vmlist); + r = radeonboreserve(bova->bo, false); + if (!r) { + listdelinit(&bova->bolist); + radeonbounreserve(bova->bo); + kfree(bova); + } + } + radeonfenceunref(&vm->fence); + radeonfenceunref(&vm->lastflush); + radeonfenceunref(&vm->lastiduse); + mutexunlock(&vm->mutex); +} -- 1.8.3.2

dri-devel mailing list dri-devel at lists.freedesktop.org http://lists.freedesktop.org/mailman/listinfo/dri-devel

More information about the dri-devel mailing list