// SPDX-License-Identifier: MIT
/*
* Copyright © 2022 Intel Corporation
*/
#include "xe_pt.h"
#include "regs/xe_gtt_defs.h"
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_drm_client.h"
#include "xe_exec_queue.h"
#include "xe_gt.h"
#include "xe_gt_stats.h"
#include "xe_migrate.h"
#include "xe_page_reclaim.h"
#include "xe_pat.h"
#include "xe_pt_types.h"
#include "xe_pt_walk.h"
#include "xe_res_cursor.h"
#include "xe_sched_job.h"
#include "xe_svm.h"
#include "xe_sync.h"
#include "xe_tlb_inval_job.h"
#include "xe_trace.h"
#include "xe_ttm_stolen_mgr.h"
#include "xe_userptr.h"
#include "xe_vm.h"
struct xe_pt_dir {
struct xe_pt pt;
/** @children: Array of page-table child nodes */
struct xe_ptw *children[XE_PDES];
/** @staging: Array of page-table staging nodes */
struct xe_ptw *staging[XE_PDES];
};
#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
#else
#define xe_pt_set_addr(__xe_pt, __addr)
#define xe_pt_addr(__xe_pt) 0ull
#endif
static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48};
static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48};
#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
{
return container_of(pt, struct xe_pt_dir, pt);
}
static struct xe_pt *
xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)
{
return container_of(pt_dir->staging[index], struct xe_pt, base);
}
static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
unsigned int level)
{
struct xe_device *xe = tile_to_xe(tile);
u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
u8 id = tile->id;
if (!xe_vm_has_scratch(vm))
return 0;
if (level > MAX_HUGEPTE_LEVEL)
return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo,
0);
return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) |
XE_PTE_NULL;
}
static void xe_pt_free(struct xe_pt *pt)
{
if (pt->level)
kfree(as_xe_pt_dir(pt));
else
kfree(pt);
}
/**
* xe_pt_create() - Create a page-table.
* @vm: The vm to create for.
* @tile: The tile to create for.
* @level: The page-table level.
* @exec: The drm_exec object used to lock the vm.
*
* Allocate and initialize a single struct xe_pt metadata structure. Also
* create the corresponding page-table bo, but don't initialize it. If the
* level is grater than zero, then it's assumed to be a directory page-
* table and the directory structure is also allocated and initialized to
* NULL pointers.
*
* Return: A valid struct xe_pt pointer on success, Pointer error code on
* error.
*/
struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
unsigned int level, struct drm_exec *exec)
{
struct xe_pt *pt;
struct xe_bo *bo;
u32 bo_flags;
int err;
if (level) {
struct xe_pt_dir *dir = kzalloc_obj(*dir);
pt = (dir) ? &dir->pt : NULL;
} else {
pt = kzalloc_obj(*pt);
}
if (!pt)
return ERR_PTR(-ENOMEM);
bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE;
if (vm->xef) /* userspace */
bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
pt->level = level;
drm_WARN_ON(&vm->xe->drm, IS_ERR_OR_NULL(exec));
bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
ttm_bo_type_kernel,
bo_flags, exec);
if (IS_ERR(bo)) {
err = PTR_ERR(bo);
goto err_kfree;
}
pt->bo = bo;
pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;
pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;
if (vm->xef)
xe_drm_client_add_bo(vm->xef->client, pt->bo);
xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);
return pt;
err_kfree:
xe_pt_free(pt);
return ERR_PTR(err);
}
ALLOW_ERROR_INJECTION(xe_pt_create, ERRNO);
/**
* xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
* entries.
* @tile: The tile the scratch pagetable of which to use.
* @vm: The vm we populate for.
* @pt: The pagetable the bo of which to initialize.
*
* Populate the page-table bo of @pt with entries pointing into the tile's
* scratch page-table tree if any. Otherwise populate with zeros.
*/
void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
struct xe_pt *pt)
{
struct iosys_map *map = &pt->bo->vmap;
u64 empty;
int i;
if (!xe_vm_has_scratch(vm)) {
/*
* FIXME: Some memory is allocated already allocated to zero?
* Find out which memory that is and avoid this memset...
*/
xe_map_memset(vm->xe, map, 0, 0, SZ_4K);
} else {
empty = __xe_pt_empty_pte(tile, vm, pt->level);
for (i = 0; i < XE_PDES; i++)
xe_pt_write(vm->xe, map, i, empty);
}
}
/**
* xe_pt_shift() - Return the ilog2 value of the size of the address range of
* a page-table at a certain level.
* @level: The level.
*
* Return: The ilog2 value of the size of the address range of a page-table
* at level @level.
*/
unsigned int xe_pt_shift(unsigned int level)
{
return XE_PTE_SHIFT + XE_PDE_SHIFT * level;
}
/**
* xe_pt_destroy() - Destroy a page-table tree.
* @pt: The root of the page-table tree to destroy.
* @flags: vm flags. Currently unused.
* @deferred: List head of lockless list for deferred putting. NULL for
* immediate putting.
*
* Puts the page-table bo, recursively calls xe_pt_destroy on all children
* and finally frees @pt. TODO: Can we remove the @flags argument?
*/
void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
{
int i;
if (!pt)
return;
XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
xe_bo_unpin(pt->bo);
xe_bo_put_deferred(pt->bo, deferred);
if (pt->level > 0 && pt->num_live) {
struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
for (i = 0; i < XE_PDES; i++) {
if (xe_pt_entry_staging(pt_dir, i))
xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags,
deferred);
}
}
xe_pt_free(pt);
}
/**
* xe_pt_clear() - Clear a page-table.
* @xe: xe device.
* @pt: The page-table.
*
* Clears page-table by setting to zero.
*/
void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt)
{
struct iosys_map *map = &pt->bo->vmap;
xe_map_memset(xe, map, 0, 0, SZ_4K);
}
/**
* DOC: Pagetable building
*
* Below we use the term "page-table" for both page-directories, containing
* pointers to lower level page-directories or page-tables, and level 0
* page-tables that contain only page-table-entries pointing to memory pages.
*
* When inserting an address range in an already existing page-table tree
* there will typically be a set of page-tables that are shared with other
* address ranges, and a set that are private to this address range.
* The set of shared page-tables can be at most two per level,
* and those can't be updated immediately because the entries of those
* page-tables may still be in use by the gpu for other mappings. Therefore
* when inserting entries into those, we instead stage those insertions by
* adding insertion data into struct xe_vm_pgtable_update structures. This
* data, (subtrees for the cpu and page-table-entries for the gpu) is then
* added in a separate commit step. CPU-data is committed while still under the
* vm lock, the object lock and for userptr, the notifier lock in read mode.
* The GPU async data is committed either by the GPU or CPU after fulfilling
* relevant dependencies.
* For non-shared page-tables (and, in fact, for shared ones that aren't
* existing at the time of staging), we add the data in-place without the
* special update structures. This private part of the page-table tree will
* remain disconnected from the vm page-table tree until data is committed to
* the shared page tables of the vm tree in the commit phase.
*/
struct xe_pt_update {
/** @update: The update structure we're building for this parent. */
struct xe_vm_pgtable_update *update;
/** @parent: The parent. Used to detect a parent change. */
struct xe_pt *parent;
/** @preexisting: Whether the parent was pre-existing or allocated */
bool preexisting;
};
/**
* struct xe_pt_stage_bind_walk - Walk state for the stage_bind walk.
*/
struct xe_pt_stage_bind_walk {
/** @base: The base class. */
struct xe_pt_walk base;
/* Input parameters for the walk */
/** @vm: The vm we're building for. */
struct xe_vm *vm;
/** @tile: The tile we're building for. */
struct xe_tile *tile;
/** @default_vram_pte: PTE flag only template for VRAM. No address is associated */
u64 default_vram_pte;
/** @default_system_pte: PTE flag only template for System. No address is associated */
u64 default_system_pte;
/** @dma_offset: DMA offset to add to the PTE. */
u64 dma_offset;
/**
* @needs_64K: This address range enforces 64K alignment and
* granularity on VRAM.
*/
bool needs_64K;
/** @clear_pt: clear page table entries during the bind walk */
bool clear_pt;
/**
* @target_leaf_level: Page-table level at which to emit leaf PTEs
* 0 for normal 4K/64K mappings, 1 for 2M huge pages, and 2 for 1G huge
* pages. The walk still traverses from the root down; this field tells
* xe_pt_stage_bind_entry() to treat the selected level as a leaf instead
* of descending further.
*/
u32 target_leaf_level;
/**
* @vma: VMA being mapped
*/
struct xe_vma *vma;
/* Also input, but is updated during the walk*/
/** @curs: The DMA address cursor. */
struct xe_res_cursor *curs;
/** @va_curs_start: The Virtual address corresponding to @curs->start */
u64 va_curs_start;
/* Output */
/** @wupd: Walk output data for page-table updates. */
struct xe_walk_update {
/** @wupd.entries: Caller provided storage. */
struct xe_vm_pgtable_update *entries;
/** @wupd.num_used_entries: Number of update @entries used. */
unsigned int num_used_entries;
/** @wupd.updates: Tracks the update entry at a given level */
struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1];
} wupd;
/* Walk state */
/**
* @l0_end_addr: The end address of the current l0 leaf. Used for
* 64K granularity detection.
*/
u64 l0_end_addr;
/** @addr_64K: The start address of the current 64K chunk. */
u64 addr_64K;
/** @found_64K: Whether @add_64K actually points to a 64K chunk. */
bool found_64K;
};
static int
xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
pgoff_t offset, bool alloc_entries)
{
struct xe_pt_update *upd = &wupd->updates[parent->level];
struct xe_vm_pgtable_update *entry;
/*
* For *each level*, we could only have one active
* struct xt_pt_update at any one time. Once we move on to a
* new parent and page-directory, the old one is complete, and
* updates are either already stored in the build tree or in
* @wupd->entries
*/
if (likely(upd->parent == parent))
return 0;
upd->parent = parent;
upd->preexisting = true;
if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1)
return -EINVAL;
entry = wupd->entries + wupd->num_used_entries++;
upd->update = entry;
entry->ofs = offset;
entry->pt_bo = parent->bo;
entry->pt = parent;
entry->flags = 0;
entry->qwords = 0;
entry->pt_bo->update_index = -1;
if (alloc_entries) {
entry->pt_entries = kmalloc_objs(*entry->pt_entries, XE_PDES);
if (!entry->pt_entries)
return -ENOMEM;
}
return 0;
}
/*
* NOTE: This is a very frequently called function so we allow ourselves
* to annotate (using branch prediction hints) the fastpath of updating a
* non-pre-existing pagetable with leaf ptes.
*/
static int
xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
pgoff_t offset, struct xe_pt *xe_child, u64 pte)
{
struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level];
struct xe_pt_update *child_upd = xe_child ?
&xe_walk->wupd.updates[xe_child->level] : NULL;
int ret;
ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true);
if (unlikely(ret))
return ret;
/*
* Register this new pagetable so that it won't be recognized as
* a shared pagetable by a subsequent insertion.
*/
if (unlikely(child_upd)) {
child_upd->update = NULL;
child_upd->parent = xe_child;
child_upd->preexisting = false;
}
if (likely(!upd->preexisting)) {
/* Continue building a non-connected subtree. */
struct iosys_map *map = &parent->bo->vmap;
if (unlikely(xe_child)) {
parent->base.children[offset] = &xe_child->base;
parent->base.staging[offset] = &xe_child->base;
}
xe_pt_write(xe_walk->vm->xe, map, offset, pte);
parent->num_live++;
} else {
/* Shared pt. Stage update. */
unsigned int idx;
struct xe_vm_pgtable_update *entry = upd->update;
idx = offset - entry->ofs;
entry->pt_entries[idx].pt = xe_child;
entry->pt_entries[idx].pte = pte;
entry->qwords++;
}
return 0;
}
static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level,
struct xe_pt_stage_bind_walk *xe_walk)
{
struct xe_bo *bo = xe_vma_bo(xe_walk->vma);
u64 size, dma;
if (level > MAX_HUGEPTE_LEVEL)
return false;
/* Does the virtual range requested cover a huge pte? */
if (!xe_pt_covers(addr, next, level, &xe_walk->base))
return false;
/* null VMA's and purged BO's do not have dma addresses */
if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo)))
return true;
/* if we are clearing page table, no dma addresses*/
if (xe_walk->clear_pt)
return true;
/* Does the DMA segment cover the whole pte? */
if (next - xe_walk->va_curs_start > xe_walk->curs->size)
return false;
/* Is the DMA address huge PTE size aligned? */
size = next - addr;
dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs);
return IS_ALIGNED(dma, size);
}
/*
* Scan the requested mapping to check whether it can be done entirely
* with 64K PTEs.
*/
static bool
xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
{
struct xe_bo *bo = xe_vma_bo(xe_walk->vma);
struct xe_res_cursor curs = *xe_walk->curs;
if (!IS_ALIGNED(addr, SZ_64K))
return false;
if (next > xe_walk->l0_end_addr)
return false;
/* null VMA's and purged BO's do not have dma addresses */
if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo)))
return true;
xe_res_next(&curs, addr - xe_walk->va_curs_start);
for (; addr < next; addr += SZ_64K) {
if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K)
return false;
xe_res_next(&curs, SZ_64K);
}
return addr == next;
}
/*
* For non-compact "normal" 4K level-0 pagetables, we want to try to group
* addresses together in 64K-contigous regions to add a 64K TLB hint for the
* device to the PTE.
* This function determines whether the address is part of such a
* segment. For VRAM in normal pagetables, this is strictly necessary on
* some devices.
*/
static bool
xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
{
/* Address is within an already found 64k region */
if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K)
return true;
xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk);
xe_walk->addr_64K = addr;
return xe_walk->found_64K;
}
static bool xe_pt_huge_leaf_allowed(u64 addr, u64 next, unsigned int level,
struct xe_pt_stage_bind_walk *xe_walk)
{
if (xe_walk->clear_pt)
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
if (!xe_debug_page_size_supported(xe_walk->vm->xe))
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
if (!xe_walk->target_leaf_level)
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
if (level == xe_walk->target_leaf_level)
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
return false;
}
static bool xe_pt_exact_leaf_required_but_invalid(u64 addr, u64 next,
unsigned int level,
struct xe_pt_stage_bind_walk *xe_walk)
{
struct xe_device *xe = xe_walk->vm->xe;
if (!xe_debug_page_size_mode_not_none(xe))
return false;
return !xe_walk->clear_pt &&
xe_walk->target_leaf_level &&
level == xe_walk->target_leaf_level &&
!xe_pt_hugepte_possible(addr, next, level, xe_walk);
}
static int
xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_stage_bind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
u16 pat_index = xe_walk->vma->attr.pat_index;
struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base);
struct xe_vm *vm = xe_walk->vm;
struct xe_pt *xe_child;
bool covers;
int ret = 0;
u64 pte;
if (xe_pt_exact_leaf_required_but_invalid(addr, next, level, xe_walk))
return -EINVAL;
/*
* Is this a leaf entry?
* Always create a 4K leaf at level 0. For huge pages (level > 0),
* validate alignment and size with xe_pt_hugepte_possible().
* When target_leaf_level is non-zero, only that huge-page level is
* accepted for normal bind walks. Clear walks remain unconstrained so
* existing huge leaves can be cleared without descending further.
*/
if (level == 0 || xe_pt_huge_leaf_allowed(addr, next, level, xe_walk)) {
struct xe_res_cursor *curs = xe_walk->curs;
struct xe_bo *bo = xe_vma_bo(xe_walk->vma);
bool is_null_or_purged = xe_vma_is_null(xe_walk->vma) ||
(bo && xe_bo_is_purged(bo));
bool is_vram = is_null_or_purged ? false : xe_res_is_vram(curs);
XE_WARN_ON(xe_walk->va_curs_start != addr);
if (xe_walk->clear_pt) {
pte = 0;
} else {
/*
* For purged BOs, treat like null VMAs - pass address 0.
* The pte_encode_vma will set XE_PTE_NULL flag for scratch mapping.
*/
pte = vm->pt_ops->pte_encode_vma(is_null_or_purged ? 0 :
xe_res_dma(curs) +
xe_walk->dma_offset,
xe_walk->vma,
pat_index, level);
if (!is_null_or_purged)
pte |= is_vram ? xe_walk->default_vram_pte :
xe_walk->default_system_pte;
/*
* Set the XE_PTE_PS64 hint if possible, otherwise if
* this device *requires* 64K PTE size for VRAM, fail.
*/
if (level == 0 && !xe_parent->is_compact) {
if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) {
xe_walk->vma->gpuva.flags |=
XE_VMA_PTE_64K;
pte |= XE_PTE_PS64;
} else if (XE_WARN_ON(xe_walk->needs_64K &&
is_vram)) {
return -EINVAL;
}
}
}
ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte);
if (unlikely(ret))
return ret;
if (!is_null_or_purged && !xe_walk->clear_pt)
xe_res_next(curs, next - addr);
xe_walk->va_curs_start = next;
xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level);
*action = ACTION_CONTINUE;
return ret;
}
/*
* Descending to lower level. Determine if we need to allocate a
* new page table or -directory, which we do if there is no
* previous one or there is one we can completely replace.
*/
if (level == 1) {
walk->shifts = xe_normal_pt_shifts;
xe_walk->l0_end_addr = next;
}
covers = xe_pt_covers(addr, next, level, &xe_walk->base);
if (covers || !*child) {
u64 flags = 0;
xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1,
xe_vm_validation_exec(vm));
if (IS_ERR(xe_child))
return PTR_ERR(xe_child);
xe_pt_set_addr(xe_child,
round_down(addr, 1ull << walk->shifts[level]));
if (!covers)
xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);
*child = &xe_child->base;
/*
* Prefer the compact pagetable layout for L0 if possible. Only
* possible if VMA covers entire 2MB region as compact 64k and
* 4k pages cannot be mixed within a 2MB region.
* TODO: Suballocate the pt bo to avoid wasting a lot of
* memory.
*/
if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&
covers && xe_pt_scan_64K(addr, next, xe_walk)) {
walk->shifts = xe_compact_pt_shifts;
xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT;
flags |= XE_PDE_64K;
xe_child->is_compact = true;
}
pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0) | flags;
ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child,
pte);
}
*action = ACTION_SUBTREE;
return ret;
}
static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
.pt_entry = xe_pt_stage_bind_entry,
};
/*
* Default atomic expectations for different allocation scenarios are as follows:
*
* 1. Traditional API: When the VM is not in LR mode:
* - Device atomics are expected to function with all allocations.
*
* 2. Compute/SVM API: When the VM is in LR mode:
* - Device atomics are the default behavior when the bo is placed in a single region.
* - In all other cases device atomics will be disabled with AE=0 until an application
* request differently using a ioctl like madvise.
*/
static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma)
{
if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
return false;
return true;
}
static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma)
{
struct xe_device *xe = vm->xe;
struct xe_bo *bo = xe_vma_bo(vma);
if (!xe->info.has_device_atomics_on_smem ||
vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
return false;
if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE)
return true;
/*
* If a SMEM+LMEM allocation is backed by SMEM, a device
* atomics will cause a gpu page fault and which then
* gets migrated to LMEM, bind such allocations with
* device atomics enabled.
*/
return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) ||
(bo && xe_bo_has_single_placement(bo))));
}
static u32 xe_pt_target_leaf_level_from_bo(struct xe_device *xe,
struct xe_vma *vma)
{
struct xe_bo *bo = xe_vma_bo(vma);
if (!xe_debug_page_size_mode_not_none(xe))
return 0;
if (!bo || !xe_bo_is_vram(bo) || !(bo->flags & XE_BO_FLAG_USER))
return 0;
if (bo->flags & XE_BO_FLAG_NEEDS_1G)
return 2;
if (bo->flags & XE_BO_FLAG_NEEDS_2M)
return 1;
return 0;
}
/**
* xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
* range.
* @tile: The tile we're building for.
* @vma: The vma indicating the address range.
* @range: The range indicating the address range.
* @entries: Storage for the update entries used for connecting the tree to
* the main tree at commit time.
* @num_entries: On output contains the number of @entries used.
* @clear_pt: Clear the page table entries.
*
* This function builds a disconnected page-table tree for a given address
* range. The tree is connected to the main vm tree for the gpu using
* xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind().
* The function builds xe_vm_pgtable_update structures for already existing
* shared page-tables, and non-existing shared and non-shared page-tables
* are built and populated directly.
*
* Return 0 on success, negative error code on error.
*/
static int
xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
struct xe_svm_range *range,
struct xe_vm_pgtable_update *entries,
u32 *num_entries, bool clear_pt)
{
struct xe_device *xe = tile_to_xe(tile);
struct xe_bo *bo = xe_vma_bo(vma);
struct xe_res_cursor curs = {};
struct xe_vm *vm = xe_vma_vm(vma);
struct xe_pt_stage_bind_walk xe_walk = {
.base = {
.ops = &xe_pt_stage_bind_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
.staging = true,
},
.vm = vm,
.tile = tile,
.curs = &curs,
.va_curs_start = range ? xe_svm_range_start(range) :
xe_vma_start(vma),
.vma = vma,
.wupd.entries = entries,
.clear_pt = clear_pt,
};
struct xe_pt *pt = vm->pt_root[tile->id];
int ret;
bool is_purged = false;
/*
* Check if BO is purged:
* - Scratch VMs: Use scratch PTEs (XE_PTE_NULL) for safe zero reads
* - Non-scratch VMs: Clear PTEs to zero (non-present) to avoid mapping to phys addr 0
*
* For non-scratch VMs, we force clear_pt=true so leaf PTEs become completely
* zero instead of creating a PRESENT mapping to physical address 0.
*/
if (bo && xe_bo_is_purged(bo)) {
is_purged = true;
/*
* For non-scratch VMs, a NULL rebind should use zero PTEs
* (non-present), not a present PTE to phys 0.
*/
if (!xe_vm_has_scratch(vm))
xe_walk.clear_pt = true;
}
if (range) {
/* Move this entire thing to xe_svm.c? */
xe_svm_notifier_lock(vm);
if (!xe_svm_range_pages_valid(range)) {
xe_svm_range_debug(range, "BIND PREPARE - RETRY");
xe_svm_notifier_unlock(vm);
return -EAGAIN;
}
if (xe_svm_range_has_dma_mapping(range)) {
xe_res_first_dma(range->pages.dma_addr, 0,
xe_svm_range_size(range),
&curs);
xe_svm_range_debug(range, "BIND PREPARE - MIXED");
} else {
xe_assert(xe, false);
}
/*
* Note, when unlocking the resource cursor dma addresses may become
* stale, but the bind will be aborted anyway at commit time.
*/
xe_svm_notifier_unlock(vm);
}
xe_walk.target_leaf_level = xe_pt_target_leaf_level_from_bo(xe, vma);
xe_walk.needs_64K = (vm->flags & XE_VM_FLAG_64K);
if (clear_pt) {
xe_assert(xe, !range);
curs.size = xe_vma_size(vma);
goto walk_pt;
}
if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) {
xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0;
xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ?
XE_USM_PPGTT_PTE_AE : 0;
}
xe_walk.default_vram_pte |= XE_PPGTT_PTE_DM;
xe_walk.dma_offset = (bo && !is_purged) ? vram_region_gpu_offset(bo->ttm.resource) : 0;
if (!range)
xe_bo_assert_held(bo);
if (!xe_vma_is_null(vma) && !range && !is_purged) {
if (xe_vma_is_userptr(vma))
xe_res_first_dma(to_userptr_vma(vma)->userptr.pages.dma_addr, 0,
xe_vma_size(vma), &curs);
else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
xe_vma_size(vma), &curs);
else
xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
xe_vma_size(vma), &curs);
} else if (!range) {
curs.size = xe_vma_size(vma);
}
walk_pt:
ret = xe_pt_walk_range(&pt->base, pt->level,
range ? xe_svm_range_start(range) : xe_vma_start(vma),
range ? xe_svm_range_end(range) : xe_vma_end(vma),
&xe_walk.base);
*num_entries = xe_walk.wupd.num_used_entries;
return ret;
}
/**
* xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
* shared pagetable.
* @addr: The start address within the non-shared pagetable.
* @end: The end address within the non-shared pagetable.
* @level: The level of the non-shared pagetable.
* @walk: Walk info. The function adjusts the walk action.
* @action: next action to perform (see enum page_walk_action)
* @offset: Ignored on input, First non-shared entry on output.
* @end_offset: Ignored on input, Last non-shared entry + 1 on output.
*
* A non-shared page-table has some entries that belong to the address range
* and others that don't. This function determines the entries that belong
* fully to the address range. Depending on level, some entries may
* partially belong to the address range (that can't happen at level 0).
* The function detects that and adjust those offsets to not include those
* partial entries. Iff it does detect partial entries, we know that there must
* be shared page tables also at lower levels, so it adjusts the walk action
* accordingly.
*
* Return: true if there were non-shared entries, false otherwise.
*/
static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level,
struct xe_pt_walk *walk,
enum page_walk_action *action,
pgoff_t *offset, pgoff_t *end_offset)
{
u64 size = 1ull << walk->shifts[level];
*offset = xe_pt_offset(addr, level, walk);
*end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset;
if (!level)
return true;
/*
* If addr or next are not size aligned, there are shared pts at lower
* level, so in that case traverse down the subtree
*/
*action = ACTION_CONTINUE;
if (!IS_ALIGNED(addr, size)) {
*action = ACTION_SUBTREE;
(*offset)++;
}
if (!IS_ALIGNED(end, size)) {
*action = ACTION_SUBTREE;
(*end_offset)--;
}
return *end_offset > *offset;
}
struct xe_pt_zap_ptes_walk {
/** @base: The walk base-class */
struct xe_pt_walk base;
/* Input parameters for the walk */
/** @tile: The tile we're building for */
struct xe_tile *tile;
/* Output */
/** @needs_invalidate: Whether we need to invalidate TLB*/
bool needs_invalidate;
};
static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_zap_ptes_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
struct xe_pt *xe_child;
pgoff_t end_offset;
XE_WARN_ON(!level);
/*
* Below would be unexpected behavior that needs to be root caused
* but better warn and bail than crash the driver.
*/
if (XE_WARN_ON(!*child))
return 0;
xe_child = container_of(*child, typeof(*xe_child), base);
/*
* Note that we're called from an entry callback, and we're dealing
* with the child of that entry rather than the parent, so need to
* adjust level down.
*/
if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset,
&end_offset)) {
xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,
offset * sizeof(u64), 0,
(end_offset - offset) * sizeof(u64));
xe_walk->needs_invalidate = true;
}
return 0;
}
static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
.pt_entry = xe_pt_zap_ptes_entry,
};
/**
* xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
* @tile: The tile we're zapping for.
* @vma: GPU VMA detailing address range.
*
* Eviction and Userptr invalidation needs to be able to zap the
* gpu ptes of a given address range in pagefaulting mode.
* In order to be able to do that, that function needs access to the shared
* page-table entrieaso it can either clear the leaf PTEs or
* clear the pointers to lower-level page-tables. The caller is required
* to hold the necessary locks to ensure neither the page-table connectivity
* nor the page-table entries of the range is updated from under us.
*
* Return: Whether ptes were actually updated and a TLB invalidation is
* required.
*/
bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
{
struct xe_pt_zap_ptes_walk xe_walk = {
.base = {
.ops = &xe_pt_zap_ptes_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
},
.tile = tile,
};
struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated);
if (xe_vma_bo(vma))
xe_bo_assert_held(xe_vma_bo(vma));
else if (xe_vma_is_userptr(vma))
lockdep_assert_held(&xe_vma_vm(vma)->svm.gpusvm.notifier_lock);
if (!(pt_mask & BIT(tile->id)))
return false;
(void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
xe_vma_end(vma), &xe_walk.base);
return xe_walk.needs_invalidate;
}
/**
* xe_pt_zap_ptes_range() - Zap (zero) gpu ptes of a SVM range
* @tile: The tile we're zapping for.
* @vm: The VM we're zapping for.
* @range: The SVM range we're zapping for.
*
* SVM invalidation needs to be able to zap the gpu ptes of a given address
* range. In order to be able to do that, that function needs access to the
* shared page-table entries so it can either clear the leaf PTEs or
* clear the pointers to lower-level page-tables. The caller is required
* to hold the SVM notifier lock.
*
* Return: Whether ptes were actually updated and a TLB invalidation is
* required.
*/
bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
struct xe_svm_range *range)
{
struct xe_pt_zap_ptes_walk xe_walk = {
.base = {
.ops = &xe_pt_zap_ptes_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
},
.tile = tile,
};
struct xe_pt *pt = vm->pt_root[tile->id];
u8 pt_mask = (range->tile_present & ~range->tile_invalidated);
/*
* Locking rules:
*
* - notifier_lock (write): full protection against page table changes
* and MMU notifier invalidations.
*
* - notifier_lock (read) + vm_lock (write): combined protection against
* invalidations and concurrent page table modifications. (e.g., madvise)
*
*/
lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
lockdep_is_held_type(&vm->lock, 0)));
if (!(pt_mask & BIT(tile->id)))
return false;
(void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range),
xe_svm_range_end(range), &xe_walk.base);
return xe_walk.needs_invalidate;
}
static void
xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
struct iosys_map *map, void *data,
u32 qword_ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update)
{
struct xe_pt_entry *ptes = update->pt_entries;
u64 *ptr = data;
u32 i;
/*
* @qword_ofs is the absolute entry offset within the page table, while
* @ptes is indexed relative to @update->ofs (its first entry). The GPU
* path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized
* chunks, calling this with an advancing @qword_ofs but a fresh @data
* pointer per chunk, so translate back into a @ptes index rather than
* assuming the chunk starts at ptes[0].
*/
for (i = 0; i < num_qwords; i++) {
u32 idx = qword_ofs - update->ofs + i;
if (map)
xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
sizeof(u64), u64, ptes[idx].pte);
else
ptr[i] = ptes[idx].pte;
}
}
static void xe_pt_cancel_bind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
u32 i, j;
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
if (!pt)
continue;
if (pt->level) {
for (j = 0; j < entries[i].qwords; j++)
xe_pt_destroy(entries[i].pt_entries[j].pt,
xe_vma_vm(vma)->flags, NULL);
}
kfree(entries[i].pt_entries);
entries[i].pt_entries = NULL;
entries[i].qwords = 0;
}
}
#define XE_INVALID_VMA ((struct xe_vma *)(0xdeaddeadull))
static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)
{
struct xe_vm *vm;
if (vma == XE_INVALID_VMA)
return;
vm = xe_vma_vm(vma);
lockdep_assert_held(&vm->lock);
if (!xe_vma_has_no_bo(vma))
dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
xe_vm_assert_held(vm);
}
static void xe_pt_commit_locks_assert(struct xe_vma *vma)
{
struct xe_vm *vm;
if (vma == XE_INVALID_VMA)
return;
vm = xe_vma_vm(vma);
xe_pt_commit_prepare_locks_assert(vma);
if (xe_vma_is_userptr(vma))
xe_svm_assert_held_read_or_inject_write(vm);
}
static void xe_pt_commit(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, struct llist_head *deferred)
{
u32 i, j;
xe_pt_commit_locks_assert(vma);
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
int j_ = j + entries[i].ofs;
pt_dir->children[j_] = pt_dir->staging[j_];
xe_pt_destroy(oldpte, (vma == XE_INVALID_VMA) ? 0 :
xe_vma_vm(vma)->flags, deferred);
}
}
}
static void xe_pt_abort_bind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, bool rebind)
{
int i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = num_entries - 1; i >= 0; --i) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!rebind)
pt->num_live -= entries[i].qwords;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
u32 j_ = j + entries[i].ofs;
struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_);
struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;
xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
}
}
}
static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, bool rebind)
{
u32 i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!rebind)
pt->num_live += entries[i].qwords;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
u32 j_ = j + entries[i].ofs;
struct xe_pt *newpte = entries[i].pt_entries[j].pt;
struct xe_pt *oldpte = NULL;
if (xe_pt_entry_staging(pt_dir, j_))
oldpte = xe_pt_entry_staging(pt_dir, j_);
pt_dir->staging[j_] = &newpte->base;
entries[i].pt_entries[j].pt = oldpte;
}
}
}
static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
u32 i;
for (i = 0; i < num_entries; i++)
kfree(entries[i].pt_entries);
}
static int
xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
struct xe_svm_range *range,
struct xe_vm_pgtable_update *entries,
u32 *num_entries, bool invalidate_on_bind)
{
int err;
*num_entries = 0;
err = xe_pt_stage_bind(tile, vma, range, entries, num_entries,
invalidate_on_bind);
if (!err)
xe_tile_assert(tile, *num_entries);
return err;
}
static void xe_vm_dbg_print_entries(struct xe_device *xe,
const struct xe_vm_pgtable_update *entries,
unsigned int num_entries, bool bind)
#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
{
unsigned int i;
vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind",
num_entries);
for (i = 0; i < num_entries; i++) {
const struct xe_vm_pgtable_update *entry = &entries[i];
struct xe_pt *xe_pt = entry->pt;
u64 page_size = 1ull << xe_pt_shift(xe_pt->level);
u64 end;
u64 start;
xe_assert(xe, !entry->pt->is_compact);
start = entry->ofs * page_size;
end = start + page_size * entry->qwords;
vm_dbg(&xe->drm,
"\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n",
i, xe_pt->level, entry->ofs, entry->qwords,
xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0);
}
}
#else
{}
#endif
static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
{
int i;
for (i = 0; i < num_syncs; i++) {
struct dma_fence *fence = syncs[i].fence;
if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
&fence->flags))
return false;
}
return true;
}
static int job_test_add_deps(struct xe_sched_job *job,
struct dma_resv *resv,
enum dma_resv_usage usage)
{
if (!job) {
if (!dma_resv_test_signaled(resv, usage))
return -ETIME;
return 0;
}
return xe_sched_job_add_deps(job, resv, usage);
}
static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job)
{
struct xe_bo *bo = xe_vma_bo(vma);
xe_bo_assert_held(bo);
if (bo && !bo->vm)
return job_test_add_deps(job, bo->ttm.base.resv,
DMA_RESV_USAGE_KERNEL);
return 0;
}
static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
struct xe_sched_job *job)
{
int err = 0;
/*
* No need to check for is_cpu_addr_mirror here as vma_add_deps is a
* NOP if VMA is_cpu_addr_mirror
*/
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if (!op->map.immediate && xe_vm_in_fault_mode(vm))
break;
err = vma_add_deps(op->map.vma, job);
break;
case DRM_GPUVA_OP_REMAP:
if (op->remap.prev)
err = vma_add_deps(op->remap.prev, job);
if (!err && op->remap.next)
err = vma_add_deps(op->remap.next, job);
break;
case DRM_GPUVA_OP_UNMAP:
break;
case DRM_GPUVA_OP_PREFETCH:
err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job);
break;
case DRM_GPUVA_OP_DRIVER:
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
return err;
}
static int xe_pt_vm_dependencies(struct xe_sched_job *job,
struct xe_tlb_inval_job *ijob,
struct xe_tlb_inval_job *mjob,
struct xe_vm *vm,
struct xe_vma_ops *vops,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_range_fence_tree *rftree)
{
struct xe_range_fence *rtfence;
struct dma_fence *fence;
struct xe_vma_op *op;
int err = 0, i;
xe_vm_assert_held(vm);
if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
return -ETIME;
if (!job && !xe_exec_queue_is_idle(pt_update_ops->q))
return -ETIME;
if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) {
err = job_test_add_deps(job, xe_vm_resv(vm),
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_BOOKKEEP :
DMA_RESV_USAGE_KERNEL);
if (err)
return err;
}
rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start,
pt_update_ops->last);
while (rtfence) {
fence = rtfence->fence;
if (!dma_fence_is_signaled(fence)) {
/*
* Is this a CPU update? GPU is busy updating, so return
* an error
*/
if (!job)
return -ETIME;
dma_fence_get(fence);
err = drm_sched_job_add_dependency(&job->drm, fence);
if (err)
return err;
}
rtfence = xe_range_fence_tree_next(rtfence,
pt_update_ops->start,
pt_update_ops->last);
}
list_for_each_entry(op, &vops->list, link) {
err = op_add_deps(vm, op, job);
if (err)
return err;
}
for (i = 0; job && !err && i < vops->num_syncs; i++)
err = xe_sync_entry_add_deps(&vops->syncs[i], job);
if (job) {
if (ijob) {
err = xe_tlb_inval_job_alloc_dep(ijob);
if (err)
return err;
}
if (mjob) {
err = xe_tlb_inval_job_alloc_dep(mjob);
if (err)
return err;
}
}
return err;
}
static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
{
struct xe_vma_ops *vops = pt_update->vops;
struct xe_vm *vm = vops->vm;
struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[pt_update->tile_id];
return xe_pt_vm_dependencies(pt_update->job, pt_update->ijob,
pt_update->mjob, vm, pt_update->vops,
pt_update_ops, rftree);
}
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
/*
* Acquire/release the svm notifier_lock around xe_pt_svm_userptr_pre_commit()
* and the matching late release in xe_pt_update_ops_run(). Read mode by
* default; write mode when CONFIG_DRM_XE_USERPTR_INVAL_INJECT is on,
* because a userptr op in this critical section may invoke the injected
* xe_vma_userptr_force_invalidate() path that calls
* drm_gpusvm_unmap_pages() with ctx->in_notifier=true, which requires the
* lock held for write.
*/
static void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm)
{
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
down_write(&vm->svm.gpusvm.notifier_lock);
#else
xe_svm_notifier_lock(vm);
#endif
}
static void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm)
{
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
up_write(&vm->svm.gpusvm.notifier_lock);
#else
xe_svm_notifier_unlock(vm);
#endif
}
#else
static inline void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) { }
static inline void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) { }
#endif
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
{
u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;
static u32 count;
if (count++ % divisor == divisor - 1) {
uvma->userptr.divisor = divisor << 1;
return true;
}
return false;
}
#else
static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
{
return false;
}
#endif
static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
struct xe_vm_pgtable_update_ops *pt_update)
{
struct xe_userptr_vma *uvma;
unsigned long notifier_seq;
xe_svm_assert_held_read_or_inject_write(vm);
if (!xe_vma_is_userptr(vma))
return 0;
uvma = to_userptr_vma(vma);
if (xe_pt_userptr_inject_eagain(uvma))
xe_vma_userptr_force_invalidate(uvma);
notifier_seq = uvma->userptr.pages.notifier_seq;
if (!mmu_interval_read_retry(&uvma->userptr.notifier,
notifier_seq))
return 0;
if (xe_vm_in_fault_mode(vm))
return -EAGAIN;
/*
* Just continue the operation since exec or rebind worker
* will take care of rebinding.
*/
return 0;
}
static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
struct xe_vm_pgtable_update_ops *pt_update)
{
int err = 0;
xe_svm_assert_held_read_or_inject_write(vm);
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if (!op->map.immediate && xe_vm_in_fault_mode(vm))
break;
err = vma_check_userptr(vm, op->map.vma, pt_update);
break;
case DRM_GPUVA_OP_REMAP:
if (op->remap.prev && !op->remap.skip_prev)
err = vma_check_userptr(vm, op->remap.prev, pt_update);
if (!err && op->remap.next && !op->remap.skip_next)
err = vma_check_userptr(vm, op->remap.next, pt_update);
break;
case DRM_GPUVA_OP_UNMAP:
break;
case DRM_GPUVA_OP_PREFETCH:
if (xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va))) {
struct xe_svm_range *range = op->map_range.range;
unsigned long i;
xe_assert(vm->xe,
xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va)));
xa_for_each(&op->prefetch_range.range, i, range) {
xe_svm_range_debug(range, "PRE-COMMIT");
if (!xe_svm_range_pages_valid(range)) {
xe_svm_range_debug(range, "PRE-COMMIT - RETRY");
return -ENODATA;
}
}
} else {
err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), pt_update);
}
break;
#if IS_ENABLED(CONFIG_DRM_XE_GPUSVM)
case DRM_GPUVA_OP_DRIVER:
if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
struct xe_svm_range *range = op->map_range.range;
xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
xe_svm_range_debug(range, "PRE-COMMIT");
if (!xe_svm_range_pages_valid(range)) {
xe_svm_range_debug(range, "PRE-COMMIT - RETRY");
return -EAGAIN;
}
}
break;
#endif
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
return err;
}
static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
{
struct xe_vm *vm = pt_update->vops->vm;
struct xe_vma_ops *vops = pt_update->vops;
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[pt_update->tile_id];
struct xe_vma_op *op;
int err;
err = xe_pt_pre_commit(pt_update);
if (err)
return err;
xe_pt_svm_userptr_notifier_lock(vm);
list_for_each_entry(op, &vops->list, link) {
err = op_check_svm_userptr(vm, op, pt_update_ops);
if (err) {
xe_pt_svm_userptr_notifier_unlock(vm);
break;
}
}
return err;
}
#endif
struct xe_pt_stage_unbind_walk {
/** @base: The pagewalk base-class. */
struct xe_pt_walk base;
/* Input parameters for the walk */
/** @tile: The tile we're unbinding from. */
struct xe_tile *tile;
/**
* @modified_start: Walk range start, modified to include any
* shared pagetables that we're the only user of and can thus
* treat as private.
*/
u64 modified_start;
/** @modified_end: Walk range start, modified like @modified_start. */
u64 modified_end;
/** @prl: Backing pointer to page reclaim list in pt_update_ops */
struct xe_page_reclaim_list *prl;
/* Output */
/* @wupd: Structure to track the page-table updates we're building */
struct xe_walk_update wupd;
};
/*
* Check whether this range is the only one populating this pagetable,
* and in that case, update the walk range checks so that higher levels don't
* view us as a shared pagetable.
*/
static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
const struct xe_pt *child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_stage_unbind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
unsigned int shift = walk->shifts[level];
u64 size = 1ull << shift;
if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) &&
((next - addr) >> shift) == child->num_live) {
u64 size = 1ull << walk->shifts[level + 1];
*action = ACTION_CONTINUE;
if (xe_walk->modified_start >= addr)
xe_walk->modified_start = round_down(addr, size);
if (xe_walk->modified_end <= next)
xe_walk->modified_end = round_up(next, size);
return true;
}
return false;
}
static int generate_reclaim_entry(struct xe_tile *tile,
struct xe_page_reclaim_list *prl,
u64 pte, struct xe_pt *xe_child)
{
struct xe_gt *gt = tile->primary_gt;
struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries;
bool is_2m = xe_child->level == 1 && (pte & XE_PDE_PS_2M);
bool is_64k = xe_child->level == 0 && ((pte & XE_PTE_PS64) || xe_child->is_compact);
u32 page_shift = is_2m ? ilog2(SZ_2M) : is_64k ? ilog2(SZ_64K) : ilog2(SZ_4K);
/* Physical address bits start at page shift: 2M->[51:21], 64K->[51:16], 4K->[51:12] */
u64 phys_addr = pte & XE_PAGE_ADDR_MASK(page_shift);
/* Page address is relative to 4K page regardless of entry level */
u64 phys_page = phys_addr >> XE_PTE_SHIFT;
int num_entries = prl->num_entries;
u32 reclamation_size = page_shift - XE_PTE_SHIFT;
xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL);
xe_tile_assert(tile, reclaim_entries);
xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1);
if (!xe_page_reclaim_list_valid(prl))
return -EINVAL;
/**
* reclamation_size indicates the size of the page to be
* invalidated and flushed from non-coherent cache.
* Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes.
* Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim
*/
if (is_2m) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT, 1);
} else if (is_64k) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1);
} else if (xe_child->level == 0) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1);
} else {
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"unsupported PTE level=%u pte=%#llx",
xe_child->level, pte);
return -EINVAL;
}
reclaim_entries[num_entries].qw =
FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) |
FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) |
FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) |
FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20);
prl->num_entries++;
vm_dbg(&tile_to_xe(tile)->drm,
"PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n",
xe_child->level, pte, reclamation_size, num_entries);
return 0;
}
static int add_pte_to_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl,
struct xe_pt *xe_child, u64 pte, u64 addr)
{
/*
* In rare scenarios, pte may not be written yet due to racy conditions.
* In such cases, invalidate the PRL and fallback to full PPC invalidation.
*/
if (!pte) {
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"found zero pte at addr=%#llx", addr);
return -EINVAL;
}
/* Ensure it is a defined page */
xe_tile_assert(tile, xe_child->level == 0 ||
(pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G)));
/* Account for NULL terminated entry on end (-1) */
if (prl->num_entries >= XE_PAGE_RECLAIM_MAX_ENTRIES - 1) {
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"overflow while adding pte=%#llx", pte);
return -ENOSPC;
}
return generate_reclaim_entry(tile, prl, pte, xe_child);
}
static bool add_compact_pt_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl,
struct xe_device *xe, struct xe_pt *compact_pt, u64 addr)
{
struct iosys_map *map = &compact_pt->bo->vmap;
for (pgoff_t i = 0; i < SZ_2M / SZ_64K && xe_page_reclaim_list_valid(prl); i++) {
u64 pte = xe_map_rd(xe, map, i * sizeof(u64), u64);
if (add_pte_to_prl(tile, prl, compact_pt, pte, addr + i * SZ_64K))
break;
}
return xe_page_reclaim_list_valid(prl);
}
static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
struct xe_pt_stage_unbind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
struct xe_page_reclaim_list *prl = xe_walk->prl;
struct xe_tile *tile = xe_walk->tile;
struct xe_device *xe = tile_to_xe(tile);
pgoff_t first = xe_pt_offset(addr, xe_child->level, walk);
bool killed;
XE_WARN_ON(!*child);
XE_WARN_ON(!level);
/* Check for leaf node */
if (prl && xe_page_reclaim_list_valid(prl) &&
xe_child->level <= MAX_HUGEPTE_LEVEL) {
struct iosys_map *leaf_map = &xe_child->bo->vmap;
pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk);
for (pgoff_t i = 0; i < count; i++) {
u64 pte;
/*
* If not a leaf pt, skip unless non-leaf pt is interleaved between
* leaf ptes which causes the page walk to skip over the child leaves
*/
if (xe_child->base.children && xe_child->base.children[first + i]) {
u64 pt_size = 1ULL << walk->shifts[xe_child->level];
bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) ||
(i == count - 1 && !IS_ALIGNED(next, pt_size));
struct xe_pt *child_pt =
container_of(xe_child->base.children[first + i],
struct xe_pt, base);
/* Compact PTs always fill a full 2M-aligned slot, never an edge. */
XE_WARN_ON(child_pt->is_compact && edge_pt);
if (edge_pt)
continue;
/* Walker never descends into compact PTs, descend now */
if (child_pt->is_compact) {
if (!add_compact_pt_prl(tile, prl, xe, child_pt,
addr + (u64)i * pt_size))
break;
} else {
xe_page_reclaim_list_abort(tile->primary_gt,
prl,
"PT is skipped by walk at level=%u offset=%lu",
xe_child->level, first + i);
break;
}
continue;
}
pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
if (add_pte_to_prl(tile, prl, xe_child, pte, addr))
break;
/* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */
if (pte & XE_PTE_PS64)
i += 15; /* Skip other 15 consecutive 4K pages in the 64K page */
}
}
killed = xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
/*
* Verify if any PTE are potentially dropped at non-leaf levels, either from being
* killed or the page walk covers the region.
*/
if (prl && xe_page_reclaim_list_valid(prl) &&
xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) {
bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base);
/*
* If aborting page walk early (kill) or page walk completes the full range
* we need to invalidate the PRL.
*/
if (killed || covered)
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"kill at level=%u addr=%#llx next=%#llx num_live=%u",
level, addr, next, xe_child->num_live);
}
return 0;
}
static int
xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_stage_unbind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
pgoff_t end_offset;
u64 size = 1ull << walk->shifts[--level];
int err;
if (!IS_ALIGNED(addr, size))
addr = xe_walk->modified_start;
if (!IS_ALIGNED(next, size))
next = xe_walk->modified_end;
/* Parent == *child is the root pt. Don't kill it. */
if (parent != *child &&
xe_pt_check_kill(addr, next, level, xe_child, action, walk))
return 0;
if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset,
&end_offset))
return 0;
err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
if (err)
return err;
xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
return 0;
}
static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = {
.pt_entry = xe_pt_stage_unbind_entry,
.pt_post_descend = xe_pt_stage_unbind_post_descend,
};
/**
* xe_pt_stage_unbind() - Build page-table update structures for an unbind
* operation
* @tile: The tile we're unbinding for.
* @vm: The vm
* @vma: The vma we're unbinding.
* @range: The range we're unbinding.
* @entries: Caller-provided storage for the update structures.
*
* Builds page-table update structures for an unbind operation. The function
* will attempt to remove all page-tables that we're the only user
* of, and for that to work, the unbind operation must be committed in the
* same critical section that blocks racing binds to the same page-table tree.
*
* Return: The number of entries used.
*/
static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
struct xe_vm *vm,
struct xe_vma *vma,
struct xe_svm_range *range,
struct xe_vm_pgtable_update *entries)
{
u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
struct xe_vm_pgtable_update_op *pt_update_op =
container_of(entries, struct xe_vm_pgtable_update_op, entries[0]);
struct xe_pt_stage_unbind_walk xe_walk = {
.base = {
.ops = &xe_pt_stage_unbind_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
.staging = true,
},
.tile = tile,
.modified_start = start,
.modified_end = end,
.wupd.entries = entries,
.prl = pt_update_op->prl,
};
struct xe_pt *pt = vm->pt_root[tile->id];
(void)xe_pt_walk_shared(&pt->base, pt->level, start, end,
&xe_walk.base);
return xe_walk.wupd.num_used_entries;
}
static void
xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
struct xe_tile *tile, struct iosys_map *map,
void *ptr, u32 qword_ofs, u32 num_qwords,
const struct xe_vm_pgtable_update *update)
{
struct xe_vm *vm = pt_update->vops->vm;
u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
int i;
if (map && map->is_iomem)
for (i = 0; i < num_qwords; ++i)
xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
sizeof(u64), u64, empty);
else if (map)
memset64(map->vaddr + qword_ofs * sizeof(u64), empty,
num_qwords);
else
memset64(ptr, empty, num_qwords);
}
static void xe_pt_abort_unbind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
int i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = num_entries - 1; i >= 0; --i) {
struct xe_vm_pgtable_update *entry = &entries[i];
struct xe_pt *pt = entry->pt;
struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
pt->num_live += entry->qwords;
if (!pt->level)
continue;
for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
pt_dir->staging[j] =
entries[i].pt_entries[j - entry->ofs].pt ?
&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
}
}
static void
xe_pt_commit_prepare_unbind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
int i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = 0; i < num_entries; ++i) {
struct xe_vm_pgtable_update *entry = &entries[i];
struct xe_pt *pt = entry->pt;
struct xe_pt_dir *pt_dir;
pt->num_live -= entry->qwords;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
entry->pt_entries[j - entry->ofs].pt =
xe_pt_entry_staging(pt_dir, j);
pt_dir->staging[j] = NULL;
}
}
}
static void
xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
u64 start, u64 end)
{
u64 last;
u32 current_op = pt_update_ops->current_op;
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
int i, level = 0;
for (i = 0; i < pt_op->num_entries; i++) {
const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
if (entry->pt->level > level)
level = entry->pt->level;
}
/* Greedy (non-optimal) calculation but simple */
start = ALIGN_DOWN(start, 0x1ull << xe_pt_shift(level));
last = ALIGN(end, 0x1ull << xe_pt_shift(level)) - 1;
if (start < pt_update_ops->start)
pt_update_ops->start = start;
if (last > pt_update_ops->last)
pt_update_ops->last = last;
}
static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma)
{
int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0;
if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv,
xe->info.tile_count << shift);
return 0;
}
static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma, bool invalidate_on_bind)
{
u32 current_op = pt_update_ops->current_op;
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
int err;
xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
xe_bo_assert_held(xe_vma_bo(vma));
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"Preparing bind, with range [%llx...%llx)\n",
xe_vma_start(vma), xe_vma_end(vma) - 1);
pt_op->vma = NULL;
pt_op->bind = true;
pt_op->rebind = BIT(tile->id) & vma->tile_present;
err = vma_reserve_fences(tile_to_xe(tile), vma);
if (err)
return err;
err = xe_pt_prepare_bind(tile, vma, NULL, pt_op->entries,
&pt_op->num_entries, invalidate_on_bind);
if (!err) {
xe_tile_assert(tile, pt_op->num_entries <=
ARRAY_SIZE(pt_op->entries));
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, true);
xe_pt_update_ops_rfence_interval(pt_update_ops,
xe_vma_start(vma),
xe_vma_end(vma));
++pt_update_ops->current_op;
pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
/*
* If rebind, we have to invalidate TLB on !LR vms to invalidate
* cached PTEs point to freed memory. On LR vms this is done
* automatically when the context is re-enabled by the rebind worker,
* or in fault mode it was invalidated on PTE zapping.
*
* If rebind, we have to invalidate TLB on context based TLB invalidation
* LR vms, as they cannot be relied on context re-enable.
*
* If !rebind, and scratch enabled VMs, there is a chance the scratch
* PTE is already cached in the TLB so it needs to be invalidated.
* On !LR VMs this is done in the ring ops preceding a batch, but on
* LR, in particular on user-space batch buffer chaining, it needs to
* be done here.
*/
if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
xe_vm_in_lr_mode(vm)))
pt_update_ops->needs_invalidation = true;
else if (pt_op->rebind && xe_vm_in_preempt_fence_mode(vm) &&
vm->xe->info.has_ctx_tlb_inval)
pt_update_ops->needs_invalidation = true;
else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
/* We bump also if batch_invalidate_tlb is true */
vm->tlb_flush_seqno++;
vma->tile_staged |= BIT(tile->id);
pt_op->vma = vma;
xe_pt_commit_prepare_bind(vma, pt_op->entries,
pt_op->num_entries, pt_op->rebind);
} else {
xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
}
return err;
}
static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma, struct xe_svm_range *range)
{
u32 current_op = pt_update_ops->current_op;
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
int err;
xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma));
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"Preparing bind, with range [%lx...%lx)\n",
xe_svm_range_start(range), xe_svm_range_end(range) - 1);
pt_op->vma = NULL;
pt_op->bind = true;
pt_op->rebind = BIT(tile->id) & range->tile_present;
err = xe_pt_prepare_bind(tile, vma, range, pt_op->entries,
&pt_op->num_entries, false);
if (!err) {
xe_tile_assert(tile, pt_op->num_entries <=
ARRAY_SIZE(pt_op->entries));
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, true);
xe_pt_update_ops_rfence_interval(pt_update_ops,
xe_svm_range_start(range),
xe_svm_range_end(range));
++pt_update_ops->current_op;
pt_update_ops->needs_svm_lock = true;
pt_op->vma = vma;
xe_pt_commit_prepare_bind(vma, pt_op->entries,
pt_op->num_entries, pt_op->rebind);
} else {
xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
}
return err;
}
static int unbind_op_prepare(struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma)
{
struct xe_device *xe = tile_to_xe(tile);
u32 current_op = pt_update_ops->current_op;
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
int err;
if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
return 0;
xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
xe_bo_assert_held(xe_vma_bo(vma));
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"Preparing unbind, with range [%llx...%llx)\n",
xe_vma_start(vma), xe_vma_end(vma) - 1);
pt_op->vma = vma;
pt_op->bind = false;
pt_op->rebind = false;
/*
* Maintain one PRL located in pt_update_ops that all others in unbind op reference.
* Ensure that PRL is allocated only once, and if invalidated, remains an invalidated PRL.
*/
if (xe->info.has_page_reclaim_hw_assist &&
xe_page_reclaim_list_is_new(&pt_update_ops->prl))
xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl);
/* Page reclaim may not be needed due to other features, so skip the corresponding VMA */
pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) &&
!xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL;
err = vma_reserve_fences(tile_to_xe(tile), vma);
if (err)
return err;
pt_op->num_entries = xe_pt_stage_unbind(tile, xe_vma_vm(vma),
vma, NULL, pt_op->entries);
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, false);
xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma),
xe_vma_end(vma));
++pt_update_ops->current_op;
pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
pt_update_ops->needs_invalidation = true;
xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
return 0;
}
static bool
xe_pt_op_check_range_skip_invalidation(struct xe_vm_pgtable_update_op *pt_op,
struct xe_svm_range *range)
{
struct xe_vm_pgtable_update *update = pt_op->entries;
XE_WARN_ON(!pt_op->num_entries);
/*
* We can't skip the invalidation if we are removing PTEs that span more
* than the range, do some checks to ensure we are removing PTEs that
* are invalid.
*/
if (pt_op->num_entries > 1)
return false;
if (update->pt->level == 0)
return true;
if (update->pt->level == 1)
return xe_svm_range_size(range) >= SZ_2M;
return false;
}
static int unbind_range_prepare(struct xe_vm *vm,
struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_svm_range *range)
{
u32 current_op = pt_update_ops->current_op;
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
if (!(range->tile_present & BIT(tile->id)))
return 0;
vm_dbg(&vm->xe->drm,
"Preparing unbind, with range [%lx...%lx)\n",
xe_svm_range_start(range), xe_svm_range_end(range) - 1);
pt_op->vma = XE_INVALID_VMA;
pt_op->bind = false;
pt_op->rebind = false;
pt_op->prl = NULL;
pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
pt_op->entries);
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, false);
xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range),
xe_svm_range_end(range));
++pt_update_ops->current_op;
pt_update_ops->needs_svm_lock = true;
pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) ||
xe_vm_has_valid_gpu_mapping(tile, range->tile_present,
range->tile_invalidated) ||
!xe_pt_op_check_range_skip_invalidation(pt_op, range);
xe_pt_commit_prepare_unbind(XE_INVALID_VMA, pt_op->entries,
pt_op->num_entries);
return 0;
}
static int op_prepare(struct xe_vm *vm,
struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma_op *op)
{
int err = 0;
xe_vm_assert_held(vm);
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if ((!op->map.immediate && xe_vm_in_fault_mode(vm) &&
!op->map.invalidate_on_bind) ||
(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
break;
err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma,
op->map.invalidate_on_bind);
pt_update_ops->wait_vm_kernel = true;
break;
case DRM_GPUVA_OP_REMAP:
{
struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
if (xe_vma_is_cpu_addr_mirror(old))
break;
err = unbind_op_prepare(tile, pt_update_ops, old);
if (!err && op->remap.prev && !op->remap.skip_prev) {
err = bind_op_prepare(vm, tile, pt_update_ops,
op->remap.prev, false);
pt_update_ops->wait_vm_bookkeep = true;
}
if (!err && op->remap.next && !op->remap.skip_next) {
err = bind_op_prepare(vm, tile, pt_update_ops,
op->remap.next, false);
pt_update_ops->wait_vm_bookkeep = true;
}
break;
}
case DRM_GPUVA_OP_UNMAP:
{
struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
if (xe_vma_is_cpu_addr_mirror(vma))
break;
err = unbind_op_prepare(tile, pt_update_ops, vma);
break;
}
case DRM_GPUVA_OP_PREFETCH:
{
struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
if (xe_vma_is_cpu_addr_mirror(vma)) {
struct xe_svm_range *range;
unsigned long i;
xa_for_each(&op->prefetch_range.range, i, range) {
err = bind_range_prepare(vm, tile, pt_update_ops,
vma, range);
/*
* Don't tell user space to retry, rather let
* page faults fixup the pages.
*/
if (err == -EAGAIN)
err = -ENODATA;
if (err)
return err;
}
} else {
err = bind_op_prepare(vm, tile, pt_update_ops, vma, false);
pt_update_ops->wait_vm_kernel = true;
}
break;
}
case DRM_GPUVA_OP_DRIVER:
if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
err = bind_range_prepare(vm, tile, pt_update_ops,
op->map_range.vma,
op->map_range.range);
} else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) {
err = unbind_range_prepare(vm, tile, pt_update_ops,
op->unmap_range.range);
}
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
return err;
}
static void
xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
{
init_llist_head(&pt_update_ops->deferred);
pt_update_ops->current_op = 0;
pt_update_ops->start = ~0x0ull;
pt_update_ops->last = 0x0ull;
pt_update_ops->needs_svm_lock = false;
pt_update_ops->needs_invalidation = false;
xe_page_reclaim_list_init(&pt_update_ops->prl);
}
/**
* xe_pt_update_ops_prepare() - Prepare PT update operations
* @tile: Tile of PT update operations
* @vops: VMA operationa
*
* Prepare PT update operations which includes updating internal PT state,
* allocate memory for page tables, populate page table being pruned in, and
* create PT update operations for leaf insertion / removal.
*
* Return: 0 on success, negative error code on error.
*/
int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
{
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
struct xe_vma_op *op;
int shift = tile->media_gt ? 1 : 0;
int err;
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
xe_pt_update_ops_init(pt_update_ops);
err = dma_resv_reserve_fences(xe_vm_resv(vops->vm),
tile_to_xe(tile)->info.tile_count << shift);
if (err)
return err;
list_for_each_entry(op, &vops->list, link) {
err = op_prepare(vops->vm, tile, pt_update_ops, op);
if (err)
return err;
}
xe_tile_assert(tile, pt_update_ops->current_op <=
pt_update_ops->num_ops);
#ifdef TEST_VM_OPS_ERROR
if (vops->inject_error &&
vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
return -ENOSPC;
#endif
return 0;
}
ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO);
static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma, struct dma_fence *fence,
struct dma_fence *fence2, bool invalidate_on_bind)
{
xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
if (fence2)
dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
}
/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id));
if (invalidate_on_bind)
WRITE_ONCE(vma->tile_invalidated,
vma->tile_invalidated | BIT(tile->id));
else
WRITE_ONCE(vma->tile_invalidated,
vma->tile_invalidated & ~BIT(tile->id));
vma->tile_staged &= ~BIT(tile->id);
if (xe_vma_is_userptr(vma)) {
xe_svm_assert_held_read_or_inject_write(vm);
to_userptr_vma(vma)->userptr.initial_bind = true;
}
/*
* Kick rebind worker if this bind triggers preempt fences and not in
* the rebind worker
*/
if (pt_update_ops->wait_vm_bookkeep &&
xe_vm_in_preempt_fence_mode(vm) &&
!current->mm)
xe_vm_queue_rebind_worker(vm);
}
static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma, struct dma_fence *fence,
struct dma_fence *fence2)
{
xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
if (fence2)
dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
}
vma->tile_present &= ~BIT(tile->id);
if (!vma->tile_present) {
list_del_init(&vma->combined_links.rebind);
if (xe_vma_is_userptr(vma)) {
xe_svm_assert_held_read_or_inject_write(vm);
spin_lock(&vm->userptr.invalidated_lock);
list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link);
spin_unlock(&vm->userptr.invalidated_lock);
}
}
}
static void range_present_and_invalidated_tile(struct xe_vm *vm,
struct xe_svm_range *range,
u8 tile_id)
{
/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
WRITE_ONCE(range->tile_present, range->tile_present | BIT(tile_id));
WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~BIT(tile_id));
}
static void op_commit(struct xe_vm *vm,
struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma_op *op, struct dma_fence *fence,
struct dma_fence *fence2)
{
xe_vm_assert_held(vm);
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) ||
(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
break;
bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,
fence2, op->map.invalidate_on_bind);
break;
case DRM_GPUVA_OP_REMAP:
{
struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
if (xe_vma_is_cpu_addr_mirror(old))
break;
unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2);
if (op->remap.prev && !op->remap.skip_prev)
bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
fence, fence2, false);
if (op->remap.next && !op->remap.skip_next)
bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
fence, fence2, false);
break;
}
case DRM_GPUVA_OP_UNMAP:
{
struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
if (!xe_vma_is_cpu_addr_mirror(vma))
unbind_op_commit(vm, tile, pt_update_ops, vma, fence,
fence2);
break;
}
case DRM_GPUVA_OP_PREFETCH:
{
struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
if (xe_vma_is_cpu_addr_mirror(vma)) {
struct xe_svm_range *range = NULL;
unsigned long i;
xa_for_each(&op->prefetch_range.range, i, range)
range_present_and_invalidated_tile(vm, range, tile->id);
} else {
bind_op_commit(vm, tile, pt_update_ops, vma, fence,
fence2, false);
}
break;
}
case DRM_GPUVA_OP_DRIVER:
{
/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
if (op->subop == XE_VMA_SUBOP_MAP_RANGE)
range_present_and_invalidated_tile(vm, op->map_range.range, tile->id);
else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE)
WRITE_ONCE(op->unmap_range.range->tile_present,
op->unmap_range.range->tile_present &
~BIT(tile->id));
break;
}
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
}
static const struct xe_migrate_pt_update_ops migrate_ops = {
.populate = xe_vm_populate_pgtable,
.clear = xe_migrate_clear_pgtable_callback,
.pre_commit = xe_pt_pre_commit,
};
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops = {
.populate = xe_vm_populate_pgtable,
.clear = xe_migrate_clear_pgtable_callback,
.pre_commit = xe_pt_svm_userptr_pre_commit,
};
#else
static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops;
#endif
static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q,
struct xe_gt *gt)
{
if (xe_gt_is_media_type(gt))
return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler;
return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler;
}
/**
* xe_pt_update_ops_run() - Run PT update operations
* @tile: Tile of PT update operations
* @vops: VMA operationa
*
* Run PT update operations which includes committing internal PT state changes,
* creating job for PT update operations for leaf insertion / removal, and
* installing job fence in various places.
*
* Return: fence on success, negative ERR_PTR on error.
*/
struct dma_fence *
xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
{
struct xe_vm *vm = vops->vm;
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
struct xe_exec_queue *q = pt_update_ops->q;
struct dma_fence *fence, *ifence = NULL, *mfence = NULL;
struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL;
struct xe_range_fence *rfence;
struct xe_vma_op *op;
int err = 0, i;
struct xe_migrate_pt_update update = {
.ops = pt_update_ops->needs_svm_lock ?
&svm_userptr_migrate_ops :
&migrate_ops,
.vops = vops,
.tile_id = tile->id,
};
lockdep_assert_held(&vm->lock);
xe_vm_assert_held(vm);
if (!pt_update_ops->current_op) {
xe_tile_assert(tile, xe_vm_in_fault_mode(vm));
return dma_fence_get_stub();
}
#ifdef TEST_VM_OPS_ERROR
if (vops->inject_error &&
vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
return ERR_PTR(-ENOSPC);
#endif
if (pt_update_ops->needs_invalidation) {
struct xe_dep_scheduler *dep_scheduler =
to_dep_scheduler(q, tile->primary_gt);
ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval,
dep_scheduler, vm,
pt_update_ops->start,
pt_update_ops->last,
XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
if (IS_ERR(ijob)) {
err = PTR_ERR(ijob);
goto kill_vm_tile1;
}
update.ijob = ijob;
/*
* Only add page reclaim for the primary GT. Media GT does not have
* any PPC to flush, so enabling the PPC flush bit for media is
* effectively a NOP and provides no performance benefit nor
* interfere with primary GT.
*/
if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) {
xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl);
/* Release ref from alloc, job will now handle it */
xe_page_reclaim_list_invalidate(&pt_update_ops->prl);
}
if (tile->media_gt) {
dep_scheduler = to_dep_scheduler(q, tile->media_gt);
mjob = xe_tlb_inval_job_create(q,
&tile->media_gt->tlb_inval,
dep_scheduler, vm,
pt_update_ops->start,
pt_update_ops->last,
XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT);
if (IS_ERR(mjob)) {
err = PTR_ERR(mjob);
goto free_ijob;
}
update.mjob = mjob;
}
}
rfence = kzalloc_obj(*rfence);
if (!rfence) {
err = -ENOMEM;
goto free_ijob;
}
fence = xe_migrate_update_pgtables(tile->migrate, &update);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
goto free_rfence;
}
/* Point of no return - VM killed if failure after this */
for (i = 0; i < pt_update_ops->current_op; ++i) {
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
xe_pt_commit(pt_op->vma, pt_op->entries,
pt_op->num_entries, &pt_update_ops->deferred);
pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */
}
if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,
&xe_range_fence_kfree_ops,
pt_update_ops->start,
pt_update_ops->last, fence))
dma_fence_wait(fence, false);
if (ijob)
ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence);
if (mjob)
mfence = xe_tlb_inval_job_push(mjob, tile->migrate, fence);
if (!mjob && !ijob) {
dma_resv_add_fence(xe_vm_resv(vm), fence,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
list_for_each_entry(op, &vops->list, link)
op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);
} else if (ijob && !mjob) {
dma_resv_add_fence(xe_vm_resv(vm), ifence,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
list_for_each_entry(op, &vops->list, link)
op_commit(vops->vm, tile, pt_update_ops, op, ifence, NULL);
} else {
dma_resv_add_fence(xe_vm_resv(vm), ifence,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
dma_resv_add_fence(xe_vm_resv(vm), mfence,
pt_update_ops->wait_vm_bookkeep ?
DMA_RESV_USAGE_KERNEL :
DMA_RESV_USAGE_BOOKKEEP);
list_for_each_entry(op, &vops->list, link)
op_commit(vops->vm, tile, pt_update_ops, op, ifence,
mfence);
}
if (pt_update_ops->needs_svm_lock)
xe_pt_svm_userptr_notifier_unlock(vm);
/*
* The last fence is only used for zero bind queue idling; migrate
* queues are not exposed to user space.
*/
if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE))
xe_exec_queue_last_fence_set(q, vm, fence);
xe_tlb_inval_job_put(mjob);
xe_tlb_inval_job_put(ijob);
dma_fence_put(ifence);
dma_fence_put(mfence);
return fence;
free_rfence:
kfree(rfence);
free_ijob:
xe_tlb_inval_job_put(mjob);
xe_tlb_inval_job_put(ijob);
kill_vm_tile1:
if (err != -EAGAIN && err != -ENODATA && tile->id)
xe_vm_kill(vops->vm, false);
return ERR_PTR(err);
}
ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO);
/**
* xe_pt_update_ops_fini() - Finish PT update operations
* @tile: Tile of PT update operations
* @vops: VMA operations
*
* Finish PT update operations by committing to destroy page table memory
*/
void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
{
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
int i;
xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
for (i = 0; i < pt_update_ops->current_op; ++i) {
struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
}
xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
}
/**
* xe_pt_update_ops_abort() - Abort PT update operations
* @tile: Tile of PT update operations
* @vops: VMA operationa
*
* Abort PT update operations by unwinding internal PT state
*/
void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
{
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
int i;
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
struct xe_vm_pgtable_update_op *pt_op =
&pt_update_ops->ops[i];
if (!pt_op->vma || i >= pt_update_ops->current_op)
continue;
if (pt_op->bind)
xe_pt_abort_bind(pt_op->vma, pt_op->entries,
pt_op->num_entries,
pt_op->rebind);
else
xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
pt_op->num_entries);
}
xe_pt_update_ops_fini(tile, vops);
}
