diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 341 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 23 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 29 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 37 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_object.c | 13 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 36 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 99 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 5 |
10 files changed, 315 insertions, 280 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 028ab6007873..3e795a099d06 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -133,12 +133,12 @@ struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_agp_tt_create); -int ttm_agp_tt_populate(struct ttm_tt *ttm) +int ttm_agp_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { if (ttm->state != tt_unpopulated) return 0; - return ttm_pool_populate(ttm); + return ttm_pool_populate(ttm, ctx); } EXPORT_SYMBOL(ttm_agp_tt_populate); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c088703777e2..893003fc76a1 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -42,11 +42,6 @@ #include <linux/atomic.h> #include <linux/reservation.h> -#define TTM_ASSERT_LOCKED(param) -#define TTM_DEBUG(fmt, arg...) -#define TTM_BO_HASH_ORDER 13 - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); static struct attribute ttm_bo_count = { @@ -165,7 +160,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -175,7 +170,8 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) list_add_tail(&bo->lru, &man->lru[bo->priority]); kref_get(&bo->list_kref); - if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { + if (bo->ttm && !(bo->ttm->page_flags & + (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { list_add_tail(&bo->swap, &bo->glob->swap_lru[bo->priority]); kref_get(&bo->list_kref); @@ -216,7 +212,7 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); ttm_bo_del_from_lru(bo); ttm_bo_add_to_lru(bo); @@ -233,7 +229,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) int ret = 0; uint32_t page_flags = 0; - TTM_ASSERT_LOCKED(&bo->mutex); + reservation_object_assert_held(bo->resv); bo->ttm = NULL; if (bdev->need_dma32) @@ -269,9 +265,8 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) } static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem, - bool evict, bool interruptible, - bool no_wait_gpu) + struct ttm_mem_reg *mem, bool evict, + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); @@ -306,7 +301,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, goto out_err; if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(bo->ttm, mem); + ret = ttm_tt_bind(bo->ttm, mem, ctx); if (ret) goto out_err; } @@ -325,12 +320,11 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem); + ret = ttm_bo_move_ttm(bo, ctx, mem); else if (bdev->driver->move) - ret = bdev->driver->move(bo, evict, interruptible, - no_wait_gpu, mem); + ret = bdev->driver->move(bo, evict, ctx, mem); else - ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem); + ret = ttm_bo_move_memcpy(bo, ctx, mem); if (ret) { if (bdev->driver->move_notify) { @@ -355,13 +349,13 @@ moved: bo->evicted = false; } - if (bo->mem.mm_node) { + if (bo->mem.mm_node) bo->offset = (bo->mem.start << PAGE_SHIFT) + bdev->man[bo->mem.mem_type].gpu_offset; - bo->cur_placement = bo->mem.placement; - } else + else bo->offset = 0; + ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; return 0; out_err: @@ -390,8 +384,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) ttm_tt_destroy(bo->ttm); bo->ttm = NULL; ttm_bo_mem_put(bo, &bo->mem); - - ww_mutex_unlock (&bo->resv->lock); } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) @@ -448,7 +440,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) } spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); + ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; if (!ret) { if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { ttm_bo_del_from_lru(bo); @@ -457,6 +449,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) reservation_object_unlock(&bo->ttm_resv); ttm_bo_cleanup_memtype_use(bo); + reservation_object_unlock(bo->resv); return; } @@ -472,7 +465,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_add_to_lru(bo); } - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } if (bo->resv != &bo->ttm_resv) reservation_object_unlock(&bo->ttm_resv); @@ -487,20 +480,21 @@ error: } /** - * function ttm_bo_cleanup_refs_and_unlock + * function ttm_bo_cleanup_refs * If bo idle, remove from delayed- and lru lists, and unref. * If not idle, do nothing. * * Must be called with lru_lock and reservation held, this function - * will drop both before returning. + * will drop the lru lock and optionally the reservation lock before returning. * * @interruptible Any sleeps should occur interruptibly. * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + * @unlock_resv Unlock the reservation lock as well. */ -static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_gpu) +static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait_gpu, + bool unlock_resv) { struct ttm_bo_global *glob = bo->glob; struct reservation_object *resv; @@ -518,7 +512,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, if (ret && !no_wait_gpu) { long lret; - ww_mutex_unlock(&bo->resv->lock); + + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); lret = reservation_object_wait_timeout_rcu(resv, true, @@ -531,24 +527,24 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, return -EBUSY; spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); - - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - if (ret) { + if (unlock_resv && !reservation_object_trylock(bo->resv)) { + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ spin_unlock(&glob->lru_lock); return 0; } + ret = 0; } if (ret || unlikely(list_empty(&bo->ddestroy))) { - __ttm_bo_unreserve(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); return ret; } @@ -560,6 +556,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); + return 0; } @@ -567,60 +566,44 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, * Traverse the delayed list, and call ttm_bo_cleanup_refs on all * encountered buffers. */ - -static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) { struct ttm_bo_global *glob = bdev->glob; - struct ttm_buffer_object *entry = NULL; - int ret = 0; + struct list_head removed; + bool empty; - spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - goto out_unlock; + INIT_LIST_HEAD(&removed); - entry = list_first_entry(&bdev->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&entry->list_kref); - - for (;;) { - struct ttm_buffer_object *nentry = NULL; + spin_lock(&glob->lru_lock); + while (!list_empty(&bdev->ddestroy)) { + struct ttm_buffer_object *bo; - if (entry->ddestroy.next != &bdev->ddestroy) { - nentry = list_first_entry(&entry->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&nentry->list_kref); - } + bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, + ddestroy); + kref_get(&bo->list_kref); + list_move_tail(&bo->ddestroy, &removed); - ret = __ttm_bo_reserve(entry, false, true, NULL); - if (remove_all && ret) { + if (remove_all || bo->resv != &bo->ttm_resv) { spin_unlock(&glob->lru_lock); - ret = __ttm_bo_reserve(entry, false, false, NULL); + reservation_object_lock(bo->resv, NULL); + spin_lock(&glob->lru_lock); - } + ttm_bo_cleanup_refs(bo, false, !remove_all, true); - if (!ret) - ret = ttm_bo_cleanup_refs_and_unlock(entry, false, - !remove_all); - else + } else if (reservation_object_trylock(bo->resv)) { + ttm_bo_cleanup_refs(bo, false, !remove_all, true); + } else { spin_unlock(&glob->lru_lock); + } - kref_put(&entry->list_kref, ttm_bo_release_list); - entry = nentry; - - if (ret || !entry) - goto out; - + kref_put(&bo->list_kref, ttm_bo_release_list); spin_lock(&glob->lru_lock); - if (list_empty(&entry->ddestroy)) - break; } - -out_unlock: + list_splice_tail(&removed, &bdev->ddestroy); + empty = list_empty(&bdev->ddestroy); spin_unlock(&glob->lru_lock); -out: - if (entry) - kref_put(&entry->list_kref, ttm_bo_release_list); - return ret; + + return empty; } static void ttm_bo_delayed_workqueue(struct work_struct *work) @@ -628,7 +611,7 @@ static void ttm_bo_delayed_workqueue(struct work_struct *work) struct ttm_bo_device *bdev = container_of(work, struct ttm_bo_device, wq.work); - if (ttm_bo_delayed_delete(bdev, false)) { + if (!ttm_bo_delayed_delete(bdev, false)) { schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); } @@ -672,15 +655,15 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) } EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); -static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, - bool no_wait_gpu) +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_reg evict_mem; struct ttm_placement placement; int ret = 0; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -690,8 +673,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, placement.num_placement = 0; placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); - ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, - no_wait_gpu); + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { pr_err("Failed to find memory space for buffer 0x%p eviction\n", @@ -701,8 +683,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, - no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); if (unlikely(ret)) { if (ret != -ERESTARTSYS) pr_err("Buffer eviction failed\n"); @@ -728,49 +709,78 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_eviction_valuable); +/** + * Check the target bo is allowable to be evicted or swapout, including cases: + * + * a. if share same reservation object with ctx->resv, have assumption + * reservation objects should already be locked, so not lock again and + * return true directly when either the opreation allow_reserved_eviction + * or the target bo already is in delayed free list; + * + * b. Otherwise, trylock it. + */ +static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, bool *locked) +{ + bool ret = false; + + *locked = false; + if (bo->resv == ctx->resv) { + reservation_object_assert_held(bo->resv); + if (ctx->allow_reserved_eviction || !list_empty(&bo->ddestroy)) + ret = true; + } else { + *locked = reservation_object_trylock(bo->resv); + ret = *locked; + } + + return ret; +} + static int ttm_mem_evict_first(struct ttm_bo_device *bdev, - uint32_t mem_type, - const struct ttm_place *place, - bool interruptible, - bool no_wait_gpu) + uint32_t mem_type, + const struct ttm_place *place, + struct ttm_operation_ctx *ctx) { struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - struct ttm_buffer_object *bo; - int ret = -EBUSY; + struct ttm_buffer_object *bo = NULL; + bool locked = false; unsigned i; + int ret; spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &man->lru[i], lru) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (ret) + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) continue; if (place && !bdev->driver->eviction_valuable(bo, place)) { - __ttm_bo_unreserve(bo); - ret = -EBUSY; + if (locked) + reservation_object_unlock(bo->resv); continue; } - break; } - if (!ret) + /* If the inner loop terminated early, we have our candidate */ + if (&bo->lru != &man->lru[i]) break; + + bo = NULL; } - if (ret) { + if (!bo) { spin_unlock(&glob->lru_lock); - return ret; + return -EBUSY; } kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, - no_wait_gpu); + ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, + ctx->no_wait_gpu, locked); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -778,10 +788,14 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); - BUG_ON(ret != 0); - - ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); - ttm_bo_unreserve(bo); + ret = ttm_bo_evict(bo, ctx); + if (locked) { + ttm_bo_unreserve(bo); + } else { + spin_lock(&glob->lru_lock); + ttm_bo_add_to_lru(bo); + spin_unlock(&glob->lru_lock); + } kref_put(&bo->list_kref, ttm_bo_release_list); return ret; @@ -832,8 +846,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, uint32_t mem_type, const struct ttm_place *place, struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; @@ -845,8 +858,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, mem_type, place, - interruptible, no_wait_gpu); + ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); if (unlikely(ret != 0)) return ret; } while (1); @@ -909,8 +921,7 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; @@ -1004,8 +1015,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, return 0; } - ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, - interruptible, no_wait_gpu); + ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; @@ -1024,14 +1034,13 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, EXPORT_SYMBOL(ttm_bo_mem_space); static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret = 0; struct ttm_mem_reg mem; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; @@ -1041,12 +1050,10 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, /* * Determine where to move the buffer. */ - ret = ttm_bo_mem_space(bo, placement, &mem, - interruptible, no_wait_gpu); + ret = ttm_bo_mem_space(bo, placement, &mem, ctx); if (ret) goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, - interruptible, no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); out_unlock: if (ret && mem.mm_node) ttm_bo_mem_put(bo, &mem); @@ -1097,20 +1104,18 @@ bool ttm_bo_mem_compat(struct ttm_placement *placement, EXPORT_SYMBOL(ttm_bo_mem_compat); int ttm_bo_validate(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret; uint32_t new_flags; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); /* * Check whether we need to move buffer. */ if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { - ret = ttm_bo_move_buffer(bo, placement, interruptible, - no_wait_gpu); + ret = ttm_bo_move_buffer(bo, placement, ctx); if (ret) return ret; } else { @@ -1139,7 +1144,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - bool interruptible, + struct ttm_operation_ctx *ctx, struct file *persistent_swap_storage, size_t acc_size, struct sg_table *sg, @@ -1151,7 +1156,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; bool locked; - ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); + ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); if (ret) { pr_err("Out of kernel memory\n"); if (destroy) @@ -1199,7 +1204,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->sg = sg; if (resv) { bo->resv = resv; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); } else { bo->resv = &bo->ttm_resv; } @@ -1221,12 +1226,12 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, * since otherwise lockdep will be angered in radeon. */ if (!resv) { - locked = ww_mutex_trylock(&bo->resv->lock); + locked = reservation_object_trylock(bo->resv); WARN_ON(!locked); } if (likely(!ret)) - ret = ttm_bo_validate(bo, placement, interruptible, false); + ret = ttm_bo_validate(bo, placement, ctx); if (unlikely(ret)) { if (!resv) @@ -1259,10 +1264,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev, struct reservation_object *resv, void (*destroy) (struct ttm_buffer_object *)) { + struct ttm_operation_ctx ctx = { interruptible, false }; int ret; ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, interruptible, + page_alignment, &ctx, persistent_swap_storage, acc_size, sg, resv, destroy); if (ret) @@ -1334,6 +1340,7 @@ EXPORT_SYMBOL(ttm_bo_create); static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, unsigned mem_type) { + struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_bo_global *glob = bdev->glob; struct dma_fence *fence; @@ -1348,7 +1355,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { while (!list_empty(&man->lru[i])) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); if (ret) return ret; spin_lock(&glob->lru_lock); @@ -1469,7 +1476,6 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj) struct ttm_bo_global *glob = container_of(kobj, struct ttm_bo_global, kobj); - ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); __free_page(glob->dummy_read_page); kfree(glob); } @@ -1494,6 +1500,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) mutex_init(&glob->device_list_mutex); spin_lock_init(&glob->lru_lock); glob->mem_glob = bo_ref->mem_glob; + glob->mem_glob->bo_glob = glob; glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); if (unlikely(glob->dummy_read_page == NULL)) { @@ -1504,14 +1511,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) INIT_LIST_HEAD(&glob->swap_lru[i]); INIT_LIST_HEAD(&glob->device_list); - - ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); - ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); - if (unlikely(ret != 0)) { - pr_err("Could not register buffer object swapout\n"); - goto out_no_shrink; - } - atomic_set(&glob->bo_count, 0); ret = kobject_init_and_add( @@ -1519,8 +1518,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref) if (unlikely(ret != 0)) kobject_put(&glob->kobj); return ret; -out_no_shrink: - __free_page(glob->dummy_read_page); out_no_drp: kfree(glob); return ret; @@ -1554,16 +1551,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) cancel_delayed_work_sync(&bdev->wq); - while (ttm_bo_delayed_delete(bdev, true)) - ; + if (ttm_bo_delayed_delete(bdev, true)) + pr_debug("Delayed destroy list was clean\n"); spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - TTM_DEBUG("Delayed destroy list was clean\n"); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) if (list_empty(&bdev->man[0].lru[0])) - TTM_DEBUG("Swap list %d was clean\n", i); + pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); drm_vma_offset_manager_destroy(&bdev->vma_manager); @@ -1706,21 +1700,20 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release); * A buffer object shrink method that tries to swap out the first * buffer object on the bo_global::swap_lru list. */ - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) +int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) { - struct ttm_bo_global *glob = - container_of(shrink, struct ttm_bo_global, shrink); struct ttm_buffer_object *bo; int ret = -EBUSY; + bool locked; unsigned i; spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &glob->swap_lru[i], swap) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (!ret) + if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { + ret = 0; break; + } } if (!ret) break; @@ -1734,7 +1727,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); + ret = ttm_bo_cleanup_refs(bo, false, false, true); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1748,6 +1741,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) if (bo->mem.mem_type != TTM_PL_SYSTEM || bo->ttm->caching_state != tt_cached) { + struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_reg evict_mem; evict_mem = bo->mem; @@ -1755,8 +1749,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; evict_mem.mem_type = TTM_PL_SYSTEM; - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, - false, false); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); if (unlikely(ret != 0)) goto out; } @@ -1787,15 +1780,21 @@ out: * Unreserve without putting on LRU to avoid swapping out an * already swapped buffer. */ - - __ttm_bo_unreserve(bo); + if (locked) + reservation_object_unlock(bo->resv); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } +EXPORT_SYMBOL(ttm_bo_swapout); void ttm_bo_swapout_all(struct ttm_bo_device *bdev) { - while (ttm_bo_swapout(&bdev->glob->shrink) == 0) + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + + while (ttm_bo_swapout(bdev->glob, &ctx) == 0) ; } EXPORT_SYMBOL(ttm_bo_swapout_all); @@ -1822,10 +1821,12 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) return -ERESTARTSYS; if (!ww_mutex_is_locked(&bo->resv->lock)) goto out_unlock; - ret = __ttm_bo_reserve(bo, true, false, NULL); + ret = reservation_object_lock_interruptible(bo->resv, NULL); + if (ret == -EINTR) + ret = -ERESTARTSYS; if (unlikely(ret != 0)) goto out_unlock; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); out_unlock: mutex_unlock(&bo->wu_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index e7a519f1849b..153de1bf0232 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -45,7 +45,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) } int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; @@ -53,7 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) @@ -73,7 +73,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, return ret; if (new_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(ttm, new_mem); + ret = ttm_tt_bind(ttm, new_mem, ctx); if (unlikely(ret != 0)) return ret; } @@ -329,7 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; @@ -345,7 +345,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, unsigned long add = 0; int dir; - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (ret) return ret; @@ -376,7 +376,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, * TTM might be null for moves within the same region. */ if (ttm && ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); if (ret) goto out1; } @@ -485,7 +485,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->acc_size = 0; fbo->resv = &fbo->ttm_resv; reservation_object_init(fbo->resv); - ret = ww_mutex_trylock(&fbo->resv->lock); + ret = reservation_object_trylock(fbo->resv); WARN_ON(!ret); *new_obj = fbo; @@ -545,14 +545,19 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, unsigned long num_pages, struct ttm_bo_kmap_obj *map) { - struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; + struct ttm_mem_reg *mem = &bo->mem; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; struct ttm_tt *ttm = bo->ttm; + pgprot_t prot; int ret; BUG_ON(!ttm); if (ttm->state == tt_unpopulated) { - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, &ctx); if (ret) return ret; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index c8ebb757e36b..08a3c324242e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -92,6 +92,18 @@ out_unlock: return ret; } +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_bo_device *bdev = bo->bdev; + + if (bdev->driver->io_mem_pfn) + return bdev->driver->io_mem_pfn(bo, page_offset); + + return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + + page_offset; +} + static int ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -215,12 +227,17 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); } else { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + ttm = bo->ttm; cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); /* Allocate all page at once, most common usage */ - if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + if (ttm->bdev->driver->ttm_tt_populate(ttm, &ctx)) { retval = VM_FAULT_OOM; goto out_io_unlock; } @@ -234,7 +251,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf) if (bo->mem.bus.is_iomem) { /* Iomem should not be marked encrypted */ cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); - pfn = bdev->driver->io_mem_pfn(bo, page_offset); + pfn = ttm_bo_io_mem_pfn(bo, page_offset); } else { page = ttm->pages[page_offset]; if (unlikely(!page && i == 0)) { @@ -404,14 +421,6 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, return bo; } -unsigned long ttm_bo_default_io_mem_pfn(struct ttm_buffer_object *bo, - unsigned long page_offset) -{ - return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) - + page_offset; -} -EXPORT_SYMBOL(ttm_bo_default_io_mem_pfn); - int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, struct ttm_bo_device *bdev) { diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 5e1bcabffef5..373ced0b2fc2 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -38,7 +38,7 @@ static void ttm_eu_backoff_reservation_reverse(struct list_head *list, list_for_each_entry_continue_reverse(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } } @@ -69,7 +69,7 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, struct ttm_buffer_object *bo = entry->bo; ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); @@ -112,7 +112,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) { - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); ret = -EBUSY; @@ -203,7 +203,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, else reservation_object_add_excl_fence(bo->resv, fence); ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); if (ticket) diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index e96374990398..aa0c38136958 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -211,35 +211,33 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, */ static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra) + uint64_t extra, struct ttm_operation_ctx *ctx) { int ret; - struct ttm_mem_shrink *shrink; spin_lock(&glob->lock); - if (glob->shrink == NULL) - goto out; while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - shrink = glob->shrink; spin_unlock(&glob->lock); - ret = shrink->do_shrink(shrink); + ret = ttm_bo_swapout(glob->bo_glob, ctx); spin_lock(&glob->lock); if (unlikely(ret != 0)) - goto out; + break; } -out: + spin_unlock(&glob->lock); } - - static void ttm_shrink_work(struct work_struct *work) { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; struct ttm_mem_global *glob = container_of(work, struct ttm_mem_global, work); - ttm_shrink(glob, true, 0ULL); + ttm_shrink(glob, true, 0ULL, &ctx); } static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, @@ -514,7 +512,7 @@ out_unlock: static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, struct ttm_mem_zone *single_zone, uint64_t memory, - bool no_wait, bool interruptible) + struct ttm_operation_ctx *ctx) { int count = TTM_MEMORY_ALLOC_RETRIES; @@ -522,33 +520,32 @@ static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, single_zone, memory, true) != 0)) { - if (no_wait) + if (ctx->no_wait_gpu) return -ENOMEM; if (unlikely(count-- == 0)) return -ENOMEM; - ttm_shrink(glob, false, memory + (memory >> 2) + 16); + ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); } return 0; } int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, - bool no_wait, bool interruptible) + struct ttm_operation_ctx *ctx) { /** * Normal allocations of kernel memory are registered in * all zones. */ - return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, - interruptible); + return ttm_mem_global_alloc_zone(glob, NULL, memory, ctx); } EXPORT_SYMBOL(ttm_mem_global_alloc); int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, - struct page *page, uint64_t size) + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx) { - struct ttm_mem_zone *zone = NULL; /** @@ -563,7 +560,7 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) zone = glob->zone_kernel; #endif - return ttm_mem_global_alloc_zone(glob, zone, size, false, false); + return ttm_mem_global_alloc_zone(glob, zone, size, ctx); } void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index 26a7ad0f4789..1aa2baa83959 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -325,6 +325,10 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, struct ttm_ref_object *ref; struct drm_hash_item *hash; struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; int ret = -EINVAL; if (base->tfile != tfile && !base->shareable) @@ -350,7 +354,7 @@ int ttm_ref_object_add(struct ttm_object_file *tfile, return -EPERM; ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref), - false, false); + &ctx); if (unlikely(ret != 0)) return ret; ref = kmalloc(sizeof(*ref), GFP_KERNEL); @@ -686,7 +690,10 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, dma_buf = prime->dma_buf; if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) { DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false + }; exp_info.ops = &tdev->ops; exp_info.size = prime->size; exp_info.flags = flags; @@ -696,7 +703,7 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile, * Need to create a new dma_buf, with memory accounting. */ ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size, - false, true); + &ctx); if (unlikely(ret != 0)) { mutex_unlock(&prime->mutex); goto out_unref; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 5d252fb27a82..2b12c55a3bff 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -477,12 +477,12 @@ ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) return count; } -static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) +static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { manager->mm_shrink.count_objects = ttm_pool_shrink_count; manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; - register_shrinker(&manager->mm_shrink); + return register_shrinker(&manager->mm_shrink); } static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) @@ -1034,15 +1034,18 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, &glob->kobj, "pool"); - if (unlikely(ret != 0)) { - kobject_put(&_manager->kobj); - _manager = NULL; - return ret; - } - - ttm_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; + ret = ttm_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; return 0; + +error: + kobject_put(&_manager->kobj); + _manager = NULL; + return ret; } void ttm_page_alloc_fini(void) @@ -1060,7 +1063,7 @@ void ttm_page_alloc_fini(void) _manager = NULL; } -int ttm_pool_populate(struct ttm_tt *ttm) +int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; unsigned i; @@ -1072,15 +1075,17 @@ int ttm_pool_populate(struct ttm_tt *ttm) ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, ttm->caching_state); if (unlikely(ret != 0)) { - ttm_pool_unpopulate(ttm); + ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, + ttm->caching_state); return ret; } for (i = 0; i < ttm->num_pages; ++i) { ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - PAGE_SIZE); + PAGE_SIZE, ctx); if (unlikely(ret != 0)) { - ttm_pool_unpopulate(ttm); + ttm_put_pages(ttm->pages, ttm->num_pages, + ttm->page_flags, ttm->caching_state); return -ENOMEM; } } @@ -1115,12 +1120,13 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_pool_unpopulate); -int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt) +int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + struct ttm_operation_ctx *ctx) { unsigned i, j; int r; - r = ttm_pool_populate(&tt->ttm); + r = ttm_pool_populate(&tt->ttm, ctx); if (r) return r; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c index 6b2627fe9bc1..a88051552ace 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -61,6 +61,7 @@ #define SMALL_ALLOCATION 4 #define FREE_ALL_PAGES (~0U) #define VADDR_FLAG_HUGE_POOL 1UL +#define VADDR_FLAG_UPDATED_COUNT 2UL enum pool_type { IS_UNDEFINED = 0, @@ -333,14 +334,18 @@ static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) { struct dma_page *d_page; + unsigned long attrs = 0; void *vaddr; d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); if (!d_page) return NULL; - vaddr = dma_alloc_coherent(pool->dev, pool->size, &d_page->dma, - pool->gfp_flags); + if (pool->type & IS_HUGE) + attrs = DMA_ATTR_NO_WARN; + + vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma, + pool->gfp_flags, attrs); if (vaddr) { if (is_vmalloc_addr(vaddr)) d_page->p = vmalloc_to_page(vaddr); @@ -870,18 +875,18 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, } /* - * @return count of pages still required to fulfill the request. * The populate list is actually a stack (not that is matters as TTM * allocates one page at a time. + * return dma_page pointer if success, otherwise NULL. */ -static int ttm_dma_pool_get_pages(struct dma_pool *pool, +static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool, struct ttm_dma_tt *ttm_dma, unsigned index) { - struct dma_page *d_page; + struct dma_page *d_page = NULL; struct ttm_tt *ttm = &ttm_dma->ttm; unsigned long irq_flags; - int count, r = -ENOMEM; + int count; spin_lock_irqsave(&pool->lock, irq_flags); count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); @@ -890,12 +895,11 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool, ttm->pages[index] = d_page->p; ttm_dma->dma_address[index] = d_page->dma; list_move_tail(&d_page->page_list, &ttm_dma->pages_list); - r = 0; pool->npages_in_use += 1; pool->npages_free -= 1; } spin_unlock_irqrestore(&pool->lock, irq_flags); - return r; + return d_page; } static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) @@ -923,12 +927,14 @@ static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) * On success pages list will hold count number of correctly * cached pages. On failure will hold the negative return value (-ENOMEM, etc). */ -int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + struct ttm_operation_ctx *ctx) { struct ttm_tt *ttm = &ttm_dma->ttm; struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; unsigned long num_pages = ttm->num_pages; struct dma_pool *pool; + struct dma_page *d_page; enum pool_type type; unsigned i; int ret; @@ -957,17 +963,18 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev) while (num_pages >= HPAGE_PMD_NR) { unsigned j; - ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); - if (ret != 0) + d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (!d_page) break; ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size); + pool->size, ctx); if (unlikely(ret != 0)) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; } + d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) { ttm->pages[j] = ttm->pages[j - 1] + 1; ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] + @@ -991,19 +998,20 @@ skip_huge: } while (num_pages) { - ret = ttm_dma_pool_get_pages(pool, ttm_dma, i); - if (ret != 0) { + d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (!d_page) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; } ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size); + pool->size, ctx); if (unlikely(ret != 0)) { ttm_dma_unpopulate(ttm_dma, dev); return -ENOMEM; } + d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; ++i; --num_pages; } @@ -1044,8 +1052,11 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) continue; count++; - ttm_mem_global_free_page(ttm->glob->mem_glob, - d_page->p, pool->size); + if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + d_page->p, pool->size); + d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; + } ttm_dma_page_put(pool, d_page); } @@ -1065,9 +1076,19 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) /* make sure pages array match list and count number of pages */ count = 0; - list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) { + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, + page_list) { ttm->pages[count] = d_page->p; count++; + + if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + d_page->p, pool->size); + d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; + } + + if (is_cached) + ttm_dma_page_put(pool, d_page); } spin_lock_irqsave(&pool->lock, irq_flags); @@ -1087,19 +1108,6 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) } spin_unlock_irqrestore(&pool->lock, irq_flags); - if (is_cached) { - list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) { - ttm_mem_global_free_page(ttm->glob->mem_glob, - d_page->p, pool->size); - ttm_dma_page_put(pool, d_page); - } - } else { - for (i = 0; i < count; i++) { - ttm_mem_global_free_page(ttm->glob->mem_glob, - ttm->pages[i], pool->size); - } - } - INIT_LIST_HEAD(&ttm_dma->pages_list); for (i = 0; i < ttm->num_pages; i++) { ttm->pages[i] = NULL; @@ -1177,12 +1185,12 @@ ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) return count; } -static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) +static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) { manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; manager->mm_shrink.seeks = 1; - register_shrinker(&manager->mm_shrink); + return register_shrinker(&manager->mm_shrink); } static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) @@ -1192,7 +1200,7 @@ static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) { - int ret = -ENOMEM; + int ret; WARN_ON(_manager); @@ -1200,7 +1208,7 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); if (!_manager) - goto err; + return -ENOMEM; mutex_init(&_manager->lock); INIT_LIST_HEAD(&_manager->pools); @@ -1212,13 +1220,17 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) /* This takes care of auto-freeing the _manager */ ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, &glob->kobj, "dma_pool"); - if (unlikely(ret != 0)) { - kobject_put(&_manager->kobj); - goto err; - } - ttm_dma_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; + + ret = ttm_dma_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; return 0; -err: + +error: + kobject_put(&_manager->kobj); + _manager = NULL; return ret; } @@ -1244,15 +1256,12 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) { struct device_pools *p; struct dma_pool *pool = NULL; - char *h[] = {"pool", "refills", "pages freed", "inuse", "available", - "name", "virt", "busaddr"}; if (!_manager) { seq_printf(m, "No pool allocator running.\n"); return 0; } - seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", - h[0], h[1], h[2], h[3], h[4], h[5]); + seq_printf(m, " pool refills pages freed inuse available name\n"); mutex_lock(&_manager->lock); list_for_each_entry(p, &_manager->pools, pools) { struct device *dev = p->dev; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 8ebc8d3560c3..5a046a3c543a 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -261,7 +261,8 @@ void ttm_tt_unbind(struct ttm_tt *ttm) } } -int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) +int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem, + struct ttm_operation_ctx *ctx) { int ret = 0; @@ -271,7 +272,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) if (ttm->state == tt_bound) return 0; - ret = ttm->bdev->driver->ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx); if (ret) return ret; |