diff options
Diffstat (limited to 'drivers/gpu/arm/midgard/mali_kbase_mmu.c')
| -rw-r--r-- | drivers/gpu/arm/midgard/mali_kbase_mmu.c | 503 |
1 files changed, 110 insertions, 393 deletions
diff --git a/drivers/gpu/arm/midgard/mali_kbase_mmu.c b/drivers/gpu/arm/midgard/mali_kbase_mmu.c index 48d53723a9b4..d81ef593e928 100644 --- a/drivers/gpu/arm/midgard/mali_kbase_mmu.c +++ b/drivers/gpu/arm/midgard/mali_kbase_mmu.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,8 +30,9 @@ #if defined(CONFIG_MALI_GATOR_SUPPORT) #include <mali_kbase_gator.h> #endif +#if defined(CONFIG_MALI_MIPE_ENABLED) #include <mali_kbase_tlstream.h> -#include <mali_kbase_instr_defs.h> +#endif #include <mali_kbase_debug.h> #define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) @@ -40,32 +41,10 @@ #include <mali_kbase_hw.h> #include <mali_kbase_mmu_hw.h> #include <mali_kbase_hwaccess_jm.h> -#include <mali_kbase_time.h> #define KBASE_MMU_PAGE_ENTRIES 512 /** - * kbase_mmu_flush_invalidate() - Flush and invalidate the GPU caches. - * @kctx: The KBase context. - * @vpfn: The virtual page frame number to start the flush on. - * @nr: The number of pages to flush. - * @sync: Set if the operation should be synchronous or not. - * - * Issue a cache flush + invalidate to the GPU caches and invalidate the TLBs. - * - * If sync is not set then transactions still in flight when the flush is issued - * may use the old page tables and the data they write will not be written out - * to memory, this function returns after the flush has been issued but - * before all accesses which might effect the flushed region have completed. - * - * If sync is set then accesses in the flushed region will be drained - * before data is flush and invalidated through L1, L2 and into memory, - * after which point this function will return. - */ -static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, - u64 vpfn, size_t nr, bool sync); - -/** * kbase_mmu_sync_pgd - sync page directory to memory * @kbdev: Device pointer. * @handle: Address of DMA region. @@ -77,12 +56,8 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, dma_addr_t handle, size_t size) { - /* If page table is not coherent then ensure the gpu can read - * the pages from memory - */ - if (kbdev->system_coherency != COHERENCY_ACE) - dma_sync_single_for_device(kbdev->dev, handle, size, - DMA_TO_DEVICE); + + dma_sync_single_for_device(kbdev->dev, handle, size, DMA_TO_DEVICE); } /* @@ -161,18 +136,6 @@ void page_fault_worker(struct work_struct *data) dev_warn(kbdev->dev, "Access flag unexpectedly set"); goto fault_done; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: - - kbase_mmu_report_fault_and_kill(kctx, faulting_as, - "Address size fault"); - goto fault_done; - - case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: - kbase_mmu_report_fault_and_kill(kctx, faulting_as, - "Memory attributes fault"); - goto fault_done; -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ default: kbase_mmu_report_fault_and_kill(kctx, faulting_as, @@ -201,13 +164,6 @@ void page_fault_worker(struct work_struct *data) goto fault_done; } - if ((region->flags & KBASE_REG_DONT_NEED)) { - kbase_gpu_vm_unlock(kctx); - kbase_mmu_report_fault_and_kill(kctx, faulting_as, - "Don't need memory can't be grown"); - goto fault_done; - } - /* find the size we need to grow it by */ /* we know the result fit in a size_t due to kbase_region_tracker_find_region_enclosing_address * validating the fault_adress to be within a size_t from the start_pfn */ @@ -277,27 +233,19 @@ void page_fault_worker(struct work_struct *data) if (grown) { - u64 pfn_offset; u32 op; /* alloc success */ KBASE_DEBUG_ASSERT(kbase_reg_current_backed_size(region) <= region->nr_pages); + /* AS transaction begin */ + mutex_lock(&faulting_as->transaction_mutex); + /* set up the new pages */ - pfn_offset = kbase_reg_current_backed_size(region) - new_pages; - /* - * Note: - * Issuing an MMU operation will unlock the MMU and cause the - * translation to be replayed. If the page insertion fails then - * rather then trying to continue the context should be killed - * so the no_flush version of insert_pages is used which allows - * us to unlock the MMU as we see fit. - */ - err = kbase_mmu_insert_pages_no_flush(kctx, - region->start_pfn + pfn_offset, - &kbase_get_gpu_phy_pages(region)[pfn_offset], - new_pages, region->flags); + err = kbase_mmu_insert_pages(kctx, region->start_pfn + kbase_reg_current_backed_size(region) - new_pages, &kbase_get_gpu_phy_pages(region)[kbase_reg_current_backed_size(region) - new_pages], new_pages, region->flags); if (err) { + /* failed to insert pages, handle as a normal PF */ + mutex_unlock(&faulting_as->transaction_mutex); kbase_free_phy_pages_helper(region->gpu_alloc, new_pages); if (region->gpu_alloc != region->cpu_alloc) kbase_free_phy_pages_helper(region->cpu_alloc, @@ -311,10 +259,9 @@ void page_fault_worker(struct work_struct *data) #if defined(CONFIG_MALI_GATOR_SUPPORT) kbase_trace_mali_page_fault_insert_pages(as_no, new_pages); #endif +#if defined(CONFIG_MALI_MIPE_ENABLED) kbase_tlstream_aux_pagefault(kctx->id, (u64)new_pages); - - /* AS transaction begin */ - mutex_lock(&faulting_as->transaction_mutex); +#endif /* flush L2 and unlock the VA (resumes the MMU) */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6367)) @@ -377,9 +324,11 @@ phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx) if (!p) goto sub_pages; +#if defined(CONFIG_MALI_MIPE_ENABLED) kbase_tlstream_aux_pagesalloc( (u32)kctx->id, (u64)new_page_count); +#endif page = kmap(p); if (NULL == page) @@ -416,7 +365,7 @@ static phys_addr_t mmu_get_next_pgd(struct kbase_context *kctx, phys_addr_t pgd, KBASE_DEBUG_ASSERT(pgd); KBASE_DEBUG_ASSERT(NULL != kctx); - lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); /* * Architecture spec defines level-0 as being the top-most. @@ -457,9 +406,8 @@ static phys_addr_t mmu_get_bottom_pgd(struct kbase_context *kctx, u64 vpfn) phys_addr_t pgd; int l; - lockdep_assert_held(&kctx->mmu_lock); - pgd = kctx->pgd; + for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { pgd = mmu_get_next_pgd(kctx, pgd, vpfn, l); /* Handle failure condition */ @@ -480,7 +428,7 @@ static phys_addr_t mmu_insert_pages_recover_get_next_pgd(struct kbase_context *k KBASE_DEBUG_ASSERT(pgd); KBASE_DEBUG_ASSERT(NULL != kctx); - lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); /* * Architecture spec defines level-0 as being the top-most. @@ -505,8 +453,6 @@ static phys_addr_t mmu_insert_pages_recover_get_bottom_pgd(struct kbase_context phys_addr_t pgd; int l; - lockdep_assert_held(&kctx->mmu_lock); - pgd = kctx->pgd; for (l = MIDGARD_MMU_TOPLEVEL; l < MIDGARD_MMU_BOTTOMLEVEL; l++) { @@ -530,7 +476,7 @@ static void mmu_insert_pages_failure_recovery(struct kbase_context *kctx, u64 vp /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); - lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); mmu_mode = kctx->kbdev->mmu_mode; @@ -578,28 +524,22 @@ int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, bool recover_required = false; u64 recover_vpfn = vpfn; size_t recover_count = 0; - size_t remain = nr; - int err; KBASE_DEBUG_ASSERT(NULL != kctx); KBASE_DEBUG_ASSERT(0 != vpfn); /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); - /* Early out if there is nothing to do */ - if (nr == 0) - return 0; - - mutex_lock(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); - while (remain) { + while (nr) { unsigned int i; unsigned int index = vpfn & 0x1FF; unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; struct page *p; - if (count > remain) - count = remain; + if (count > nr) + count = nr; /* * Repeatedly calling mmu_get_bottom_pte() is clearly @@ -618,8 +558,7 @@ int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, recover_vpfn, recover_count); } - err = -EINVAL; - goto fail_unlock; + return -EINVAL; } p = pfn_to_page(PFN_DOWN(pgd)); @@ -633,8 +572,7 @@ int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, recover_vpfn, recover_count); } - err = -ENOMEM; - goto fail_unlock; + return -ENOMEM; } for (i = 0; i < count; i++) { @@ -646,7 +584,7 @@ int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, } vpfn += count; - remain -= count; + nr -= count; kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p) + (index * sizeof(u64)), @@ -659,17 +597,13 @@ int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, recover_required = true; recover_count += count; } - mutex_unlock(&kctx->mmu_lock); - kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); return 0; - -fail_unlock: - mutex_unlock(&kctx->mmu_lock); - kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); - return err; } -int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, +/* + * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' + */ +int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags) { @@ -680,28 +614,22 @@ int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, bool recover_required = false; u64 recover_vpfn = vpfn; size_t recover_count = 0; - size_t remain = nr; - int err; KBASE_DEBUG_ASSERT(NULL != kctx); KBASE_DEBUG_ASSERT(0 != vpfn); /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); - /* Early out if there is nothing to do */ - if (nr == 0) - return 0; - - mutex_lock(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); - while (remain) { + while (nr) { unsigned int i; unsigned int index = vpfn & 0x1FF; unsigned int count = KBASE_MMU_PAGE_ENTRIES - index; struct page *p; - if (count > remain) - count = remain; + if (count > nr) + count = nr; /* * Repeatedly calling mmu_get_bottom_pte() is clearly @@ -720,8 +648,7 @@ int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, recover_vpfn, recover_count); } - err = -EINVAL; - goto fail_unlock; + return -EINVAL; } p = pfn_to_page(PFN_DOWN(pgd)); @@ -735,8 +662,7 @@ int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, recover_vpfn, recover_count); } - err = -ENOMEM; - goto fail_unlock; + return -ENOMEM; } for (i = 0; i < count; i++) { @@ -749,7 +675,7 @@ int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, phys += count; vpfn += count; - remain -= count; + nr -= count; kbase_mmu_sync_pgd(kctx->kbdev, kbase_dma_addr(p) + (index * sizeof(u64)), @@ -762,209 +688,81 @@ int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn, recover_required = true; recover_count += count; } - - mutex_unlock(&kctx->mmu_lock); return 0; - -fail_unlock: - mutex_unlock(&kctx->mmu_lock); - return err; -} - -/* - * Map 'nr' pages pointed to by 'phys' at GPU PFN 'vpfn' - */ -int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn, - phys_addr_t *phys, size_t nr, - unsigned long flags) -{ - int err; - - err = kbase_mmu_insert_pages_no_flush(kctx, vpfn, phys, nr, flags); - kbase_mmu_flush_invalidate(kctx, vpfn, nr, false); - return err; } KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages); /** - * kbase_mmu_flush_invalidate_noretain() - Flush and invalidate the GPU caches - * without retaining the kbase context. - * @kctx: The KBase context. - * @vpfn: The virtual page frame number to start the flush on. - * @nr: The number of pages to flush. - * @sync: Set if the operation should be synchronous or not. + * This function is responsible for validating the MMU PTs + * triggering reguired flushes. * - * As per kbase_mmu_flush_invalidate but doesn't retain the kctx or do any - * other locking. + * * IMPORTANT: This uses kbasep_js_runpool_release_ctx() when the context is + * currently scheduled into the runpool, and so potentially uses a lot of locks. + * These locks must be taken in the correct order with respect to others + * already held by the caller. Refer to kbasep_js_runpool_release_ctx() for more + * information. */ -static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, - u64 vpfn, size_t nr, bool sync) -{ - struct kbase_device *kbdev = kctx->kbdev; - int err; - u32 op; - - /* Early out if there is nothing to do */ - if (nr == 0) - return; - - if (sync) - op = AS_COMMAND_FLUSH_MEM; - else - op = AS_COMMAND_FLUSH_PT; - - err = kbase_mmu_hw_do_operation(kbdev, - &kbdev->as[kctx->as_nr], - kctx, vpfn, nr, op, 0); -#if KBASE_GPU_RESET_EN - if (err) { - /* Flush failed to complete, assume the - * GPU has hung and perform a reset to - * recover */ - dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); - - if (kbase_prepare_to_reset_gpu_locked(kbdev)) - kbase_reset_gpu_locked(kbdev); - } -#endif /* KBASE_GPU_RESET_EN */ - -#ifndef CONFIG_MALI_NO_MALI - /* - * As this function could be called in interrupt context the sync - * request can't block. Instead log the request and the next flush - * request will pick it up. - */ - if ((!err) && sync && - kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_6367)) - atomic_set(&kctx->drain_pending, 1); -#endif /* !CONFIG_MALI_NO_MALI */ -} - -static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, - u64 vpfn, size_t nr, bool sync) +static void kbase_mmu_flush(struct kbase_context *kctx, u64 vpfn, size_t nr) { struct kbase_device *kbdev; bool ctx_is_in_runpool; -#ifndef CONFIG_MALI_NO_MALI - bool drain_pending = false; - if (atomic_xchg(&kctx->drain_pending, 0)) - drain_pending = true; -#endif /* !CONFIG_MALI_NO_MALI */ - - /* Early out if there is nothing to do */ - if (nr == 0) - return; + KBASE_DEBUG_ASSERT(NULL != kctx); kbdev = kctx->kbdev; + + /* We must flush if we're currently running jobs. At the very least, we need to retain the + * context to ensure it doesn't schedule out whilst we're trying to flush it */ ctx_is_in_runpool = kbasep_js_runpool_retain_ctx(kbdev, kctx); if (ctx_is_in_runpool) { KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); - if (!kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - int err; - u32 op; - - /* AS transaction begin */ - mutex_lock(&kbdev->as[ - kctx->as_nr].transaction_mutex); - - if (sync) - op = AS_COMMAND_FLUSH_MEM; - else - op = AS_COMMAND_FLUSH_PT; - - err = kbase_mmu_hw_do_operation(kbdev, - &kbdev->as[kctx->as_nr], - kctx, vpfn, nr, op, 0); - + /* Second level check is to try to only do this when jobs are running. The refcount is + * a heuristic for this. */ + if (kbdev->js_data.runpool_irq.per_as_data[kctx->as_nr].as_busy_refcount >= 2) { + if (!kbase_pm_context_active_handle_suspend(kbdev, + KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { + int ret; + u32 op; + + /* AS transaction begin */ + mutex_lock(&kbdev->as[ + kctx->as_nr].transaction_mutex); + + if (kbase_hw_has_issue(kbdev, + BASE_HW_ISSUE_6367)) + op = AS_COMMAND_FLUSH; + else + op = AS_COMMAND_FLUSH_MEM; + + ret = kbase_mmu_hw_do_operation(kbdev, + &kbdev->as[kctx->as_nr], + kctx, vpfn, nr, + op, 0); #if KBASE_GPU_RESET_EN - if (err) { - /* Flush failed to complete, assume the - * GPU has hung and perform a reset to - * recover */ - dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); - - if (kbase_prepare_to_reset_gpu(kbdev)) - kbase_reset_gpu(kbdev); - } + if (ret) { + /* Flush failed to complete, assume the + * GPU has hung and perform a reset to + * recover */ + dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); + if (kbase_prepare_to_reset_gpu(kbdev)) + kbase_reset_gpu(kbdev); + } #endif /* KBASE_GPU_RESET_EN */ - mutex_unlock(&kbdev->as[ - kctx->as_nr].transaction_mutex); - /* AS transaction end */ + mutex_unlock(&kbdev->as[ + kctx->as_nr].transaction_mutex); + /* AS transaction end */ -#ifndef CONFIG_MALI_NO_MALI - /* - * The transaction lock must be dropped before here - * as kbase_wait_write_flush could take it if - * the GPU was powered down (static analysis doesn't - * know this can't happen). - */ - drain_pending |= (!err) && sync && - kbase_hw_has_issue(kctx->kbdev, - BASE_HW_ISSUE_6367); - if (drain_pending) { - /* Wait for GPU to flush write buffer */ - kbase_wait_write_flush(kctx); + kbase_pm_context_idle(kbdev); } -#endif /* !CONFIG_MALI_NO_MALI */ - - kbase_pm_context_idle(kbdev); } kbasep_js_runpool_release_ctx(kbdev, kctx); } } -void kbase_mmu_update(struct kbase_context *kctx) -{ - lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - /* ASSERT that the context has a valid as_nr, which is only the case - * when it's scheduled in. - * - * as_nr won't change because the caller has the runpool_irq lock */ - KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); - lockdep_assert_held(&kctx->kbdev->as[kctx->as_nr].transaction_mutex); - - kctx->kbdev->mmu_mode->update(kctx); -} -KBASE_EXPORT_TEST_API(kbase_mmu_update); - -void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr) -{ - lockdep_assert_held(&kbdev->as[as_nr].transaction_mutex); - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); - - kbdev->mmu_mode->disable_as(kbdev, as_nr); -} - -void kbase_mmu_disable(struct kbase_context *kctx) -{ - /* ASSERT that the context has a valid as_nr, which is only the case - * when it's scheduled in. - * - * as_nr won't change because the caller has the runpool_irq lock */ - KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); - - lockdep_assert_held(&kctx->kbdev->as[kctx->as_nr].transaction_mutex); - lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - - /* - * The address space is being disabled, drain all knowledge of it out - * from the caches as pages and page tables might be freed after this. - * - * The job scheduler code will already be holding the locks and context - * so just do the flush. - */ - kbase_mmu_flush_invalidate_noretain(kctx, 0, ~0, true); - - kctx->kbdev->mmu_mode->disable_as(kctx->kbdev, kctx->as_nr); -} -KBASE_EXPORT_TEST_API(kbase_mmu_disable); - /* * We actually only discard the ATE, and not the page table * pages. There is a potential DoS here, as we'll leak memory by @@ -984,18 +782,17 @@ int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) struct kbase_device *kbdev; size_t requested_nr = nr; struct kbase_mmu_mode const *mmu_mode; - int err; KBASE_DEBUG_ASSERT(NULL != kctx); beenthere(kctx, "kctx %p vpfn %lx nr %zd", (void *)kctx, (unsigned long)vpfn, nr); + lockdep_assert_held(&kctx->reg_lock); + if (0 == nr) { /* early out if nothing to do */ return 0; } - mutex_lock(&kctx->mmu_lock); - kbdev = kctx->kbdev; mmu_mode = kbdev->mmu_mode; @@ -1011,16 +808,14 @@ int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) pgd = mmu_get_bottom_pgd(kctx, vpfn); if (!pgd) { dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: mmu_get_bottom_pgd failure\n"); - err = -EINVAL; - goto fail_unlock; + return -EINVAL; } p = pfn_to_page(PFN_DOWN(pgd)); pgd_page = kmap(p); if (!pgd_page) { dev_warn(kbdev->dev, "kbase_mmu_teardown_pages: kmap failure\n"); - err = -ENOMEM; - goto fail_unlock; + return -ENOMEM; } for (i = 0; i < count; i++) @@ -1036,14 +831,8 @@ int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr) kunmap(p); } - mutex_unlock(&kctx->mmu_lock); - kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); + kbase_mmu_flush(kctx, vpfn, requested_nr); return 0; - -fail_unlock: - mutex_unlock(&kctx->mmu_lock); - kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); - return err; } KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages); @@ -1066,17 +855,12 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *ph u64 *pgd_page; size_t requested_nr = nr; struct kbase_mmu_mode const *mmu_mode; - int err; KBASE_DEBUG_ASSERT(NULL != kctx); KBASE_DEBUG_ASSERT(0 != vpfn); KBASE_DEBUG_ASSERT(vpfn <= (U64_MAX / PAGE_SIZE)); - /* Early out if there is nothing to do */ - if (nr == 0) - return 0; - - mutex_lock(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); mmu_mode = kctx->kbdev->mmu_mode; @@ -1095,16 +879,14 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *ph pgd = mmu_get_bottom_pgd(kctx, vpfn); if (!pgd) { dev_warn(kctx->kbdev->dev, "mmu_get_bottom_pgd failure\n"); - err = -EINVAL; - goto fail_unlock; + return -EINVAL; } p = pfn_to_page(PFN_DOWN(pgd)); pgd_page = kmap(p); if (!pgd_page) { dev_warn(kctx->kbdev->dev, "kmap failure\n"); - err = -ENOMEM; - goto fail_unlock; + return -ENOMEM; } for (i = 0; i < count; i++) @@ -1122,14 +904,9 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *ph kunmap(pfn_to_page(PFN_DOWN(pgd))); } - mutex_unlock(&kctx->mmu_lock); - kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); - return 0; + kbase_mmu_flush(kctx, vpfn, requested_nr); -fail_unlock: - mutex_unlock(&kctx->mmu_lock); - kbase_mmu_flush_invalidate(kctx, vpfn, requested_nr, true); - return err; + return 0; } /* This is a debug feature only */ @@ -1157,7 +934,7 @@ static void mmu_teardown_level(struct kbase_context *kctx, phys_addr_t pgd, int struct kbase_mmu_mode const *mmu_mode; KBASE_DEBUG_ASSERT(NULL != kctx); - lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); pgd_page = kmap_atomic(pfn_to_page(PFN_DOWN(pgd))); /* kmap_atomic should NEVER fail. */ @@ -1202,8 +979,6 @@ int kbase_mmu_init(struct kbase_context *kctx) KBASE_DEBUG_ASSERT(NULL != kctx); KBASE_DEBUG_ASSERT(NULL == kctx->mmu_teardown_pages); - mutex_init(&kctx->mmu_lock); - /* Preallocate MMU depth of four pages for mmu_teardown_level to use */ kctx->mmu_teardown_pages = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); @@ -1229,9 +1004,9 @@ void kbase_mmu_free_pgd(struct kbase_context *kctx) KBASE_DEBUG_ASSERT(NULL != kctx); KBASE_DEBUG_ASSERT(NULL != kctx->mmu_teardown_pages); - mutex_lock(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); + mmu_teardown_level(kctx, kctx->pgd, MIDGARD_MMU_TOPLEVEL, 1, kctx->mmu_teardown_pages); - mutex_unlock(&kctx->mmu_lock); beenthere(kctx, "pgd %lx", (unsigned long)kctx->pgd); kbase_mem_pool_free(&kctx->mem_pool, phys_to_page(kctx->pgd), true); @@ -1239,9 +1014,11 @@ void kbase_mmu_free_pgd(struct kbase_context *kctx) new_page_count = kbase_atomic_sub_pages(1, &kctx->used_pages); kbase_atomic_sub_pages(1, &kctx->kbdev->memdev.used_pages); +#if defined(CONFIG_MALI_MIPE_ENABLED) kbase_tlstream_aux_pagesalloc( (u32)kctx->id, (u64)new_page_count); +#endif } KBASE_EXPORT_TEST_API(kbase_mmu_free_pgd); @@ -1256,7 +1033,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, struct kbase_mmu_mode const *mmu_mode; KBASE_DEBUG_ASSERT(NULL != kctx); - lockdep_assert_held(&kctx->mmu_lock); + lockdep_assert_held(&kctx->reg_lock); mmu_mode = kctx->kbdev->mmu_mode; @@ -1311,13 +1088,13 @@ void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) KBASE_DEBUG_ASSERT(kctx); + lockdep_assert_held(&kctx->reg_lock); + if (0 == nr_pages) { /* can't dump in a 0 sized buffer, early out */ return NULL; } - mutex_lock(&kctx->mmu_lock); - size_left = nr_pages * PAGE_SIZE; KBASE_DEBUG_ASSERT(0 != size_left); @@ -1339,7 +1116,7 @@ void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) kctx->kbdev->mmu_mode->get_as_setup(kctx, &as_setup); config[0] = as_setup.transtab; config[1] = as_setup.memattr; - config[2] = as_setup.transcfg; + config[2] = 0; memcpy(buffer, &config, sizeof(config)); mmu_dump_buffer += sizeof(config); size_left -= sizeof(config); @@ -1353,8 +1130,10 @@ void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) &mmu_dump_buffer, &size_left); - if (!size) - goto fail_free; + if (!size) { + vfree(kaddr); + return NULL; + } /* Add on the size for the end marker */ size += sizeof(u64); @@ -1365,20 +1144,15 @@ void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages) if (size > nr_pages * PAGE_SIZE || size_left < sizeof(u64)) { /* The buffer isn't big enough - free the memory and return failure */ - goto fail_free; + vfree(kaddr); + return NULL; } /* Add the end marker */ memcpy(mmu_dump_buffer, &end_marker, sizeof(u64)); } - mutex_unlock(&kctx->mmu_lock); return kaddr; - -fail_free: - vfree(kaddr); - mutex_unlock(&kctx->mmu_lock); - return NULL; } KBASE_EXPORT_TEST_API(kbase_mmu_dump); @@ -1419,17 +1193,13 @@ void bus_fault_worker(struct work_struct *data) #endif /* KBASE_GPU_RESET_EN */ /* NOTE: If GPU already powered off for suspend, we don't need to switch to unmapped */ if (!kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - unsigned long flags; /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ /* AS transaction begin */ mutex_lock(&kbdev->as[as_no].transaction_mutex); /* Set the MMU into unmapped mode */ - spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags); - kbase_mmu_disable(kctx); - spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, - flags); + kbase_mmu_disable_as(kbdev, as_no); mutex_unlock(&kbdev->as[as_no].transaction_mutex); /* AS transaction end */ @@ -1547,15 +1317,6 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) e = "TRANSLATION_FAULT"; break; case 0xC8: -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - case 0xC9: - case 0xCA: - case 0xCB: - case 0xCC: - case 0xCD: - case 0xCE: - case 0xCF: -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ e = "PERMISSION_FAULT"; break; case 0xD0: @@ -1569,38 +1330,8 @@ const char *kbase_exception_name(struct kbase_device *kbdev, u32 exception_code) e = "TRANSTAB_BUS_FAULT"; break; case 0xD8: -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - case 0xD9: - case 0xDA: - case 0xDB: - case 0xDC: - case 0xDD: - case 0xDE: - case 0xDF: -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ e = "ACCESS_FLAG"; break; -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - e = "ADDRESS_SIZE_FAULT"; - break; - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - e = "MEMORY_ATTRIBUTES_FAULT"; -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ break; default: e = "UNKNOWN"; @@ -1614,12 +1345,7 @@ static const char *access_type_name(struct kbase_device *kbdev, u32 fault_status) { switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { - case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - return "ATOMIC"; -#else return "UNKNOWN"; -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ case AS_FAULTSTATUS_ACCESS_TYPE_READ: return "READ"; case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: @@ -1715,9 +1441,7 @@ static void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, } #endif /* KBASE_GPU_RESET_EN */ /* switch to UNMAPPED mode, will abort all jobs and stop any hw counter dumping */ - spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); - kbase_mmu_disable(kctx); - spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); + kbase_mmu_disable_as(kbdev, as_no); mutex_unlock(&as->transaction_mutex); /* AS transaction end */ @@ -1954,15 +1678,8 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, struct kbase_contex */ kbasep_js_clear_submit_allowed(js_devdata, kctx); -#ifdef CONFIG_MALI_GPU_MMU_AARCH64 - dev_warn(kbdev->dev, - "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", - as->number, as->fault_addr, - as->fault_extra_addr); -#else dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", as->number, as->fault_addr); -#endif /* CONFIG_MALI_GPU_MMU_AARCH64 */ /* * We need to switch to UNMAPPED mode - but we do this in a |