/* * * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms * of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. * */ #if defined(CONFIG_DMA_SHARED_BUFFER) #include #endif /* defined(CONFIG_DMA_SHARED_BUFFER) */ #ifdef CONFIG_COMPAT #include #endif #include #include #include #include #include #include #include #include #include "mali_kbase_dma_fence.h" #define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) /* random32 was renamed to prandom_u32 in 3.8 */ #define prandom_u32 random32 #endif /* Return whether katom will run on the GPU or not. Currently only soft jobs and * dependency-only atoms do not run on the GPU */ #define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ BASE_JD_REQ_DEP))) /* * This is the kernel side of the API. Only entry points are: * - kbase_jd_submit(): Called from userspace to submit a single bag * - kbase_jd_done(): Called from interrupt context to track the * completion of a job. * Callouts: * - to the job manager (enqueue a job) * - to the event subsystem (signals the completion/failure of bag/job-chains). */ static void __user * get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p) { #ifdef CONFIG_COMPAT if (kbase_ctx_flag(kctx, KCTX_COMPAT)) return compat_ptr(p->compat_value); #endif return p->value; } /* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs * * Returns whether the JS needs a reschedule. * * Note that the caller must also check the atom status and * if it is KBASE_JD_ATOM_STATE_COMPLETED must call jd_done_nolock */ static int jd_run_atom(struct kbase_jd_atom *katom) { struct kbase_context *kctx = katom->kctx; KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == BASE_JD_REQ_DEP) { /* Dependency only atom */ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; return 0; } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { /* Soft-job */ if (katom->will_fail_event_code) { katom->status = KBASE_JD_ATOM_STATE_COMPLETED; return 0; } if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_REPLAY) { if (!kbase_replay_process(katom)) katom->status = KBASE_JD_ATOM_STATE_COMPLETED; } else if (kbase_process_soft_job(katom) == 0) { kbase_finish_soft_job(katom); katom->status = KBASE_JD_ATOM_STATE_COMPLETED; } return 0; } katom->status = KBASE_JD_ATOM_STATE_IN_JS; /* Queue an action about whether we should try scheduling a context */ return kbasep_js_add_job(kctx, katom); } #if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom) { struct kbase_device *kbdev; KBASE_DEBUG_ASSERT(katom); kbdev = katom->kctx->kbdev; KBASE_DEBUG_ASSERT(kbdev); /* Check whether the atom's other dependencies were already met. If * katom is a GPU atom then the job scheduler may be able to represent * the dependencies, hence we may attempt to submit it before they are * met. Other atoms must have had both dependencies resolved. */ if (IS_GPU_ATOM(katom) || (!kbase_jd_katom_dep_atom(&katom->dep[0]) && !kbase_jd_katom_dep_atom(&katom->dep[1]))) { /* katom dep complete, attempt to run it */ bool resched = false; resched = jd_run_atom(katom); if (katom->status == KBASE_JD_ATOM_STATE_COMPLETED) { /* The atom has already finished */ resched |= jd_done_nolock(katom, NULL); } if (resched) kbase_js_sched_all(kbdev); } } #endif #ifdef CONFIG_KDS /* Add the katom to the kds waiting list. * Atoms must be added to the waiting list after a successful call to kds_async_waitall. * The caller must hold the kbase_jd_context.lock */ static void kbase_jd_kds_waiters_add(struct kbase_jd_atom *katom) { struct kbase_context *kctx; KBASE_DEBUG_ASSERT(katom); kctx = katom->kctx; list_add_tail(&katom->node, &kctx->waiting_kds_resource); } /* Remove the katom from the kds waiting list. * Atoms must be removed from the waiting list before a call to kds_resource_set_release_sync. * The supplied katom must first have been added to the list with a call to kbase_jd_kds_waiters_add. * The caller must hold the kbase_jd_context.lock */ static void kbase_jd_kds_waiters_remove(struct kbase_jd_atom *katom) { KBASE_DEBUG_ASSERT(katom); list_del(&katom->node); } static void kds_dep_clear(void *callback_parameter, void *callback_extra_parameter) { struct kbase_jd_atom *katom; struct kbase_jd_context *ctx; katom = (struct kbase_jd_atom *)callback_parameter; KBASE_DEBUG_ASSERT(katom); ctx = &katom->kctx->jctx; /* If KDS resource has already been satisfied (e.g. due to zapping) * do nothing. */ mutex_lock(&ctx->lock); if (!katom->kds_dep_satisfied) { katom->kds_dep_satisfied = true; kbase_jd_dep_clear_locked(katom); } mutex_unlock(&ctx->lock); } static void kbase_cancel_kds_wait_job(struct kbase_jd_atom *katom) { KBASE_DEBUG_ASSERT(katom); /* Prevent job_done_nolock from being called twice on an atom when * there is a race between job completion and cancellation */ if (katom->status == KBASE_JD_ATOM_STATE_QUEUED) { /* Wait was cancelled - zap the atom */ katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; if (jd_done_nolock(katom, NULL)) kbase_js_sched_all(katom->kctx->kbdev); } } #endif /* CONFIG_KDS */ void kbase_jd_free_external_resources(struct kbase_jd_atom *katom) { #ifdef CONFIG_KDS if (katom->kds_rset) { struct kbase_jd_context *jctx = &katom->kctx->jctx; /* * As the atom is no longer waiting, remove it from * the waiting list. */ mutex_lock(&jctx->lock); kbase_jd_kds_waiters_remove(katom); mutex_unlock(&jctx->lock); /* Release the kds resource or cancel if zapping */ kds_resource_set_release_sync(&katom->kds_rset); } #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE /* Flush dma-fence workqueue to ensure that any callbacks that may have * been queued are done before continuing. * Any successfully completed atom would have had all it's callbacks * completed before the atom was run, so only flush for failed atoms. */ if (katom->event_code != BASE_JD_EVENT_DONE) flush_workqueue(katom->kctx->dma_fence.wq); #endif /* CONFIG_MALI_DMA_FENCE */ } static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) { KBASE_DEBUG_ASSERT(katom); KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); #ifdef CONFIG_KDS /* Prevent the KDS resource from triggering the atom in case of zapping */ if (katom->kds_rset) katom->kds_dep_satisfied = true; #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE kbase_dma_fence_signal(katom); #endif /* CONFIG_MALI_DMA_FENCE */ kbase_gpu_vm_lock(katom->kctx); /* only roll back if extres is non-NULL */ if (katom->extres) { u32 res_no; res_no = katom->nr_extres; while (res_no-- > 0) { struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; struct kbase_va_region *reg; reg = kbase_region_tracker_find_region_base_address( katom->kctx, katom->extres[res_no].gpu_address); kbase_unmap_external_resource(katom->kctx, reg, alloc); } kfree(katom->extres); katom->extres = NULL; } kbase_gpu_vm_unlock(katom->kctx); } /* * Set up external resources needed by this job. * * jctx.lock must be held when this is called. */ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) { int err_ret_val = -EINVAL; u32 res_no; #ifdef CONFIG_KDS u32 kds_res_count = 0; struct kds_resource **kds_resources = NULL; unsigned long *kds_access_bitmap = NULL; #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE struct kbase_dma_fence_resv_info info = { .dma_fence_resv_count = 0, }; #ifdef CONFIG_SYNC /* * When both dma-buf fence and Android native sync is enabled, we * disable dma-buf fence for contexts that are using Android native * fences. */ const bool implicit_sync = !kbase_ctx_flag(katom->kctx, KCTX_NO_IMPLICIT_SYNC); #else /* CONFIG_SYNC */ const bool implicit_sync = true; #endif /* CONFIG_SYNC */ #endif /* CONFIG_MALI_DMA_FENCE */ struct base_external_resource *input_extres; KBASE_DEBUG_ASSERT(katom); KBASE_DEBUG_ASSERT(katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES); /* no resources encoded, early out */ if (!katom->nr_extres) return -EINVAL; katom->extres = kmalloc_array(katom->nr_extres, sizeof(*katom->extres), GFP_KERNEL); if (NULL == katom->extres) { err_ret_val = -ENOMEM; goto early_err_out; } /* copy user buffer to the end of our real buffer. * Make sure the struct sizes haven't changed in a way * we don't support */ BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); input_extres = (struct base_external_resource *) (((unsigned char *)katom->extres) + (sizeof(*katom->extres) - sizeof(*input_extres)) * katom->nr_extres); if (copy_from_user(input_extres, get_compat_pointer(katom->kctx, &user_atom->extres_list), sizeof(*input_extres) * katom->nr_extres) != 0) { err_ret_val = -EINVAL; goto early_err_out; } #ifdef CONFIG_KDS /* assume we have to wait for all */ KBASE_DEBUG_ASSERT(0 != katom->nr_extres); kds_resources = kmalloc_array(katom->nr_extres, sizeof(struct kds_resource *), GFP_KERNEL); if (!kds_resources) { err_ret_val = -ENOMEM; goto early_err_out; } KBASE_DEBUG_ASSERT(0 != katom->nr_extres); kds_access_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), sizeof(unsigned long), GFP_KERNEL); if (!kds_access_bitmap) { err_ret_val = -ENOMEM; goto early_err_out; } #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE if (implicit_sync) { info.resv_objs = kmalloc_array(katom->nr_extres, sizeof(struct reservation_object *), GFP_KERNEL); if (!info.resv_objs) { err_ret_val = -ENOMEM; goto early_err_out; } info.dma_fence_excl_bitmap = kcalloc(BITS_TO_LONGS(katom->nr_extres), sizeof(unsigned long), GFP_KERNEL); if (!info.dma_fence_excl_bitmap) { err_ret_val = -ENOMEM; goto early_err_out; } } #endif /* CONFIG_MALI_DMA_FENCE */ /* Take the processes mmap lock */ down_read(¤t->mm->mmap_sem); /* need to keep the GPU VM locked while we set up UMM buffers */ kbase_gpu_vm_lock(katom->kctx); for (res_no = 0; res_no < katom->nr_extres; res_no++) { struct base_external_resource *res; struct kbase_va_region *reg; struct kbase_mem_phy_alloc *alloc; bool exclusive; res = &input_extres[res_no]; exclusive = (res->ext_resource & BASE_EXT_RES_ACCESS_EXCLUSIVE) ? true : false; reg = kbase_region_tracker_find_region_enclosing_address( katom->kctx, res->ext_resource & ~BASE_EXT_RES_ACCESS_EXCLUSIVE); /* did we find a matching region object? */ if (NULL == reg || (reg->flags & KBASE_REG_FREE)) { /* roll back */ goto failed_loop; } if (!(katom->core_req & BASE_JD_REQ_SOFT_JOB) && (reg->flags & KBASE_REG_SECURE)) { katom->atom_flags |= KBASE_KATOM_FLAG_PROTECTED; } alloc = kbase_map_external_resource(katom->kctx, reg, current->mm #ifdef CONFIG_KDS , &kds_res_count, kds_resources, kds_access_bitmap, exclusive #endif ); if (!alloc) { err_ret_val = -EINVAL; goto failed_loop; } #ifdef CONFIG_MALI_DMA_FENCE if (implicit_sync && reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { struct reservation_object *resv; resv = reg->gpu_alloc->imported.umm.dma_buf->resv; if (resv) kbase_dma_fence_add_reservation(resv, &info, exclusive); } #endif /* CONFIG_MALI_DMA_FENCE */ /* finish with updating out array with the data we found */ /* NOTE: It is important that this is the last thing we do (or * at least not before the first write) as we overwrite elements * as we loop and could be overwriting ourself, so no writes * until the last read for an element. * */ katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ katom->extres[res_no].alloc = alloc; } /* successfully parsed the extres array */ /* drop the vm lock before we call into kds */ kbase_gpu_vm_unlock(katom->kctx); /* Release the processes mmap lock */ up_read(¤t->mm->mmap_sem); #ifdef CONFIG_KDS if (kds_res_count) { int wait_failed; /* We have resources to wait for with kds */ katom->kds_dep_satisfied = false; wait_failed = kds_async_waitall(&katom->kds_rset, &katom->kctx->jctx.kds_cb, katom, NULL, kds_res_count, kds_access_bitmap, kds_resources); if (wait_failed) goto failed_kds_setup; else kbase_jd_kds_waiters_add(katom); } else { /* Nothing to wait for, so kds dep met */ katom->kds_dep_satisfied = true; } kfree(kds_resources); kfree(kds_access_bitmap); #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE if (implicit_sync) { if (info.dma_fence_resv_count) { int ret; ret = kbase_dma_fence_wait(katom, &info); if (ret < 0) goto failed_dma_fence_setup; } kfree(info.resv_objs); kfree(info.dma_fence_excl_bitmap); } #endif /* CONFIG_MALI_DMA_FENCE */ /* all done OK */ return 0; /* error handling section */ #ifdef CONFIG_MALI_DMA_FENCE failed_dma_fence_setup: #ifdef CONFIG_KDS /* If we are here, dma_fence setup failed but KDS didn't. * Revert KDS setup if any. */ if (kds_res_count) { mutex_unlock(&katom->kctx->jctx.lock); kds_resource_set_release_sync(&katom->kds_rset); mutex_lock(&katom->kctx->jctx.lock); kbase_jd_kds_waiters_remove(katom); katom->kds_dep_satisfied = true; } #endif /* CONFIG_KDS */ #endif /* CONFIG_MALI_DMA_FENCE */ #ifdef CONFIG_KDS failed_kds_setup: #endif #if defined(CONFIG_KDS) || defined(CONFIG_MALI_DMA_FENCE) /* Lock the processes mmap lock */ down_read(¤t->mm->mmap_sem); /* lock before we unmap */ kbase_gpu_vm_lock(katom->kctx); #endif failed_loop: /* undo the loop work */ while (res_no-- > 0) { struct kbase_mem_phy_alloc *alloc = katom->extres[res_no].alloc; kbase_unmap_external_resource(katom->kctx, NULL, alloc); } kbase_gpu_vm_unlock(katom->kctx); /* Release the processes mmap lock */ up_read(¤t->mm->mmap_sem); early_err_out: kfree(katom->extres); katom->extres = NULL; #ifdef CONFIG_KDS kfree(kds_resources); kfree(kds_access_bitmap); #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE if (implicit_sync) { kfree(info.resv_objs); kfree(info.dma_fence_excl_bitmap); } #endif return err_ret_val; } static inline void jd_resolve_dep(struct list_head *out_list, struct kbase_jd_atom *katom, u8 d, bool ctx_is_dying) { u8 other_d = !d; while (!list_empty(&katom->dep_head[d])) { struct kbase_jd_atom *dep_atom; struct kbase_jd_atom *other_dep_atom; u8 dep_type; dep_atom = list_entry(katom->dep_head[d].next, struct kbase_jd_atom, dep_item[d]); list_del(katom->dep_head[d].next); dep_type = kbase_jd_katom_dep_type(&dep_atom->dep[d]); kbase_jd_katom_dep_clear(&dep_atom->dep[d]); if (katom->event_code != BASE_JD_EVENT_DONE && (dep_type != BASE_JD_DEP_TYPE_ORDER)) { #ifdef CONFIG_KDS if (!dep_atom->kds_dep_satisfied) { /* Just set kds_dep_satisfied to true. If the callback happens after this then it will early out and * do nothing. If the callback doesn't happen then kbase_jd_post_external_resources will clean up */ dep_atom->kds_dep_satisfied = true; } #endif #ifdef CONFIG_MALI_DMA_FENCE kbase_dma_fence_cancel_callbacks(dep_atom); #endif dep_atom->event_code = katom->event_code; KBASE_DEBUG_ASSERT(dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED); if ((dep_atom->core_req & BASE_JD_REQ_SOFT_REPLAY) != BASE_JD_REQ_SOFT_REPLAY) { dep_atom->will_fail_event_code = dep_atom->event_code; } else { dep_atom->status = KBASE_JD_ATOM_STATE_COMPLETED; } } other_dep_atom = (struct kbase_jd_atom *) kbase_jd_katom_dep_atom(&dep_atom->dep[other_d]); if (!dep_atom->in_jd_list && (!other_dep_atom || (IS_GPU_ATOM(dep_atom) && !ctx_is_dying && !dep_atom->will_fail_event_code && !other_dep_atom->will_fail_event_code))) { bool dep_satisfied = true; #ifdef CONFIG_MALI_DMA_FENCE int dep_count; dep_count = kbase_fence_dep_count_read(dep_atom); if (likely(dep_count == -1)) { dep_satisfied = true; } else { /* * There are either still active callbacks, or * all fences for this @dep_atom has signaled, * but the worker that will queue the atom has * not yet run. * * Wait for the fences to signal and the fence * worker to run and handle @dep_atom. If * @dep_atom was completed due to error on * @katom, then the fence worker will pick up * the complete status and error code set on * @dep_atom above. */ dep_satisfied = false; } #endif /* CONFIG_MALI_DMA_FENCE */ #ifdef CONFIG_KDS dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; #endif if (dep_satisfied) { dep_atom->in_jd_list = true; list_add_tail(&dep_atom->jd_item, out_list); } } } } KBASE_EXPORT_TEST_API(jd_resolve_dep); #if MALI_CUSTOMER_RELEASE == 0 static void jd_force_failure(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { kbdev->force_replay_count++; if (kbdev->force_replay_count >= kbdev->force_replay_limit) { kbdev->force_replay_count = 0; katom->event_code = BASE_JD_EVENT_FORCE_REPLAY; if (kbdev->force_replay_random) kbdev->force_replay_limit = (prandom_u32() % KBASEP_FORCE_REPLAY_RANDOM_LIMIT) + 1; dev_info(kbdev->dev, "force_replay : promoting to error\n"); } } /** Test to see if atom should be forced to fail. * * This function will check if an atom has a replay job as a dependent. If so * then it will be considered for forced failure. */ static void jd_check_force_failure(struct kbase_jd_atom *katom) { struct kbase_context *kctx = katom->kctx; struct kbase_device *kbdev = kctx->kbdev; int i; if ((kbdev->force_replay_limit == KBASEP_FORCE_REPLAY_DISABLED) || (katom->core_req & BASEP_JD_REQ_EVENT_NEVER)) return; for (i = 1; i < BASE_JD_ATOM_COUNT; i++) { if (kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[0]) == katom || kbase_jd_katom_dep_atom(&kctx->jctx.atoms[i].dep[1]) == katom) { struct kbase_jd_atom *dep_atom = &kctx->jctx.atoms[i]; if ((dep_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_REPLAY && (dep_atom->core_req & kbdev->force_replay_core_req) == kbdev->force_replay_core_req) { jd_force_failure(kbdev, katom); return; } } } } #endif /** * is_dep_valid - Validate that a dependency is valid for early dependency * submission * @katom: Dependency atom to validate * * A dependency is valid if any of the following are true : * - It does not exist (a non-existent dependency does not block submission) * - It is in the job scheduler * - It has completed, does not have a failure event code, and has not been * marked to fail in the future * * Return: true if valid, false otherwise */ static bool is_dep_valid(struct kbase_jd_atom *katom) { /* If there's no dependency then this is 'valid' from the perspective of * early dependency submission */ if (!katom) return true; /* Dependency must have reached the job scheduler */ if (katom->status < KBASE_JD_ATOM_STATE_IN_JS) return false; /* If dependency has completed and has failed or will fail then it is * not valid */ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && (katom->event_code != BASE_JD_EVENT_DONE || katom->will_fail_event_code)) return false; return true; } static void jd_try_submitting_deps(struct list_head *out_list, struct kbase_jd_atom *node) { int i; for (i = 0; i < 2; i++) { struct list_head *pos; list_for_each(pos, &node->dep_head[i]) { struct kbase_jd_atom *dep_atom = list_entry(pos, struct kbase_jd_atom, dep_item[i]); if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) { /*Check if atom deps look sane*/ bool dep0_valid = is_dep_valid( dep_atom->dep[0].atom); bool dep1_valid = is_dep_valid( dep_atom->dep[1].atom); bool dep_satisfied = true; #ifdef CONFIG_MALI_DMA_FENCE int dep_count; dep_count = kbase_fence_dep_count_read( dep_atom); if (likely(dep_count == -1)) { dep_satisfied = true; } else { /* * There are either still active callbacks, or * all fences for this @dep_atom has signaled, * but the worker that will queue the atom has * not yet run. * * Wait for the fences to signal and the fence * worker to run and handle @dep_atom. If * @dep_atom was completed due to error on * @katom, then the fence worker will pick up * the complete status and error code set on * @dep_atom above. */ dep_satisfied = false; } #endif /* CONFIG_MALI_DMA_FENCE */ #ifdef CONFIG_KDS dep_satisfied = dep_satisfied && dep_atom->kds_dep_satisfied; #endif if (dep0_valid && dep1_valid && dep_satisfied) { dep_atom->in_jd_list = true; list_add(&dep_atom->jd_item, out_list); } } } } } /* * Perform the necessary handling of an atom that has finished running * on the GPU. * * Note that if this is a soft-job that has had kbase_prepare_soft_job called on it then the caller * is responsible for calling kbase_finish_soft_job *before* calling this function. * * The caller must hold the kbase_jd_context.lock. */ bool jd_done_nolock(struct kbase_jd_atom *katom, struct list_head *completed_jobs_ctx) { struct kbase_context *kctx = katom->kctx; struct kbase_device *kbdev = kctx->kbdev; struct list_head completed_jobs; struct list_head runnable_jobs; bool need_to_try_schedule_context = false; int i; INIT_LIST_HEAD(&completed_jobs); INIT_LIST_HEAD(&runnable_jobs); KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); #if MALI_CUSTOMER_RELEASE == 0 jd_check_force_failure(katom); #endif /* This is needed in case an atom is failed due to being invalid, this * can happen *before* the jobs that the atom depends on have completed */ for (i = 0; i < 2; i++) { if (kbase_jd_katom_dep_atom(&katom->dep[i])) { list_del(&katom->dep_item[i]); kbase_jd_katom_dep_clear(&katom->dep[i]); } } /* With PRLAM-10817 or PRLAM-10959 the last tile of a fragment job being soft-stopped can fail with * BASE_JD_EVENT_TILE_RANGE_FAULT. * * So here if the fragment job failed with TILE_RANGE_FAULT and it has been soft-stopped, then we promote the * error code to BASE_JD_EVENT_DONE */ if ((kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10817) || kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10959)) && katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT) { if ((katom->core_req & BASE_JD_REQ_FS) && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED)) { /* Promote the failure to job done */ katom->event_code = BASE_JD_EVENT_DONE; katom->atom_flags = katom->atom_flags & (~KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED); } } katom->status = KBASE_JD_ATOM_STATE_COMPLETED; list_add_tail(&katom->jd_item, &completed_jobs); while (!list_empty(&completed_jobs)) { katom = list_entry(completed_jobs.prev, struct kbase_jd_atom, jd_item); list_del(completed_jobs.prev); KBASE_DEBUG_ASSERT(katom->status == KBASE_JD_ATOM_STATE_COMPLETED); for (i = 0; i < 2; i++) jd_resolve_dep(&runnable_jobs, katom, i, kbase_ctx_flag(kctx, KCTX_DYING)); if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) kbase_jd_post_external_resources(katom); while (!list_empty(&runnable_jobs)) { struct kbase_jd_atom *node; node = list_entry(runnable_jobs.next, struct kbase_jd_atom, jd_item); list_del(runnable_jobs.next); node->in_jd_list = false; KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); if (node->status != KBASE_JD_ATOM_STATE_COMPLETED && !kbase_ctx_flag(kctx, KCTX_DYING)) { need_to_try_schedule_context |= jd_run_atom(node); } else { node->event_code = katom->event_code; if ((node->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_REPLAY) { if (kbase_replay_process(node)) /* Don't complete this atom */ continue; } else if (node->core_req & BASE_JD_REQ_SOFT_JOB) { /* If this is a fence wait soft job * then remove it from the list of sync * waiters. */ if (BASE_JD_REQ_SOFT_FENCE_WAIT == node->core_req) kbasep_remove_waiting_soft_job(node); kbase_finish_soft_job(node); } node->status = KBASE_JD_ATOM_STATE_COMPLETED; } if (node->status == KBASE_JD_ATOM_STATE_COMPLETED) { list_add_tail(&node->jd_item, &completed_jobs); } else if (node->status == KBASE_JD_ATOM_STATE_IN_JS && !node->will_fail_event_code) { /* Node successfully submitted, try submitting * dependencies as they may now be representable * in JS */ jd_try_submitting_deps(&runnable_jobs, node); } } /* Register a completed job as a disjoint event when the GPU * is in a disjoint state (ie. being reset or replaying jobs). */ kbase_disjoint_event_potential(kctx->kbdev); if (completed_jobs_ctx) list_add_tail(&katom->jd_item, completed_jobs_ctx); else kbase_event_post(kctx, katom); /* Decrement and check the TOTAL number of jobs. This includes * those not tracked by the scheduler: 'not ready to run' and * 'dependency-only' jobs. */ if (--kctx->jctx.job_nr == 0) wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter * that we've got no more jobs (so we can be safely terminated) */ } return need_to_try_schedule_context; } KBASE_EXPORT_TEST_API(jd_done_nolock); #ifdef CONFIG_GPU_TRACEPOINTS enum { CORE_REQ_DEP_ONLY, CORE_REQ_SOFT, CORE_REQ_COMPUTE, CORE_REQ_FRAGMENT, CORE_REQ_VERTEX, CORE_REQ_TILER, CORE_REQ_FRAGMENT_VERTEX, CORE_REQ_FRAGMENT_VERTEX_TILER, CORE_REQ_FRAGMENT_TILER, CORE_REQ_VERTEX_TILER, CORE_REQ_UNKNOWN }; static const char * const core_req_strings[] = { "Dependency Only Job", "Soft Job", "Compute Shader Job", "Fragment Shader Job", "Vertex/Geometry Shader Job", "Tiler Job", "Fragment Shader + Vertex/Geometry Shader Job", "Fragment Shader + Vertex/Geometry Shader Job + Tiler Job", "Fragment Shader + Tiler Job", "Vertex/Geometry Shader Job + Tiler Job", "Unknown Job" }; static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) { if (core_req & BASE_JD_REQ_SOFT_JOB) return core_req_strings[CORE_REQ_SOFT]; if (core_req & BASE_JD_REQ_ONLY_COMPUTE) return core_req_strings[CORE_REQ_COMPUTE]; switch (core_req & (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T)) { case BASE_JD_REQ_DEP: return core_req_strings[CORE_REQ_DEP_ONLY]; case BASE_JD_REQ_FS: return core_req_strings[CORE_REQ_FRAGMENT]; case BASE_JD_REQ_CS: return core_req_strings[CORE_REQ_VERTEX]; case BASE_JD_REQ_T: return core_req_strings[CORE_REQ_TILER]; case (BASE_JD_REQ_FS | BASE_JD_REQ_CS): return core_req_strings[CORE_REQ_FRAGMENT_VERTEX]; case (BASE_JD_REQ_FS | BASE_JD_REQ_T): return core_req_strings[CORE_REQ_FRAGMENT_TILER]; case (BASE_JD_REQ_CS | BASE_JD_REQ_T): return core_req_strings[CORE_REQ_VERTEX_TILER]; case (BASE_JD_REQ_FS | BASE_JD_REQ_CS | BASE_JD_REQ_T): return core_req_strings[CORE_REQ_FRAGMENT_VERTEX_TILER]; } return core_req_strings[CORE_REQ_UNKNOWN]; } #endif bool jd_submit_atom(struct kbase_context *kctx, const struct base_jd_atom_v2 *user_atom, struct kbase_jd_atom *katom) { struct kbase_jd_context *jctx = &kctx->jctx; int queued = 0; int i; int sched_prio; bool ret; bool will_fail = false; /* Update the TOTAL number of jobs. This includes those not tracked by * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ jctx->job_nr++; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) katom->start_timestamp.tv64 = 0; #else katom->start_timestamp = 0; #endif katom->udata = user_atom->udata; katom->kctx = kctx; katom->nr_extres = user_atom->nr_extres; katom->extres = NULL; katom->device_nr = user_atom->device_nr; katom->affinity = 0; katom->jc = user_atom->jc; katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED; katom->core_req = user_atom->core_req; katom->atom_flags = 0; katom->retry_count = 0; katom->need_cache_flush_cores_retained = 0; katom->pre_dep = NULL; katom->post_dep = NULL; katom->x_pre_dep = NULL; katom->x_post_dep = NULL; katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED; /* Implicitly sets katom->protected_state.enter as well. */ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; katom->age = kctx->age_count++; INIT_LIST_HEAD(&katom->jd_item); #ifdef CONFIG_KDS /* Start by assuming that the KDS dependencies are satisfied, * kbase_jd_pre_external_resources will correct this if there are dependencies */ katom->kds_dep_satisfied = true; katom->kds_rset = NULL; #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE kbase_fence_dep_count_set(katom, -1); #endif /* Don't do anything if there is a mess up with dependencies. This is done in a separate cycle to check both the dependencies at ones, otherwise it will be extra complexity to deal with 1st dependency ( just added to the list ) if only the 2nd one has invalid config. */ for (i = 0; i < 2; i++) { int dep_atom_number = user_atom->pre_dep[i].atom_id; base_jd_dep_type dep_atom_type = user_atom->pre_dep[i].dependency_type; if (dep_atom_number) { if (dep_atom_type != BASE_JD_DEP_TYPE_ORDER && dep_atom_type != BASE_JD_DEP_TYPE_DATA) { katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; katom->status = KBASE_JD_ATOM_STATE_COMPLETED; /* Wrong dependency setup. Atom will be sent * back to user space. Do not record any * dependencies. */ KBASE_TLSTREAM_TL_NEW_ATOM( katom, kbase_jd_atom_id(kctx, katom)); KBASE_TLSTREAM_TL_RET_ATOM_CTX( katom, kctx); KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); ret = jd_done_nolock(katom, NULL); goto out; } } } /* Add dependencies */ for (i = 0; i < 2; i++) { int dep_atom_number = user_atom->pre_dep[i].atom_id; base_jd_dep_type dep_atom_type; struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; dep_atom_type = user_atom->pre_dep[i].dependency_type; kbase_jd_katom_dep_clear(&katom->dep[i]); if (!dep_atom_number) continue; if (dep_atom->status == KBASE_JD_ATOM_STATE_UNUSED || dep_atom->status == KBASE_JD_ATOM_STATE_COMPLETED) { if (dep_atom->event_code == BASE_JD_EVENT_DONE) continue; /* don't stop this atom if it has an order dependency * only to the failed one, try to submit it through * the normal path */ if (dep_atom_type == BASE_JD_DEP_TYPE_ORDER && dep_atom->event_code > BASE_JD_EVENT_ACTIVE) { continue; } /* Atom has completed, propagate the error code if any */ katom->event_code = dep_atom->event_code; katom->status = KBASE_JD_ATOM_STATE_QUEUED; /* This atom is going through soft replay or * will be sent back to user space. Do not record any * dependencies. */ KBASE_TLSTREAM_TL_NEW_ATOM( katom, kbase_jd_atom_id(kctx, katom)); KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_REPLAY) { if (kbase_replay_process(katom)) { ret = false; goto out; } } will_fail = true; } else { /* Atom is in progress, add this atom to the list */ list_add_tail(&katom->dep_item[i], &dep_atom->dep_head[i]); kbase_jd_katom_dep_set(&katom->dep[i], dep_atom, dep_atom_type); queued = 1; } } if (will_fail) { if (!queued) { ret = jd_done_nolock(katom, NULL); goto out; } else { katom->will_fail_event_code = katom->event_code; ret = false; goto out; } } else { /* These must occur after the above loop to ensure that an atom * that depends on a previous atom with the same number behaves * as expected */ katom->event_code = BASE_JD_EVENT_DONE; katom->status = KBASE_JD_ATOM_STATE_QUEUED; } /* For invalid priority, be most lenient and choose the default */ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; katom->sched_priority = sched_prio; /* Create a new atom recording all dependencies it was set up with. */ KBASE_TLSTREAM_TL_NEW_ATOM( katom, kbase_jd_atom_id(kctx, katom)); KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_IDLE); KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(katom, katom->sched_priority); KBASE_TLSTREAM_TL_RET_ATOM_CTX(katom, kctx); for (i = 0; i < 2; i++) if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type( &katom->dep[i])) { KBASE_TLSTREAM_TL_DEP_ATOM_ATOM( (void *)kbase_jd_katom_dep_atom( &katom->dep[i]), (void *)katom); } else if (BASE_JD_DEP_TYPE_INVALID != user_atom->pre_dep[i].dependency_type) { /* Resolved dependency. */ int dep_atom_number = user_atom->pre_dep[i].atom_id; struct kbase_jd_atom *dep_atom = &jctx->atoms[dep_atom_number]; KBASE_TLSTREAM_TL_RDEP_ATOM_ATOM( (void *)dep_atom, (void *)katom); } /* Reject atoms with job chain = NULL, as these cause issues with soft-stop */ if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { dev_warn(kctx->kbdev->dev, "Rejecting atom with jc = NULL"); katom->event_code = BASE_JD_EVENT_JOB_INVALID; ret = jd_done_nolock(katom, NULL); goto out; } /* Reject atoms with an invalid device_nr */ if ((katom->core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) && (katom->device_nr >= kctx->kbdev->gpu_props.num_core_groups)) { dev_warn(kctx->kbdev->dev, "Rejecting atom with invalid device_nr %d", katom->device_nr); katom->event_code = BASE_JD_EVENT_JOB_INVALID; ret = jd_done_nolock(katom, NULL); goto out; } /* Reject atoms with invalid core requirements */ if ((katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) && (katom->core_req & BASE_JD_REQ_EVENT_COALESCE)) { dev_warn(kctx->kbdev->dev, "Rejecting atom with invalid core requirements"); katom->event_code = BASE_JD_EVENT_JOB_INVALID; katom->core_req &= ~BASE_JD_REQ_EVENT_COALESCE; ret = jd_done_nolock(katom, NULL); goto out; } if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) { /* handle what we need to do to access the external resources */ if (kbase_jd_pre_external_resources(katom, user_atom) != 0) { /* setup failed (no access, bad resource, unknown resource types, etc.) */ katom->event_code = BASE_JD_EVENT_JOB_INVALID; ret = jd_done_nolock(katom, NULL); goto out; } } /* Validate the atom. Function will return error if the atom is * malformed. * * Soft-jobs never enter the job scheduler but have their own initialize method. * * If either fail then we immediately complete the atom with an error. */ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0) { if (!kbase_js_is_atom_valid(kctx->kbdev, katom)) { katom->event_code = BASE_JD_EVENT_JOB_INVALID; ret = jd_done_nolock(katom, NULL); goto out; } } else { /* Soft-job */ if (kbase_prepare_soft_job(katom) != 0) { katom->event_code = BASE_JD_EVENT_JOB_INVALID; ret = jd_done_nolock(katom, NULL); goto out; } } #ifdef CONFIG_GPU_TRACEPOINTS katom->work_id = atomic_inc_return(&jctx->work_id); trace_gpu_job_enqueue((u32)kctx->id, katom->work_id, kbasep_map_core_reqs_to_string(katom->core_req)); #endif if (queued && !IS_GPU_ATOM(katom)) { ret = false; goto out; } #ifdef CONFIG_KDS if (!katom->kds_dep_satisfied) { /* Queue atom due to KDS dependency */ ret = false; goto out; } #endif /* CONFIG_KDS */ #ifdef CONFIG_MALI_DMA_FENCE if (kbase_fence_dep_count_read(katom) != -1) { ret = false; goto out; } #endif /* CONFIG_MALI_DMA_FENCE */ if ((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == BASE_JD_REQ_SOFT_REPLAY) { if (kbase_replay_process(katom)) ret = false; else ret = jd_done_nolock(katom, NULL); goto out; } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { if (kbase_process_soft_job(katom) == 0) { kbase_finish_soft_job(katom); ret = jd_done_nolock(katom, NULL); goto out; } ret = false; } else if ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { katom->status = KBASE_JD_ATOM_STATE_IN_JS; ret = kbasep_js_add_job(kctx, katom); /* If job was cancelled then resolve immediately */ if (katom->event_code == BASE_JD_EVENT_JOB_CANCELLED) ret = jd_done_nolock(katom, NULL); } else { /* This is a pure dependency. Resolve it immediately */ ret = jd_done_nolock(katom, NULL); } out: return ret; } int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_atoms, u32 stride, bool uk6_atom) { struct kbase_jd_context *jctx = &kctx->jctx; int err = 0; int i; bool need_to_try_schedule_context = false; struct kbase_device *kbdev; u32 latest_flush; /* * kbase_jd_submit isn't expected to fail and so all errors with the * jobs are reported by immediately failing them (through event system) */ kbdev = kctx->kbdev; beenthere(kctx, "%s", "Enter"); if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) { dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it"); return -EINVAL; } if (stride != sizeof(base_jd_atom_v2)) { dev_err(kbdev->dev, "Stride passed to job_submit doesn't match kernel"); return -EINVAL; } KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_add_return(nr_atoms, &kctx->timeline.jd_atoms_in_flight)); /* All atoms submitted in this call have the same flush ID */ latest_flush = kbase_backend_get_current_flush_id(kbdev); for (i = 0; i < nr_atoms; i++) { struct base_jd_atom_v2 user_atom; struct kbase_jd_atom *katom; #ifdef BASE_LEGACY_UK6_SUPPORT BUILD_BUG_ON(sizeof(struct base_jd_atom_v2_uk6) != sizeof(base_jd_atom_v2)); if (uk6_atom) { struct base_jd_atom_v2_uk6 user_atom_v6; base_jd_dep_type dep_types[2] = {BASE_JD_DEP_TYPE_DATA, BASE_JD_DEP_TYPE_DATA}; if (copy_from_user(&user_atom_v6, user_addr, sizeof(user_atom_v6))) { err = -EINVAL; KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return( nr_atoms - i, &kctx->timeline.jd_atoms_in_flight)); break; } /* Convert from UK6 atom format to UK7 format */ user_atom.jc = user_atom_v6.jc; user_atom.udata = user_atom_v6.udata; user_atom.extres_list = user_atom_v6.extres_list; user_atom.nr_extres = user_atom_v6.nr_extres; user_atom.core_req = (u32)(user_atom_v6.core_req & 0x7fff); /* atom number 0 is used for no dependency atoms */ if (!user_atom_v6.pre_dep[0]) dep_types[0] = BASE_JD_DEP_TYPE_INVALID; base_jd_atom_dep_set(&user_atom.pre_dep[0], user_atom_v6.pre_dep[0], dep_types[0]); /* atom number 0 is used for no dependency atoms */ if (!user_atom_v6.pre_dep[1]) dep_types[1] = BASE_JD_DEP_TYPE_INVALID; base_jd_atom_dep_set(&user_atom.pre_dep[1], user_atom_v6.pre_dep[1], dep_types[1]); user_atom.atom_number = user_atom_v6.atom_number; user_atom.prio = user_atom_v6.prio; user_atom.device_nr = user_atom_v6.device_nr; } else { #endif /* BASE_LEGACY_UK6_SUPPORT */ if (copy_from_user(&user_atom, user_addr, sizeof(user_atom)) != 0) { err = -EINVAL; KBASE_TIMELINE_ATOMS_IN_FLIGHT(kctx, atomic_sub_return(nr_atoms - i, &kctx->timeline.jd_atoms_in_flight)); break; } #ifdef BASE_LEGACY_UK6_SUPPORT } #endif #ifdef BASE_LEGACY_UK10_2_SUPPORT if (KBASE_API_VERSION(10, 3) > kctx->api_version) user_atom.core_req = (u32)(user_atom.compat_core_req & 0x7fff); #endif /* BASE_LEGACY_UK10_2_SUPPORT */ user_addr = (void __user *)((uintptr_t) user_addr + stride); mutex_lock(&jctx->lock); #ifndef compiletime_assert #define compiletime_assert_defined #define compiletime_assert(x, msg) do { switch (0) { case 0: case (x):; } } \ while (false) #endif compiletime_assert((1 << (8*sizeof(user_atom.atom_number))) >= BASE_JD_ATOM_COUNT, "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); compiletime_assert(sizeof(user_atom.pre_dep[0].atom_id) == sizeof(user_atom.atom_number), "BASE_JD_ATOM_COUNT and base_atom_id type out of sync"); #ifdef compiletime_assert_defined #undef compiletime_assert #undef compiletime_assert_defined #endif katom = &jctx->atoms[user_atom.atom_number]; /* Record the flush ID for the cache flush optimisation */ katom->flush_id = latest_flush; while (katom->status != KBASE_JD_ATOM_STATE_UNUSED) { /* Atom number is already in use, wait for the atom to * complete */ mutex_unlock(&jctx->lock); /* This thread will wait for the atom to complete. Due * to thread scheduling we are not sure that the other * thread that owns the atom will also schedule the * context, so we force the scheduler to be active and * hence eventually schedule this context at some point * later. */ kbase_js_sched_all(kbdev); if (wait_event_killable(katom->completed, katom->status == KBASE_JD_ATOM_STATE_UNUSED) != 0) { /* We're being killed so the result code * doesn't really matter */ return 0; } mutex_lock(&jctx->lock); } need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, katom); /* Register a completed job as a disjoint event when the GPU is in a disjoint state * (ie. being reset or replaying jobs). */ kbase_disjoint_event_potential(kbdev); mutex_unlock(&jctx->lock); } if (need_to_try_schedule_context) kbase_js_sched_all(kbdev); return err; } KBASE_EXPORT_TEST_API(kbase_jd_submit); void kbase_jd_done_worker(struct work_struct *data) { struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); struct kbase_jd_context *jctx; struct kbase_context *kctx; struct kbasep_js_kctx_info *js_kctx_info; struct kbase_device *kbdev; struct kbasep_js_device_data *js_devdata; u64 cache_jc = katom->jc; struct kbasep_js_atom_retained_state katom_retained_state; bool context_idle; base_jd_core_req core_req = katom->core_req; u64 affinity = katom->affinity; enum kbase_atom_coreref_state coreref_state = katom->coreref_state; /* Soft jobs should never reach this function */ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); kctx = katom->kctx; jctx = &kctx->jctx; kbdev = kctx->kbdev; js_kctx_info = &kctx->jctx.sched_info; js_devdata = &kbdev->js_data; KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); kbase_backend_complete_wq(kbdev, katom); /* * Begin transaction on JD context and JS context */ mutex_lock(&jctx->lock); KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(katom, TL_ATOM_STATE_DONE); mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); /* This worker only gets called on contexts that are scheduled *in*. This is * because it only happens in response to an IRQ from a job that was * running. */ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED)); if (katom->event_code == BASE_JD_EVENT_STOPPED) { /* Atom has been promoted to stopped */ unsigned long flags; mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); katom->status = KBASE_JD_ATOM_STATE_IN_JS; kbase_js_unpull(kctx, katom); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&jctx->lock); return; } if (katom->event_code != BASE_JD_EVENT_DONE) dev_err(kbdev->dev, "t6xx: GPU fault 0x%02lx from job slot %d\n", (unsigned long)katom->event_code, katom->slot_nr); if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) kbase_as_poking_timer_release_atom(kbdev, kctx, katom); /* Retain state before the katom disappears */ kbasep_js_atom_retained_state_copy(&katom_retained_state, katom); context_idle = kbase_js_complete_atom_wq(kctx, katom); KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state)); kbasep_js_remove_job(kbdev, kctx, katom); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF; /* jd_done_nolock() requires the jsctx_mutex lock to be dropped */ jd_done_nolock(katom, &kctx->completed_jobs); /* katom may have been freed now, do not use! */ if (context_idle) { unsigned long flags; context_idle = false; mutex_lock(&js_devdata->queue_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); /* If kbase_sched() has scheduled this context back in then * KCTX_ACTIVE will have been set after we marked it as * inactive, and another pm reference will have been taken, so * drop our reference. But do not call kbase_jm_idle_ctx(), as * the context is active and fast-starting is allowed. * * If an atom has been fast-started then kctx->atoms_pulled will * be non-zero but KCTX_ACTIVE will still be false (as the * previous pm reference has been inherited). Do NOT drop our * reference, as it has been re-used, and leave the context as * active. * * If no new atoms have been started then KCTX_ACTIVE will still * be false and atoms_pulled will be zero, so drop the reference * and call kbase_jm_idle_ctx(). * * As the checks are done under both the queue_mutex and * hwaccess_lock is should be impossible for this to race * with the scheduler code. */ if (kbase_ctx_flag(kctx, KCTX_ACTIVE) || !atomic_read(&kctx->atoms_pulled)) { /* Calling kbase_jm_idle_ctx() here will ensure that * atoms are not fast-started when we drop the * hwaccess_lock. This is not performed if * KCTX_ACTIVE is set as in that case another pm * reference has been taken and a fast-start would be * valid. */ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE)) kbase_jm_idle_ctx(kbdev, kctx); context_idle = true; } else { kbase_ctx_flag_set(kctx, KCTX_ACTIVE); } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&js_devdata->queue_mutex); } /* * Transaction complete */ mutex_unlock(&jctx->lock); /* Job is now no longer running, so can now safely release the context * reference, and handle any actions that were logged against the atom's retained state */ kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); kbase_js_sched_all(kbdev); if (!atomic_dec_return(&kctx->work_count)) { /* If worker now idle then post all events that jd_done_nolock() * has queued */ mutex_lock(&jctx->lock); while (!list_empty(&kctx->completed_jobs)) { struct kbase_jd_atom *atom = list_entry( kctx->completed_jobs.next, struct kbase_jd_atom, jd_item); list_del(kctx->completed_jobs.next); kbase_event_post(kctx, atom); } mutex_unlock(&jctx->lock); } kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity, coreref_state); if (context_idle) kbase_pm_context_idle(kbdev); KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); } /** * jd_cancel_worker - Work queue job cancel function. * @data: a &struct work_struct * * Only called as part of 'Zapping' a context (which occurs on termination). * Operates serially with the kbase_jd_done_worker() on the work queue. * * This can only be called on contexts that aren't scheduled. * * We don't need to release most of the resources that would occur on * kbase_jd_done() or kbase_jd_done_worker(), because the atoms here must not be * running (by virtue of only being called on contexts that aren't * scheduled). */ static void jd_cancel_worker(struct work_struct *data) { struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); struct kbase_jd_context *jctx; struct kbase_context *kctx; struct kbasep_js_kctx_info *js_kctx_info; bool need_to_try_schedule_context; bool attr_state_changed; struct kbase_device *kbdev; /* Soft jobs should never reach this function */ KBASE_DEBUG_ASSERT((katom->core_req & BASE_JD_REQ_SOFT_JOB) == 0); kctx = katom->kctx; kbdev = kctx->kbdev; jctx = &kctx->jctx; js_kctx_info = &kctx->jctx.sched_info; KBASE_TRACE_ADD(kbdev, JD_CANCEL_WORKER, kctx, katom, katom->jc, 0); /* This only gets called on contexts that are scheduled out. Hence, we must * make sure we don't de-ref the number of running jobs (there aren't * any), nor must we try to schedule out the context (it's already * scheduled out). */ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); /* Scheduler: Remove the job from the system */ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); attr_state_changed = kbasep_js_remove_cancelled_job(kbdev, kctx, katom); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_lock(&jctx->lock); need_to_try_schedule_context = jd_done_nolock(katom, NULL); /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to * schedule the context. There's also no need for the jsctx_mutex to have been taken * around this too. */ KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); /* katom may have been freed now, do not use! */ mutex_unlock(&jctx->lock); if (attr_state_changed) kbase_js_sched_all(kbdev); } /** * kbase_jd_done - Complete a job that has been removed from the Hardware * @katom: atom which has been completed * @slot_nr: slot the atom was on * @end_timestamp: completion time * @done_code: completion code * * This must be used whenever a job has been removed from the Hardware, e.g.: * An IRQ indicates that the job finished (for both error and 'done' codes), or * the job was evicted from the JS_HEAD_NEXT registers during a Soft/Hard stop. * * Some work is carried out immediately, and the rest is deferred onto a * workqueue * * Context: * This can be called safely from atomic context. * The caller must hold kbdev->hwaccess_lock */ void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, ktime_t *end_timestamp, kbasep_js_atom_done_code done_code) { struct kbase_context *kctx; struct kbase_device *kbdev; KBASE_DEBUG_ASSERT(katom); kctx = katom->kctx; KBASE_DEBUG_ASSERT(kctx); kbdev = kctx->kbdev; KBASE_DEBUG_ASSERT(kbdev); if (done_code & KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT) katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT; KBASE_TRACE_ADD(kbdev, JD_DONE, kctx, katom, katom->jc, 0); kbase_job_check_leave_disjoint(kbdev, katom); katom->slot_nr = slot_nr; atomic_inc(&kctx->work_count); #ifdef CONFIG_DEBUG_FS /* a failed job happened and is waiting for dumping*/ if (!katom->will_fail_event_code && kbase_debug_job_fault_process(katom, katom->event_code)) return; #endif WARN_ON(work_pending(&katom->work)); KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); INIT_WORK(&katom->work, kbase_jd_done_worker); queue_work(kctx->jctx.job_done_wq, &katom->work); } KBASE_EXPORT_TEST_API(kbase_jd_done); void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { struct kbase_context *kctx; KBASE_DEBUG_ASSERT(NULL != kbdev); KBASE_DEBUG_ASSERT(NULL != katom); kctx = katom->kctx; KBASE_DEBUG_ASSERT(NULL != kctx); KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); /* This should only be done from a context that is not scheduled */ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); WARN_ON(work_pending(&katom->work)); katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; KBASE_DEBUG_ASSERT(0 == object_is_on_stack(&katom->work)); INIT_WORK(&katom->work, jd_cancel_worker); queue_work(kctx->jctx.job_done_wq, &katom->work); } void kbase_jd_zap_context(struct kbase_context *kctx) { struct kbase_jd_atom *katom; struct list_head *entry, *tmp; struct kbase_device *kbdev; KBASE_DEBUG_ASSERT(kctx); kbdev = kctx->kbdev; KBASE_TRACE_ADD(kbdev, JD_ZAP_CONTEXT, kctx, NULL, 0u, 0u); kbase_js_zap_context(kctx); mutex_lock(&kctx->jctx.lock); /* * While holding the struct kbase_jd_context lock clean up jobs which are known to kbase but are * queued outside the job scheduler. */ del_timer_sync(&kctx->soft_job_timeout); list_for_each_safe(entry, tmp, &kctx->waiting_soft_jobs) { katom = list_entry(entry, struct kbase_jd_atom, queue); kbase_cancel_soft_job(katom); } #ifdef CONFIG_KDS /* For each job waiting on a kds resource, cancel the wait and force the job to * complete early, this is done so that we don't leave jobs outstanding waiting * on kds resources which may never be released when contexts are zapped, resulting * in a hang. * * Note that we can safely iterate over the list as the struct kbase_jd_context lock is held, * this prevents items being removed when calling job_done_nolock in kbase_cancel_kds_wait_job. */ list_for_each(entry, &kctx->waiting_kds_resource) { katom = list_entry(entry, struct kbase_jd_atom, node); kbase_cancel_kds_wait_job(katom); } #endif #ifdef CONFIG_MALI_DMA_FENCE kbase_dma_fence_cancel_all_atoms(kctx); #endif mutex_unlock(&kctx->jctx.lock); #ifdef CONFIG_MALI_DMA_FENCE /* Flush dma-fence workqueue to ensure that any callbacks that may have * been queued are done before continuing. */ flush_workqueue(kctx->dma_fence.wq); #endif kbase_jm_wait_for_zero_jobs(kctx); } KBASE_EXPORT_TEST_API(kbase_jd_zap_context); int kbase_jd_init(struct kbase_context *kctx) { int i; int mali_err = 0; #ifdef CONFIG_KDS int err; #endif /* CONFIG_KDS */ KBASE_DEBUG_ASSERT(kctx); kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", WQ_HIGHPRI | WQ_UNBOUND, 1); if (NULL == kctx->jctx.job_done_wq) { mali_err = -ENOMEM; goto out1; } for (i = 0; i < BASE_JD_ATOM_COUNT; i++) { init_waitqueue_head(&kctx->jctx.atoms[i].completed); INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[0]); INIT_LIST_HEAD(&kctx->jctx.atoms[i].dep_head[1]); /* Catch userspace attempting to use an atom which doesn't exist as a pre-dependency */ kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_INVALID; kctx->jctx.atoms[i].status = KBASE_JD_ATOM_STATE_UNUSED; #if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) kctx->jctx.atoms[i].dma_fence.context = dma_fence_context_alloc(1); atomic_set(&kctx->jctx.atoms[i].dma_fence.seqno, 0); INIT_LIST_HEAD(&kctx->jctx.atoms[i].dma_fence.callbacks); #endif } mutex_init(&kctx->jctx.lock); init_waitqueue_head(&kctx->jctx.zero_jobs_wait); spin_lock_init(&kctx->jctx.tb_lock); #ifdef CONFIG_KDS err = kds_callback_init(&kctx->jctx.kds_cb, 0, kds_dep_clear); if (0 != err) { mali_err = -EINVAL; goto out2; } #endif /* CONFIG_KDS */ kctx->jctx.job_nr = 0; INIT_LIST_HEAD(&kctx->completed_jobs); atomic_set(&kctx->work_count, 0); return 0; #ifdef CONFIG_KDS out2: destroy_workqueue(kctx->jctx.job_done_wq); #endif /* CONFIG_KDS */ out1: return mali_err; } KBASE_EXPORT_TEST_API(kbase_jd_init); void kbase_jd_exit(struct kbase_context *kctx) { KBASE_DEBUG_ASSERT(kctx); #ifdef CONFIG_KDS kds_callback_term(&kctx->jctx.kds_cb); #endif /* CONFIG_KDS */ /* Work queue is emptied by this */ destroy_workqueue(kctx->jctx.job_done_wq); } KBASE_EXPORT_TEST_API(kbase_jd_exit);