diff options
Diffstat (limited to 'drivers/gpu/arm/midgard/mali_kbase_js.c')
| -rw-r--r-- | drivers/gpu/arm/midgard/mali_kbase_js.c | 978 |
1 files changed, 622 insertions, 356 deletions
diff --git a/drivers/gpu/arm/midgard/mali_kbase_js.c b/drivers/gpu/arm/midgard/mali_kbase_js.c index ac6c3ce333ed..78761e69d04d 100644 --- a/drivers/gpu/arm/midgard/mali_kbase_js.c +++ b/drivers/gpu/arm/midgard/mali_kbase_js.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -25,7 +25,9 @@ #if defined(CONFIG_MALI_GATOR_SUPPORT) #include <mali_kbase_gator.h> #endif +#if defined(CONFIG_MALI_MIPE_ENABLED) #include <mali_kbase_tlstream.h> +#endif #include <mali_kbase_hw.h> #include <mali_kbase_defs.h> @@ -77,6 +79,13 @@ static int kbase_js_get_slot(struct kbase_device *kbdev, static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, kbasep_js_policy_ctx_job_cb callback); +static bool kbase_js_evict_atom(struct kbase_context *kctx, + struct kbase_jd_atom *katom_evict, + struct kbase_jd_atom *start_katom, + struct kbase_jd_atom *head_katom, + struct list_head *evict_list, + struct jsctx_rb *rb, int idx); + /* Helper for trace subcodes */ #if KBASE_TRACE_ENABLE static int kbasep_js_trace_get_refcnt(struct kbase_device *kbdev, @@ -231,6 +240,26 @@ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, } /** + * jsctx_rb_is_empty_prio(): - Check if ring buffer is empty + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority to check. + * + * Caller must hold runpool_irq.lock + * + * Return: true if the ring buffer is empty, false otherwise. + */ +static inline bool +jsctx_rb_is_empty_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + + lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); + + return rb->running_idx == rb->write_idx; +} + +/** * jsctx_rb_none_to_pull_prio(): - Check if there are no pullable atoms * @kctx: Pointer to kbase context with ring buffer. * @js: Job slot id to check. @@ -241,16 +270,18 @@ bool kbasep_js_runpool_retain_ctx_nolock(struct kbase_device *kbdev, * ring buffer to be full (with running atoms) when this functions returns * true. * + * Caller must hold runpool_irq.lock + * * Return: true if there are no atoms to pull, false otherwise. */ static inline bool jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) { - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - return RB_EMPTY_ROOT(&rb->runnable_tree); + return rb->read_idx == rb->write_idx; } /** @@ -280,68 +311,115 @@ jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) } /** - * jsctx_queue_foreach_prio(): - Execute callback for each entry in the queue. - * @kctx: Pointer to kbase context with the queue. + * jsctx_rb_compact_prio(): - Compact a ring buffer + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to compact. + * @prio: Priority id to compact. + */ +static inline void +jsctx_rb_compact_prio(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + u16 compact_idx = rb->write_idx - 1; + u16 end_idx = rb->running_idx - 1; + u16 i; + + lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kctx->jctx.lock); + + for (i = compact_idx; i != end_idx; i--) { + if (rb->entries[i & JSCTX_RB_MASK].atom_id != + KBASEP_ATOM_ID_INVALID) { + WARN_ON(compact_idx < rb->running_idx); + rb->entries[compact_idx & JSCTX_RB_MASK].atom_id = + rb->entries[i & JSCTX_RB_MASK].atom_id; + + compact_idx--; + } + if (rb->read_idx == i) + rb->read_idx = compact_idx + 1; + } + + rb->running_idx = compact_idx + 1; +} + +/** + * jsctx_rb_compact(): - Compact all priority ring buffers + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to compact. + */ +static inline void +jsctx_rb_compact(struct kbase_context *kctx, int js) +{ + int prio; + + for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) + jsctx_rb_compact_prio(kctx, js, prio); +} + +/** + * jsctx_rb_foreach_prio(): - Execute callback for each entry in ring buffer + * @kctx: Pointer to kbase context with ring buffer. * @js: Job slot id to iterate. * @prio: Priority id to iterate. * @callback: Function pointer to callback. * - * Iterate over a queue and invoke @callback for each entry in the queue, and - * remove the entry from the queue. + * Iterate over a ring buffer and invoke @callback for each entry in buffer, and + * remove the entry from the buffer. * - * If entries are added to the queue while this is running those entries may, or - * may not be covered. To ensure that all entries in the buffer have been - * enumerated when this function returns jsctx->lock must be held when calling - * this function. + * If entries are added to the ring buffer while this is running those entries + * may, or may not be covered. To ensure that all entries in the buffer have + * been enumerated when this function returns jsctx->lock must be held when + * calling this function. * * The HW access lock, js_data.runpool_irq.lock, must always be held when * calling this function. */ static void -jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, +jsctx_rb_foreach_prio(struct kbase_context *kctx, int js, int prio, kbasep_js_policy_ctx_job_cb callback) { - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + struct kbase_jd_atom *katom; + u16 write_idx = ACCESS_ONCE(rb->write_idx); lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - while (!RB_EMPTY_ROOT(&queue->runnable_tree)) { - struct rb_node *node = rb_first(&queue->runnable_tree); - struct kbase_jd_atom *entry = rb_entry(node, - struct kbase_jd_atom, runnable_tree_node); + /* There must be no jobs currently in HW access */ + WARN_ON(rb->read_idx != rb->running_idx); - rb_erase(node, &queue->runnable_tree); - callback(kctx->kbdev, entry); - } + /* Invoke callback on all kbase_jd_atoms in the ring buffer, and + * removes them from the buffer */ + while (rb->read_idx != write_idx) { + int id = rb->entries[rb->read_idx & JSCTX_RB_MASK].atom_id; - while (!list_empty(&queue->x_dep_head)) { - struct kbase_jd_atom *entry = list_entry(queue->x_dep_head.next, - struct kbase_jd_atom, queue); + katom = kbase_jd_atom_from_id(kctx, id); - list_del(queue->x_dep_head.next); + rb->read_idx++; + rb->running_idx++; - callback(kctx->kbdev, entry); + callback(kctx->kbdev, katom); } } /** - * jsctx_queue_foreach(): - Execute callback for each entry in every queue - * @kctx: Pointer to kbase context with queue. + * jsctx_rb_foreach(): - Execute callback for each entry in all priority rb + * @kctx: Pointer to kbase context with ring buffer. * @js: Job slot id to iterate. * @callback: Function pointer to callback. * * Iterate over all the different priorities, and for each call - * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback - * for each entry, and remove the entry from the queue. + * jsctx_rb_foreach_prio() to iterate over the ring buffer and invoke @callback + * for each entry in buffer, and remove the entry from the buffer. */ static inline void -jsctx_queue_foreach(struct kbase_context *kctx, int js, +jsctx_rb_foreach(struct kbase_context *kctx, int js, kbasep_js_policy_ctx_job_cb callback) { int prio; for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) - jsctx_queue_foreach_prio(kctx, js, prio, callback); + jsctx_rb_foreach_prio(kctx, js, prio, callback); } /** @@ -358,16 +436,16 @@ jsctx_queue_foreach(struct kbase_context *kctx, int js, static inline struct kbase_jd_atom * jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) { - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; - struct rb_node *node; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + int id; lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - node = rb_first(&rb->runnable_tree); - if (!node) + if (jsctx_rb_none_to_pull_prio(kctx, js, prio)) return NULL; - return rb_entry(node, struct kbase_jd_atom, runnable_tree_node); + id = rb->entries[rb->read_idx & JSCTX_RB_MASK].atom_id; + return kbase_jd_atom_from_id(kctx, id); } /** @@ -379,8 +457,6 @@ jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a * pointer to the next atom, unless all the priority's ring buffers are empty. * - * Caller must hold the runpool_irq.lock. - * * Return: Pointer to next atom in buffer, or NULL if there is no atom. */ static inline struct kbase_jd_atom * @@ -388,8 +464,6 @@ jsctx_rb_peek(struct kbase_context *kctx, int js) { int prio; - lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - for (prio = 0; prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { struct kbase_jd_atom *katom; @@ -402,6 +476,35 @@ jsctx_rb_peek(struct kbase_context *kctx, int js) } /** + * jsctx_rb_peek_last(): - Check a ring buffer and get the last atom + * @kctx: Pointer to kbase context with ring buffer. + * @js: Job slot id to check. + * @prio: Priority id to check. + * + * Check the ring buffer for the specified @js and @prio and return a + * pointer to the last atom, unless all the priority's ring buffers are empty. + * + * The last atom is the atom that was added using jsctx_rb_add() most recently. + * + * Return: Pointer to last atom in buffer, or NULL if there is no atom. + */ +static inline struct kbase_jd_atom * +jsctx_rb_peek_last(struct kbase_context *kctx, int js, int prio) +{ + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + int id; + + lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kctx->jctx.lock); + + if (jsctx_rb_is_empty_prio(kctx, js, prio)) + return NULL; + + id = rb->entries[(rb->write_idx - 1) & JSCTX_RB_MASK].atom_id; + return kbase_jd_atom_from_id(kctx, id); +} + +/** * jsctx_rb_pull(): - Mark atom in list as running * @kctx: Pointer to kbase context with ring buffer. * @katom: Pointer to katom to pull. @@ -415,71 +518,158 @@ jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) { int prio = katom->sched_priority; int js = katom->slot_nr; - struct jsctx_queue *rb = &kctx->jsctx_queue[prio][js]; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); /* Atoms must be pulled in the correct order. */ WARN_ON(katom != jsctx_rb_peek_prio(kctx, js, prio)); - rb_erase(&katom->runnable_tree_node, &rb->runnable_tree); + rb->read_idx++; } -#define LESS_THAN_WRAP(a, b) ((s32)(a - b) < 0) - -static void -jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) +/** + * jsctx_rb_unpull(): - Undo marking of atom in list as running + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to unpull. + * + * Undo jsctx_rb_pull() and put @katom back in the queue. + * + * jsctx_rb_unpull() must be called on atoms in the same order the atoms were + * pulled. + */ +static inline void +jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) { int prio = katom->sched_priority; int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - struct rb_node **new = &(queue->runnable_tree.rb_node), *parent = NULL; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - while (*new) { - struct kbase_jd_atom *entry = container_of(*new, - struct kbase_jd_atom, runnable_tree_node); + /* Atoms must be unpulled in correct order. */ + WARN_ON(rb->entries[(rb->read_idx - 1) & JSCTX_RB_MASK].atom_id != + kbase_jd_atom_id(kctx, katom)); - parent = *new; - if (LESS_THAN_WRAP(katom->age, entry->age)) - new = &((*new)->rb_left); - else - new = &((*new)->rb_right); - } + rb->read_idx--; +} - /* Add new node and rebalance tree. */ - rb_link_node(&katom->runnable_tree_node, parent, new); - rb_insert_color(&katom->runnable_tree_node, &queue->runnable_tree); +/** + * jsctx_rb_add(): - Add atom to ring buffer + * @kctx: Pointer to kbase context with ring buffer. + * @katom: Pointer to katom to add. + * + * Add @katom to the ring buffer determined by the atom's priority and job slot + * number. + * + * If the ring buffer is full -EBUSY will be returned. + * + * Return: On success 0 is returned, on failure a negative error code. + */ +static int +jsctx_rb_add_atom(struct kbase_context *kctx, struct kbase_jd_atom *katom) +{ + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + + lockdep_assert_held(&kctx->jctx.lock); + + /* Check if the ring buffer is full */ + if ((rb->write_idx - rb->running_idx) >= JSCTX_RB_SIZE) + return -EBUSY; + + rb->entries[rb->write_idx & JSCTX_RB_MASK].atom_id = + kbase_jd_atom_id(kctx, katom); + rb->write_idx++; + + return 0; } /** - * jsctx_rb_unpull(): - Undo marking of atom in list as running + * jsctx_rb_remove(): - Remove atom from ring buffer * @kctx: Pointer to kbase context with ring buffer. - * @katom: Pointer to katom to unpull. + * @katom: Pointer to katom to remove. * - * Undo jsctx_rb_pull() and put @katom back in the queue. + * Remove @katom from the ring buffer. * - * jsctx_rb_unpull() must be called on atoms in the same order the atoms were - * pulled. + * @katom must have been pulled from the buffer earlier by jsctx_rb_pull(), and + * atoms must be removed in the same order they were pulled from the ring + * buffer. */ static inline void -jsctx_rb_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) +jsctx_rb_remove(struct kbase_context *kctx, struct kbase_jd_atom *katom) { + int prio = katom->sched_priority; + int js = katom->slot_nr; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - jsctx_tree_add(kctx, katom); + /* Atoms must be completed in order. */ + WARN_ON(rb->entries[rb->running_idx & JSCTX_RB_MASK].atom_id != + kbase_jd_atom_id(kctx, katom)); + + rb->running_idx++; } -static bool kbase_js_ctx_pullable(struct kbase_context *kctx, - int js, - bool is_scheduled); -static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, - int js); -static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, - struct kbase_context *kctx, - int js); +/** + * jsctx_rb_evict(): - Evict atom, and dependents, from ring buffer + * @kctx: Pointer to kbase context with ring buffer. + * @start_katom: Pointer to the first katom to evict. + * @head_katom: Pointer to head katom. + * @evict_list: Pointer to head of list where evicted atoms are added. + * + * Iterate over the ring buffer starting at @start_katom and evict @start_atom + * and dependent atoms in ring buffer. + * + * @evict_list and @head_katom is passed on to kbase_js_evict_atom() which will + * examine the atom dependencies. + * + * jsctx_rb_evict() is only called by kbase_js_evict_deps(). + */ +static void +jsctx_rb_evict(struct kbase_context *kctx, + struct kbase_jd_atom *start_katom, + struct kbase_jd_atom *head_katom, + struct list_head *evict_list) +{ + int prio = start_katom->sched_priority; + int js = start_katom->slot_nr; + struct jsctx_rb *rb = &kctx->jsctx_rb[prio][js]; + bool atom_in_rb = false; + u16 i, start_idx; + + lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kctx->jctx.lock); + + for (i = rb->running_idx; i != rb->write_idx; i++) { + if (rb->entries[i & JSCTX_RB_MASK].atom_id == + kbase_jd_atom_id(kctx, start_katom)) { + start_idx = i; + atom_in_rb = true; + break; + } + } + + /* start_katom must still be in ring buffer. */ + if (i == rb->write_idx || !atom_in_rb) + return; + + /* Evict all dependencies on same slot. */ + for (i = start_idx; i != rb->write_idx; i++) { + u8 katom_evict; + + katom_evict = rb->entries[i & JSCTX_RB_MASK].atom_id; + if (katom_evict != KBASEP_ATOM_ID_INVALID) { + if (!kbase_js_evict_atom(kctx, + &kctx->jctx.atoms[katom_evict], + start_katom, head_katom, + evict_list, rb, i)) + break; + } + } +} /* * Functions private to KBase ('Protected' functions) @@ -552,7 +742,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) DEFAULT_JS_CFS_CTX_RUNTIME_INIT_SLICES; jsdd->cfs_ctx_runtime_min_slices = DEFAULT_JS_CFS_CTX_RUNTIME_MIN_SLICES; - atomic_set(&jsdd->soft_job_timeout_ms, DEFAULT_JS_SOFT_JOB_TIMEOUT); dev_dbg(kbdev->dev, "JS Config Attribs: "); dev_dbg(kbdev->dev, "\tscheduling_period_ns:%u", @@ -579,8 +768,6 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) jsdd->cfs_ctx_runtime_init_slices); dev_dbg(kbdev->dev, "\tcfs_ctx_runtime_min_slices:%u", jsdd->cfs_ctx_runtime_min_slices); - dev_dbg(kbdev->dev, "\tsoft_job_timeout:%i", - atomic_read(&jsdd->soft_job_timeout_ms)); if (!(jsdd->soft_stop_ticks < jsdd->hard_stop_ticks_ss && jsdd->hard_stop_ticks_ss < jsdd->gpu_reset_ticks_ss && @@ -685,7 +872,7 @@ int kbasep_js_kctx_init(struct kbase_context * const kctx) struct kbase_device *kbdev; struct kbasep_js_kctx_info *js_kctx_info; int err; - int i, j; + int i; KBASE_DEBUG_ASSERT(kctx != NULL); @@ -725,13 +912,6 @@ int kbasep_js_kctx_init(struct kbase_context * const kctx) if (js_kctx_info->init_status != JS_KCTX_INIT_ALL) return -EINVAL; - for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { - INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); - kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; - } - } - return 0; } @@ -786,25 +966,30 @@ void kbasep_js_kctx_term(struct kbase_context *kctx) } /** - * kbase_js_ctx_list_add_pullable_nolock - Variant of - * kbase_jd_ctx_list_add_pullable() - * where the caller must hold - * runpool_irq.lock + * kbase_js_ctx_list_add_pullable - Add context to the tail of the per-slot + * pullable context queue * @kbdev: Device pointer * @kctx: Context to add to queue * @js: Job slot to use * - * Caller must hold runpool_irq.lock + * If the context is on either the pullable or unpullable queues, then it is + * removed before being added to the tail. + * + * This function should be used when queueing a context for the first time, or + * re-queueing a context that has been pulled from. + * + * Caller must hold kbasep_jd_device_data.queue_mutex * * Return: true if caller should call kbase_backend_ctx_count_changed() */ -static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, +static bool kbase_js_ctx_list_add_pullable(struct kbase_device *kbdev, struct kbase_context *kctx, int js) { bool ret = false; - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kbdev->js_data.queue_mutex); + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); @@ -827,24 +1012,30 @@ static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, } /** - * kbase_js_ctx_list_add_pullable_head_nolock - Variant of - * kbase_js_ctx_list_add_pullable_head() - * where the caller must hold - * runpool_irq.lock + * kbase_js_ctx_list_add_pullable_head - Add context to the head of the + * per-slot pullable context queue * @kbdev: Device pointer * @kctx: Context to add to queue * @js: Job slot to use * - * Caller must hold runpool_irq.lock + * If the context is on either the pullable or unpullable queues, then it is + * removed before being added to the head. + * + * This function should be used when a context has been scheduled, but no jobs + * can currently be pulled from it. + * + * Caller must hold kbasep_jd_device_data.queue_mutex * * Return: true if caller should call kbase_backend_ctx_count_changed() */ -static bool kbase_js_ctx_list_add_pullable_head_nolock( - struct kbase_device *kbdev, struct kbase_context *kctx, int js) +static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, + struct kbase_context *kctx, + int js) { bool ret = false; - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kbdev->js_data.queue_mutex); + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); @@ -867,37 +1058,8 @@ static bool kbase_js_ctx_list_add_pullable_head_nolock( } /** - * kbase_js_ctx_list_add_pullable_head - Add context to the head of the - * per-slot pullable context queue - * @kbdev: Device pointer - * @kctx: Context to add to queue - * @js: Job slot to use - * - * If the context is on either the pullable or unpullable queues, then it is - * removed before being added to the head. - * - * This function should be used when a context has been scheduled, but no jobs - * can currently be pulled from it. - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, - struct kbase_context *kctx, - int js) -{ - bool ret; - unsigned long flags; - - spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags); - ret = kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, js); - spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags); - - return ret; -} - -/** - * kbase_js_ctx_list_add_unpullable_nolock - Add context to the tail of the - * per-slot unpullable context queue + * kbase_js_ctx_list_add_unpullable - Add context to the tail of the per-slot + * unpullable context queue * @kbdev: Device pointer * @kctx: Context to add to queue * @js: Job slot to use @@ -908,17 +1070,18 @@ static bool kbase_js_ctx_list_add_pullable_head(struct kbase_device *kbdev, * This function should be used when a context has been pulled from, and there * are no jobs remaining on the specified slot. * - * Caller must hold runpool_irq.lock + * Caller must hold kbasep_jd_device_data.queue_mutex * * Return: true if caller should call kbase_backend_ctx_count_changed() */ -static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, +static bool kbase_js_ctx_list_add_unpullable(struct kbase_device *kbdev, struct kbase_context *kctx, int js) { bool ret = false; - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kbdev->js_data.queue_mutex); + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], &kbdev->js_data.ctx_list_unpullable[js]); @@ -938,8 +1101,8 @@ static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, } /** - * kbase_js_ctx_list_remove_nolock - Remove context from the per-slot pullable - * or unpullable context queues + * kbase_js_ctx_list_remove - Remove context from the per-slot pullable or + * unpullable context queues * @kbdev: Device pointer * @kctx: Context to remove from queue * @js: Job slot to use @@ -949,17 +1112,18 @@ static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, * This function should be used when a context has no jobs on the GPU, and no * jobs remaining for the specified slot. * - * Caller must hold runpool_irq.lock + * Caller must hold kbasep_jd_device_data.queue_mutex * * Return: true if caller should call kbase_backend_ctx_count_changed() */ -static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, +static bool kbase_js_ctx_list_remove(struct kbase_device *kbdev, struct kbase_context *kctx, int js) { bool ret = false; - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kbdev->js_data.queue_mutex); + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); WARN_ON(list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])); @@ -980,24 +1144,23 @@ static bool kbase_js_ctx_list_remove_nolock(struct kbase_device *kbdev, } /** - * kbase_js_ctx_list_pop_head_nolock - Variant of kbase_js_ctx_list_pop_head() - * where the caller must hold - * runpool_irq.lock + * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable + * queue. * @kbdev: Device pointer * @js: Job slot to use * - * Caller must hold runpool_irq.lock + * Caller must hold kbasep_jd_device_data::queue_mutex * * Return: Context to use for specified slot. * NULL if no contexts present for specified slot */ -static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( +static struct kbase_context *kbase_js_ctx_list_pop_head( struct kbase_device *kbdev, int js) { struct kbase_context *kctx; - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + lockdep_assert_held(&kbdev->js_data.queue_mutex); if (list_empty(&kbdev->js_data.ctx_list_pullable[js])) return NULL; @@ -1012,28 +1175,6 @@ static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( } /** - * kbase_js_ctx_list_pop_head - Pop the head context off the per-slot pullable - * queue. - * @kbdev: Device pointer - * @js: Job slot to use - * - * Return: Context to use for specified slot. - * NULL if no contexts present for specified slot - */ -static struct kbase_context *kbase_js_ctx_list_pop_head( - struct kbase_device *kbdev, int js) -{ - struct kbase_context *kctx; - unsigned long flags; - - spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags); - kctx = kbase_js_ctx_list_pop_head_nolock(kbdev, js); - spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags); - - return kctx; -} - -/** * kbase_js_ctx_pullable - Return if a context can be pulled from on the * specified slot * @kctx: Context pointer @@ -1066,8 +1207,7 @@ static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, return false; /* next atom blocked */ if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { if (katom->x_pre_dep->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) return false; if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) @@ -1096,7 +1236,7 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, /* Dependent atom must already have been submitted */ if (!(dep_atom->atom_flags & - KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { + KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED)) { ret = false; break; } @@ -1115,12 +1255,6 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, ret = false; break; } - /* Each dependee atom can only have one - * same-slot dependency */ - if (dep_atom->post_dep) { - ret = false; - break; - } has_dep = true; } else { /* Only one cross-slot dependency can be @@ -1160,6 +1294,21 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, has_x_dep = true; } + if (kbase_jd_katom_dep_type(&katom->dep[i]) == + BASE_JD_DEP_TYPE_DATA && + js == dep_js) { + struct kbase_jd_atom *last_atom = + jsctx_rb_peek_last(kctx, js, + prio); + + /* Last atom on slot must be pre-dep for this + * atom */ + if (last_atom != dep_atom) { + ret = false; + break; + } + } + /* Dependency can be represented in ringbuffers */ } } @@ -1193,10 +1342,9 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, } if ((kbase_jd_katom_dep_type(&katom->dep[i]) == BASE_JD_DEP_TYPE_DATA) && - (js == dep_js)) { - katom->pre_dep = dep_atom; - dep_atom->post_dep = katom; - } + (js == dep_js)) + katom->atom_flags |= + KBASE_KATOM_FLAG_FAIL_PREV; list_del(&katom->dep_item[i]); kbase_jd_katom_dep_clear(&katom->dep[i]); @@ -1262,7 +1410,17 @@ bool kbasep_js_add_job(struct kbase_context *kctx, KBASE_TIMELINE_ATOM_READY(kctx, kbase_jd_atom_id(kctx, atom)); - enqueue_required = kbase_js_dep_resolved_submit(kctx, atom); + if (kbase_js_dep_resolved_submit(kctx, atom, &enqueue_required) != 0) { + /* Ringbuffer was full (should be impossible) - fail the job */ + --(js_kctx_info->ctx.nr_jobs); + + spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + + atom->event_code = BASE_JD_EVENT_JOB_CANCELLED; + + goto out_unlock; + } KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_ADD_JOB, kctx, atom, atom->jc, kbasep_js_trace_get_refcnt_nolock(kbdev, kctx)); @@ -1272,11 +1430,11 @@ bool kbasep_js_add_job(struct kbase_context *kctx, if (enqueue_required) { if (kbase_js_ctx_pullable(kctx, atom->slot_nr, false)) - timer_sync = kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, atom->slot_nr); + timer_sync = kbase_js_ctx_list_add_pullable(kbdev, kctx, + atom->slot_nr); else - timer_sync = kbase_js_ctx_list_add_unpullable_nolock( - kbdev, kctx, atom->slot_nr); + timer_sync = kbase_js_ctx_list_add_unpullable(kbdev, + kctx, atom->slot_nr); } /* If this context is active and the atom is the first on its slot, * kick the job manager to attempt to fast-start the atom */ @@ -1604,7 +1762,9 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( #if defined(CONFIG_MALI_GATOR_SUPPORT) kbase_trace_mali_mmu_as_released(kctx->as_nr); #endif +#if defined(CONFIG_MALI_MIPE_ENABLED) kbase_tlstream_tl_nret_as_ctx(&kbdev->as[kctx->as_nr], kctx); +#endif kbase_backend_release_ctx_irq(kbdev, kctx); @@ -1657,8 +1817,7 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( wake_up(&js_kctx_info->ctx.is_scheduled_wait); /* Queue an action to occur after we've dropped the lock */ - release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED | - KBASEP_JS_RELEASE_RESULT_SCHED_ALL; + release_result |= KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED; } else { kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, kctx, katom_retained_state, runpool_ctx_attr_change); @@ -1716,12 +1875,14 @@ void kbasep_js_runpool_release_ctx_and_katom_retained_state( { struct kbasep_js_device_data *js_devdata; struct kbasep_js_kctx_info *js_kctx_info; + base_jd_event_code event_code; kbasep_js_release_result release_result; KBASE_DEBUG_ASSERT(kbdev != NULL); KBASE_DEBUG_ASSERT(kctx != NULL); js_kctx_info = &kctx->jctx.sched_info; js_devdata = &kbdev->js_data; + event_code = katom_retained_state->event_code; mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); @@ -1793,11 +1954,77 @@ static void kbasep_js_runpool_release_ctx_no_schedule( */ } -void kbase_js_set_timeouts(struct kbase_device *kbdev) +/** + * kbase_js_set_timeouts - update all JS timeouts with user specified data + * @kbdev: Device pointer + * + * Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is + * set to a positive number then that becomes the new value used, if a timeout + * is negative then the default is set. + */ +static void kbase_js_set_timeouts(struct kbase_device *kbdev) { - lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + struct kbasep_js_device_data *js_data = &kbdev->js_data; + + if (kbdev->js_scheduling_period_ns < 0) + js_data->scheduling_period_ns = DEFAULT_JS_SCHEDULING_PERIOD_NS; + else if (kbdev->js_scheduling_period_ns > 0) + js_data->scheduling_period_ns = kbdev->js_scheduling_period_ns; + + if (kbdev->js_soft_stop_ticks < 0) + js_data->soft_stop_ticks = DEFAULT_JS_SOFT_STOP_TICKS; + else if (kbdev->js_soft_stop_ticks > 0) + js_data->soft_stop_ticks = kbdev->js_soft_stop_ticks; + + if (kbdev->js_soft_stop_ticks_cl < 0) + js_data->soft_stop_ticks_cl = DEFAULT_JS_SOFT_STOP_TICKS_CL; + else if (kbdev->js_soft_stop_ticks_cl > 0) + js_data->soft_stop_ticks_cl = kbdev->js_soft_stop_ticks_cl; + + if (kbdev->js_hard_stop_ticks_ss < 0) { + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) + js_data->hard_stop_ticks_ss = + DEFAULT_JS_HARD_STOP_TICKS_SS_8408; + else + js_data->hard_stop_ticks_ss = + DEFAULT_JS_HARD_STOP_TICKS_SS; + } else if (kbdev->js_hard_stop_ticks_ss > 0) { + js_data->hard_stop_ticks_ss = kbdev->js_hard_stop_ticks_ss; + } + + if (kbdev->js_hard_stop_ticks_cl < 0) + js_data->hard_stop_ticks_cl = DEFAULT_JS_HARD_STOP_TICKS_CL; + else if (kbdev->js_hard_stop_ticks_cl > 0) + js_data->hard_stop_ticks_cl = kbdev->js_hard_stop_ticks_cl; + + if (kbdev->js_hard_stop_ticks_dumping < 0) + js_data->hard_stop_ticks_dumping = + DEFAULT_JS_HARD_STOP_TICKS_DUMPING; + else if (kbdev->js_hard_stop_ticks_dumping > 0) + js_data->hard_stop_ticks_dumping = + kbdev->js_hard_stop_ticks_dumping; + + if (kbdev->js_reset_ticks_ss < 0) { + if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) + js_data->gpu_reset_ticks_ss = + DEFAULT_JS_RESET_TICKS_SS_8408; + else + js_data->gpu_reset_ticks_ss = DEFAULT_JS_RESET_TICKS_SS; + } else if (kbdev->js_reset_ticks_ss > 0) { + js_data->gpu_reset_ticks_ss = kbdev->js_reset_ticks_ss; + } + + if (kbdev->js_reset_ticks_cl < 0) + js_data->gpu_reset_ticks_cl = DEFAULT_JS_RESET_TICKS_CL; + else if (kbdev->js_reset_ticks_cl > 0) + js_data->gpu_reset_ticks_cl = kbdev->js_reset_ticks_cl; - kbase_backend_timeouts_changed(kbdev); + if (kbdev->js_reset_ticks_dumping < 0) + js_data->gpu_reset_ticks_dumping = + DEFAULT_JS_RESET_TICKS_DUMPING; + else if (kbdev->js_reset_ticks_dumping > 0) + js_data->gpu_reset_ticks_dumping = + kbdev->js_reset_ticks_dumping; } static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, @@ -1844,6 +2071,16 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, 0u, kbasep_js_trace_get_refcnt(kbdev, kctx)); + if (js_devdata->nr_user_contexts_running == 0 && + kbdev->js_timeouts_updated) { + /* Only when there are no other contexts submitting jobs: + * Latch in run-time job scheduler timeouts that were set + * through js_timeouts sysfs file */ + kbase_js_set_timeouts(kbdev); + + kbdev->js_timeouts_updated = false; + } + js_kctx_info->ctx.is_scheduled = true; mutex_lock(&new_address_space->transaction_mutex); @@ -1853,10 +2090,13 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, if (!kbase_backend_use_ctx(kbdev, kctx, as_nr)) { spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); mutex_unlock(&new_address_space->transaction_mutex); - /* Roll back the transaction so far and return */ - js_kctx_info->ctx.is_scheduled = false; + /* If address space is not pending, then kbase_backend_use_ctx() + * failed. Roll back the transaction so far and return */ + if (!kctx->as_pending) { + js_kctx_info->ctx.is_scheduled = false; - kbase_backend_release_free_address_space(kbdev, as_nr); + kbase_backend_release_free_address_space(kbdev, as_nr); + } mutex_unlock(&js_devdata->runpool_mutex); @@ -1869,7 +2109,9 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, #if defined(CONFIG_MALI_GATOR_SUPPORT) kbase_trace_mali_mmu_as_in_use(kctx->as_nr); #endif +#if defined(CONFIG_MALI_MIPE_ENABLED) kbase_tlstream_tl_ret_as_ctx(&kbdev->as[kctx->as_nr], kctx); +#endif /* Cause any future waiter-on-termination to wait until the context is * descheduled */ @@ -1927,6 +2169,11 @@ static bool kbase_js_use_ctx(struct kbase_device *kbdev, unsigned long flags; spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); + if (kctx->as_pending) { + /* Context waiting for AS to be assigned */ + spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); + return false; + } if (kbase_backend_use_ctx_sched(kbdev, kctx)) { /* Context already has ASID - mark as active */ kbdev->hwaccess.active_kctx = kctx; @@ -1965,7 +2212,7 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, is_scheduled = js_kctx_info->ctx.is_scheduled; if (!is_scheduled) { /* Add the context to the pullable list */ - if (kbase_js_ctx_list_add_pullable_head(kbdev, kctx, 0)) + if (kbase_js_ctx_list_add_pullable(kbdev, kctx, 0)) kbase_js_sync_timers(kbdev); /* Fast-starting requires the jsctx_mutex to be dropped, @@ -1993,6 +2240,7 @@ void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) { struct kbasep_js_kctx_info *js_kctx_info; + bool pending; KBASE_DEBUG_ASSERT(kctx != NULL); js_kctx_info = &kctx->jctx.sched_info; @@ -2000,10 +2248,13 @@ void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, /* We don't need to use the address space anymore */ mutex_lock(&js_kctx_info->ctx.jsctx_mutex); js_kctx_info->ctx.flags &= (~KBASE_CTX_FLAG_PRIVILEGED); + pending = kctx->as_pending; mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - /* Release the context - it will be scheduled out */ - kbasep_js_runpool_release_ctx(kbdev, kctx); + /* Release the context - it will be scheduled out if there is no + * pending job */ + if (!pending) + kbasep_js_runpool_release_ctx(kbdev, kctx); kbase_js_sched_all(kbdev); } @@ -2042,10 +2293,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) * the instrumented context. It'll be suspended by * disabling instrumentation */ if (kctx->jctx.sched_info.ctx.flags & - KBASE_CTX_FLAG_PRIVILEGED) { - ++nr_privileged_ctx; - WARN_ON(nr_privileged_ctx != 1); - } + KBASE_CTX_FLAG_PRIVILEGED) + KBASE_DEBUG_ASSERT(++nr_privileged_ctx == 1); } } CSTD_UNUSED(nr_privileged_ctx); @@ -2096,8 +2345,7 @@ void kbasep_js_resume(struct kbase_device *kbdev) if (!js_kctx_info->ctx.is_scheduled && kbase_js_ctx_pullable(kctx, js, false)) - timer_sync = - kbase_js_ctx_list_add_pullable_nolock( + timer_sync = kbase_js_ctx_list_add_pullable( kbdev, kctx, js); spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, @@ -2149,118 +2397,37 @@ static int kbase_js_get_slot(struct kbase_device *kbdev, return 1; } -bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, - struct kbase_jd_atom *katom) +int kbase_js_dep_resolved_submit(struct kbase_context *kctx, + struct kbase_jd_atom *katom, + bool *enqueue_required) { - bool enqueue_required; - katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - lockdep_assert_held(&kctx->jctx.lock); /* If slot will transition from unpullable to pullable then add to * pullable list */ if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { - enqueue_required = true; + *enqueue_required = true; } else { - enqueue_required = false; + *enqueue_required = false; } /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); - if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || - (katom->pre_dep && (katom->pre_dep->atom_flags & - KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - int prio = katom->sched_priority; - int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - - list_add_tail(&katom->queue, &queue->x_dep_head); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - enqueue_required = false; - } else { - /* Add atom to ring buffer. */ - jsctx_tree_add(kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } - - return enqueue_required; -} - -/** - * kbase_js_move_to_tree - Move atom (and any dependent atoms) to the - * runnable_tree, ready for execution - * @katom: Atom to submit - * - * It is assumed that @katom does not have KBASE_KATOM_FLAG_X_DEP_BLOCKED set, - * but is still present in the x_dep list. If @katom has a same-slot dependent - * atom then that atom (and any dependents) will also be moved. - */ -static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) -{ - lockdep_assert_held(&katom->kctx->kbdev->js_data.runpool_irq.lock); - - while (katom) { - WARN_ON(!(katom->atom_flags & - KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - - if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { - list_del(&katom->queue); - katom->atom_flags &= - ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - jsctx_tree_add(katom->kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } else { - break; - } - - katom = katom->post_dep; - } -} - - -/** - * kbase_js_evict_deps - Evict dependencies of a failed atom. - * @kctx: Context pointer - * @katom: Pointer to the atom that has failed. - * @js: The job slot the katom was run on. - * @prio: Priority of the katom. - * - * Remove all post dependencies of an atom from the context ringbuffers. - * - * The original atom's event_code will be propogated to all dependent atoms. - * - * Context: Caller must hold the HW access lock - */ -static void kbase_js_evict_deps(struct kbase_context *kctx, - struct kbase_jd_atom *katom, int js, int prio) -{ - struct kbase_jd_atom *x_dep = katom->x_post_dep; - struct kbase_jd_atom *next_katom = katom->post_dep; - - lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - - if (next_katom) { - KBASE_DEBUG_ASSERT(next_katom->status != - KBASE_JD_ATOM_STATE_HW_COMPLETED); - next_katom->will_fail_event_code = katom->event_code; - + /* Add atom to ring buffer. */ + if (unlikely(jsctx_rb_add_atom(kctx, katom))) { + /* The ring buffer is full. This should be impossible as the + * job dispatcher can not submit enough atoms to exceed the + * ring buffer size. Fail the job. + */ + WARN(1, "Job submit while JSCTX ringbuffer already full\n"); + return -EINVAL; } - /* Has cross slot depenency. */ - if (x_dep && (x_dep->atom_flags & (KBASE_KATOM_FLAG_JSCTX_IN_TREE | - KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - /* Remove dependency.*/ - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED; - /* Fail if it had a data dependency. */ - if (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) { - x_dep->will_fail_event_code = katom->event_code; - } - if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) - kbase_js_move_to_tree(x_dep); - } + return 0; } struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) @@ -2289,7 +2456,8 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) /* Due to ordering restrictions when unpulling atoms on failure, we do * not allow multiple runs of fail-dep atoms from the same context to be * present on the same slot */ - if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { + if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_PREV) && + atomic_read(&kctx->atoms_pulled_slot[js])) { struct kbase_jd_atom *prev_atom = kbase_backend_inspect_tail(kctx->kbdev, js); @@ -2299,8 +2467,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { if (katom->x_pre_dep->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) + KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) return NULL; if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) @@ -2343,8 +2510,6 @@ static void js_return_worker(struct work_struct *data) u64 affinity = katom->affinity; enum kbase_atom_coreref_state coreref_state = katom->coreref_state; - kbase_tlstream_aux_job_softstop_ex(katom); - kbase_backend_complete_wq(kbdev, katom); if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) @@ -2364,14 +2529,13 @@ static void js_return_worker(struct work_struct *data) if (!atomic_read(&kctx->atoms_pulled_slot[js]) && jsctx_rb_none_to_pull(kctx, js)) - timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); + timer_sync |= kbase_js_ctx_list_remove(kbdev, kctx, js); if (!atomic_read(&kctx->atoms_pulled)) { if (!kctx->slots_pullable) { WARN_ON(!kctx->ctx_runnable_ref); kctx->ctx_runnable_ref = false; atomic_dec(&kbdev->js_data.nr_contexts_runnable); - timer_sync = true; } if (kctx->as_nr != KBASEP_AS_NR_INVALID && @@ -2385,7 +2549,7 @@ static void js_return_worker(struct work_struct *data) for (slot = 0; slot < num_slots; slot++) { if (kbase_js_ctx_pullable(kctx, slot, true)) timer_sync |= - kbase_js_ctx_list_add_pullable_nolock( + kbase_js_ctx_list_add_pullable( kbdev, kctx, slot); } } @@ -2437,6 +2601,112 @@ void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) queue_work(kctx->jctx.job_done_wq, &katom->work); } +static bool kbase_js_evict_atom(struct kbase_context *kctx, + struct kbase_jd_atom *katom_evict, + struct kbase_jd_atom *start_katom, + struct kbase_jd_atom *head_katom, + struct list_head *evict_list, + struct jsctx_rb *rb, int idx) +{ + struct kbase_jd_atom *x_dep = katom_evict->x_post_dep; + + if (!(katom_evict->atom_flags & KBASE_KATOM_FLAG_FAIL_PREV) && + katom_evict != start_katom) + return false; + + if (katom_evict->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { + WARN_ON(katom_evict->event_code != head_katom->event_code); + + return false; + } + + if (katom_evict->status == KBASE_JD_ATOM_STATE_HW_COMPLETED && + katom_evict != head_katom) + return false; + + /* Evict cross dependency if present */ + if (x_dep && (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED) + && (x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) + list_add_tail(&x_dep->dep_item[0], evict_list); + + /* If cross dependency is present and does not have a data dependency + * then unblock */ + if (x_dep && (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED) + && !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) + x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; + + if (katom_evict != head_katom) { + rb->entries[idx & JSCTX_RB_MASK].atom_id = + KBASEP_ATOM_ID_INVALID; + + katom_evict->event_code = head_katom->event_code; + katom_evict->atom_flags &= + ~KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED; + + if (katom_evict->atom_flags & KBASE_KATOM_FLAG_HOLDING_CTX_REF) + kbase_jd_done(katom_evict, katom_evict->slot_nr, NULL, + 0); + else + kbase_jd_evict(kctx->kbdev, katom_evict); + } + + return true; +} + +/** + * kbase_js_evict_deps - Evict dependencies + * @kctx: Context pointer + * @head_katom: Pointer to the atom to evict + * + * Remove all post dependencies of an atom from the context ringbuffers. + * + * The original atom's event_code will be propogated to all dependent atoms. + * + * Context: Caller must hold both jctx and HW access locks + */ +static void kbase_js_evict_deps(struct kbase_context *kctx, + struct kbase_jd_atom *head_katom) +{ + struct list_head evict_list; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); + + INIT_LIST_HEAD(&evict_list); + + list_add_tail(&head_katom->dep_item[0], &evict_list); + + while (!list_empty(&evict_list)) { + struct kbase_jd_atom *start_katom; + + start_katom = list_entry(evict_list.prev, struct kbase_jd_atom, + dep_item[0]); + list_del(evict_list.prev); + + jsctx_rb_evict(kctx, start_katom, head_katom, &evict_list); + } +} + +/** + * kbase_js_compact - Compact JSCTX ringbuffers + * @kctx: Context pointer + * + * Compact the JSCTX ringbuffers, removing any NULL entries + * + * Context: Caller must hold both jctx and HW access locks + */ +static void kbase_js_compact(struct kbase_context *kctx) +{ + struct kbase_device *kbdev = kctx->kbdev; + int js; + + lockdep_assert_held(&kctx->jctx.sched_info.ctx.jsctx_mutex); + lockdep_assert_held(&kbdev->js_data.runpool_irq.lock); + + for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) + jsctx_rb_compact(kctx, js); +} + bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom) { @@ -2459,7 +2729,12 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, mutex_lock(&js_devdata->runpool_mutex); spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); - if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { + if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_RB_SUBMITTED) { + if (katom->event_code != BASE_JD_EVENT_DONE) + kbase_js_evict_deps(kctx, katom); + + jsctx_rb_remove(kctx, katom); + context_idle = !atomic_dec_return(&kctx->atoms_pulled); atomic_dec(&kctx->atoms_pulled_slot[atom_slot]); @@ -2468,18 +2743,16 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, WARN_ON(!kctx->ctx_runnable_ref); kctx->ctx_runnable_ref = false; atomic_dec(&kbdev->js_data.nr_contexts_runnable); - timer_sync = true; } + + if (katom->event_code != BASE_JD_EVENT_DONE) + kbase_js_compact(kctx); } - WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); if (!atomic_read(&kctx->atoms_pulled_slot[atom_slot]) && - jsctx_rb_none_to_pull(kctx, atom_slot)) { - if (!list_empty( - &kctx->jctx.sched_info.ctx.ctx_list_entry[atom_slot])) - timer_sync |= kbase_js_ctx_list_remove_nolock( - kctx->kbdev, kctx, atom_slot); - } + jsctx_rb_none_to_pull(kctx, atom_slot)) + timer_sync |= kbase_js_ctx_list_remove(kctx->kbdev, kctx, + atom_slot); /* * If submission is disabled on this context (most likely due to an @@ -2495,8 +2768,7 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock( + timer_sync |= kbase_js_ctx_list_add_pullable( kbdev, kctx, js); } } else if (katom->x_post_dep && @@ -2505,8 +2777,7 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock( + timer_sync |= kbase_js_ctx_list_add_pullable( kbdev, kctx, js); } } @@ -2531,29 +2802,32 @@ void kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp) struct kbase_device *kbdev; struct kbase_context *kctx = katom->kctx; union kbasep_js_policy *js_policy; - struct kbase_jd_atom *x_dep = katom->x_post_dep; + struct kbasep_js_device_data *js_devdata; kbdev = kctx->kbdev; js_policy = &kbdev->js_data.policy; + js_devdata = &kbdev->js_data; lockdep_assert_held(&kctx->kbdev->js_data.runpool_irq.lock); - if (katom->will_fail_event_code) - katom->event_code = katom->will_fail_event_code; - katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; - if (katom->event_code != BASE_JD_EVENT_DONE) { - kbase_js_evict_deps(kctx, katom, katom->slot_nr, - katom->sched_priority); - } - #if defined(CONFIG_MALI_GATOR_SUPPORT) kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_STOP, katom->slot_nr), NULL, 0); #endif - +#if defined(CONFIG_MALI_MIPE_ENABLED) + kbase_tlstream_tl_nret_atom_lpu( + katom, + &kbdev->gpu_props.props.raw_props.js_features[ + katom->slot_nr]); + kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]); + kbase_tlstream_tl_nret_ctx_lpu( + kctx, + &kbdev->gpu_props.props.raw_props.js_features[ + katom->slot_nr]); +#endif /* Calculate the job's time used */ if (end_timestamp != NULL) { /* Only calculating it for jobs that really run on the HW (e.g. @@ -2577,26 +2851,21 @@ void kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp) kbase_jd_done(katom, katom->slot_nr, end_timestamp, 0); /* Unblock cross dependency if present */ - if (x_dep && (katom->event_code == BASE_JD_EVENT_DONE || - !(x_dep->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER)) && - (x_dep->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { - bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, - false); - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; - kbase_js_move_to_tree(x_dep); - if (!was_pullable && kbase_js_ctx_pullable(kctx, x_dep->slot_nr, - false)) - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, - x_dep->slot_nr); - } + if (katom->x_post_dep && (katom->event_code == BASE_JD_EVENT_DONE || + !(katom->x_post_dep->atom_flags & + KBASE_KATOM_FLAG_FAIL_BLOCKER))) + katom->x_post_dep->atom_flags &= + ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; } void kbase_js_sched(struct kbase_device *kbdev, int js_mask) { struct kbasep_js_device_data *js_devdata; + union kbasep_js_policy *js_policy; bool timer_sync = false; js_devdata = &kbdev->js_data; + js_policy = &js_devdata->policy; down(&js_devdata->schedule_sem); mutex_lock(&js_devdata->queue_mutex); @@ -2646,15 +2915,16 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) /* Context can not be used at this time */ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); - if (kbase_js_ctx_pullable(kctx, js, false) + if (kctx->as_pending || + kbase_js_ctx_pullable(kctx, js, false) || (kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED)) timer_sync |= - kbase_js_ctx_list_add_pullable_head_nolock( + kbase_js_ctx_list_add_pullable_head( kctx->kbdev, kctx, js); else timer_sync |= - kbase_js_ctx_list_add_unpullable_nolock( + kbase_js_ctx_list_add_unpullable( kctx->kbdev, kctx, js); spin_unlock_irqrestore( &js_devdata->runpool_irq.lock, flags); @@ -2683,12 +2953,12 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) /* Failed to pull jobs - push to head of list */ if (kbase_js_ctx_pullable(kctx, js, true)) timer_sync |= - kbase_js_ctx_list_add_pullable_head_nolock( + kbase_js_ctx_list_add_pullable_head( kctx->kbdev, kctx, js); else timer_sync |= - kbase_js_ctx_list_add_unpullable_nolock( + kbase_js_ctx_list_add_unpullable( kctx->kbdev, kctx, js); @@ -2714,12 +2984,10 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) /* Push to back of list */ if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= - kbase_js_ctx_list_add_pullable_nolock( + timer_sync |= kbase_js_ctx_list_add_pullable( kctx->kbdev, kctx, js); else - timer_sync |= - kbase_js_ctx_list_add_unpullable_nolock( + timer_sync |= kbase_js_ctx_list_add_unpullable( kctx->kbdev, kctx, js); spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); @@ -2750,7 +3018,6 @@ void kbase_js_zap_context(struct kbase_context *kctx) /* First, atomically do the following: * - mark the context as dying * - try to evict it from the policy queue */ - mutex_lock(&kctx->jctx.lock); mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); js_kctx_info->ctx.is_dying = true; @@ -2826,7 +3093,6 @@ void kbase_js_zap_context(struct kbase_context *kctx) mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); } else { unsigned long flags; bool was_retained; @@ -2862,7 +3128,6 @@ void kbase_js_zap_context(struct kbase_context *kctx) spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", kctx); @@ -2929,6 +3194,7 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, u32 js; kbdev = kctx->kbdev; + js_devdata = &kbdev->js_data; spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); @@ -2938,7 +3204,7 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, /* Invoke callback on jobs on each slot in turn */ for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) - jsctx_queue_foreach(kctx, js, callback); + jsctx_rb_foreach(kctx, js, callback); spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); } |