diff options
author | Zhen Chen <chenzhen@rock-chips.com> | 2018-05-04 11:26:50 +0800 |
---|---|---|
committer | Tao Huang <huangtao@rock-chips.com> | 2018-05-21 11:37:35 +0800 |
commit | b1111d30eac9f266580325a542e738cb939daa0f (patch) | |
tree | f18c34dfc12cf9a9bccb2dce8f1c4870927066f1 /drivers/gpu/arm | |
parent | fda77a25309fa05ddce10f1d87a9eec5b1a71f6d (diff) |
MALI: rockchip: upgrade bifrost DDK to r12p0-01rel0
in addition,
resolve all the conflicts;
rename all the configs and macros that have a same name in midgard/;
adjust "platform specific code" for the change of interface
to get GPU busy/idle time from "common part";
Change-Id: Iad3493c15c95653a1c72c5375f510e44c4535d0c
Signed-off-by: Zhen Chen <chenzhen@rock-chips.com>
Diffstat (limited to 'drivers/gpu/arm')
41 files changed, 605 insertions, 647 deletions
diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild index deb6e8f976b0..65a3468736a8 100644 --- a/drivers/gpu/arm/bifrost/Kbuild +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2012-2016, 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -20,22 +20,17 @@ # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= "r11p0-01rel0" +MALI_RELEASE_NAME ?= "r12p0-01rel0" # Paths required for build KBASE_PATH = $(src) KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy UMP_PATH = $(src)/../../../base -ifeq ($(CONFIG_MALI_BIFROST_ERROR_INJECT),y) -MALI_ERROR_INJECT_ON = 1 -endif - # Set up defaults if not defined by build system MALI_CUSTOMER_RELEASE ?= 1 MALI_UNIT_TEST ?= 0 MALI_KERNEL_TEST_API ?= 0 -MALI_ERROR_INJECT_ON ?= 0 MALI_MOCK_TEST ?= 0 MALI_COVERAGE ?= 0 CONFIG_MALI_PLATFORM_NAME ?= "devicetree" @@ -56,7 +51,6 @@ DEFINES = \ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ - -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ -DMALI_MOCK_TEST=$(MALI_MOCK_TEST) \ -DMALI_COVERAGE=$(MALI_COVERAGE) \ -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index 075998977066..f4fecdaca0e4 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -176,17 +176,17 @@ config MALI_BIFROST_SYSTEM_TRACE default n help Choose this option to enable system trace events for each - kbase event. This is typically used for debugging but has + kbase event. This is typically used for debugging but has minimal overhead when not in use. Enable only if you know what you are doing. -config MALI_JOB_DUMPING +config MALI_JOB_DUMP bool "Enable system level support needed for job dumping" depends on MALI_BIFROST && MALI_BIFROST_EXPERT default n help Choose this option to enable system level support needed for - job dumping. This is typically used for instrumentation but has + job dumping. This is typically used for instrumentation but has minimal overhead when not in use. Enable only if you know what you are doing. diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile index cfe6fc34ad29..13af9f473890 100644 --- a/drivers/gpu/arm/bifrost/Makefile +++ b/drivers/gpu/arm/bifrost/Makefile @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2010-2016, 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -23,9 +23,7 @@ KDIR ?= /lib/modules/$(shell uname -r)/build BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. -UMP_PATH_RELATIVE = $(CURDIR)/../../../base/ump KBASE_PATH_RELATIVE = $(CURDIR) -EXTRA_SYMBOLS = $(UMP_PATH_RELATIVE)/src/Module.symvers ifeq ($(MALI_UNIT_TEST), 1) EXTRA_SYMBOLS += $(KBASE_PATH_RELATIVE)/tests/internal/src/kernel_assert_module/linux/Module.symvers diff --git a/drivers/gpu/arm/bifrost/Mconfig b/drivers/gpu/arm/bifrost/Mconfig index 47a744ba8fcf..d9847e3dffa2 100644 --- a/drivers/gpu/arm/bifrost/Mconfig +++ b/drivers/gpu/arm/bifrost/Mconfig @@ -205,3 +205,5 @@ config MALI_PWRSOFT_765 PWRSOFT-765 fixes devfreq cooling devices issues. However, they are not merged in mainline kernel yet. So this define helps to guard those parts of the code. + +source "kernel/drivers/gpu/arm/midgard/tests/Mconfig" diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c index f9605fc1ae53..0ea045c23f3e 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c @@ -171,8 +171,6 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) KBASE_TLSTREAM_AUX_DEVFREQ_TARGET((u64)nominal_freq); - kbase_pm_reset_dvfs_utilisation(kbdev); - return err; } @@ -190,10 +188,13 @@ static int kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) { struct kbase_device *kbdev = dev_get_drvdata(dev); + struct kbasep_pm_metrics diff; - kbase_pm_get_dvfs_utilisation(kbdev, - &stat->total_time, &stat->busy_time); + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->last_devfreq_metrics, &diff); + stat->busy_time = diff.time_busy; + stat->total_time = diff.time_busy + diff.time_idle; + stat->current_frequency = kbdev->current_nominal_freq; stat->private_data = NULL; return 0; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c index 4c991522eaf1..c8153ba4c121 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -68,11 +68,12 @@ static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, } bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, - struct kbase_context *kctx) + struct kbase_context *kctx, + int js) { int i; - if (kbdev->hwaccess.active_kctx == kctx) { + if (kbdev->hwaccess.active_kctx[js] == kctx) { /* Context is already active */ return true; } @@ -213,12 +214,15 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev, { struct kbasep_js_device_data *js_devdata; struct kbase_as *new_address_space = NULL; + int js; js_devdata = &kbdev->js_data; - if (kbdev->hwaccess.active_kctx == kctx) { - WARN(1, "Context is already scheduled in\n"); - return false; + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + if (kbdev->hwaccess.active_kctx[js] == kctx) { + WARN(1, "Context is already scheduled in\n"); + return false; + } } new_address_space = &kbdev->as[as_nr]; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h index 36492c68e1a2..2b3b1f864ccc 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -71,58 +71,67 @@ enum kbase_pm_core_type { }; /** - * struct kbasep_pm_metrics_data - Metrics data collected for use by the power - * management framework. + * struct kbasep_pm_metrics - Metrics data collected for use by the power + * management framework. * - * @time_period_start: time at which busy/idle measurements started * @time_busy: number of ns the GPU was busy executing jobs since the * @time_period_start timestamp. * @time_idle: number of ns since time_period_start the GPU was not executing * jobs since the @time_period_start timestamp. - * @prev_busy: busy time in ns of previous time period. - * Updated when metrics are reset. - * @prev_idle: idle time in ns of previous time period - * Updated when metrics are reset. - * @gpu_active: true when the GPU is executing jobs. false when - * not. Updated when the job scheduler informs us a job in submitted - * or removed from a GPU slot. * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that * if two CL jobs were active for 400ns, this value would be updated * with 800. * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that * if two GL jobs were active for 400ns, this value would be updated * with 800. + */ +struct kbasep_pm_metrics { + u32 time_busy; + u32 time_idle; + u32 busy_cl[2]; + u32 busy_gl; +}; + +/** + * struct kbasep_pm_metrics_state - State required to collect the metrics in + * struct kbasep_pm_metrics + * @time_period_start: time at which busy/idle measurements started + * @gpu_active: true when the GPU is executing jobs. false when + * not. Updated when the job scheduler informs us a job in submitted + * or removed from a GPU slot. * @active_cl_ctx: number of CL jobs active on the GPU. Array is per-device. * @active_gl_ctx: number of GL jobs active on the GPU. Array is per-slot. As * GL jobs never run on slot 2 this slot is not recorded. * @lock: spinlock protecting the kbasep_pm_metrics_data structure + * @platform_data: pointer to data controlled by platform specific code + * @kbdev: pointer to kbase device for which metrics are collected + * @values: The current values of the power management metrics. The + * kbase_pm_get_dvfs_metrics() function is used to compare these + * current values with the saved values from a previous invocation. * @timer: timer to regularly make DVFS decisions based on the power * management metrics. * @timer_active: boolean indicating @timer is running - * @platform_data: pointer to data controlled by platform specific code - * @kbdev: pointer to kbase device for which metrics are collected - * + * @dvfs_last: values of the PM metrics from the last DVFS tick + * @dvfs_diff: different between the current and previous PM metrics. */ -struct kbasep_pm_metrics_data { +struct kbasep_pm_metrics_state { ktime_t time_period_start; - u32 time_busy; - u32 time_idle; - u32 prev_busy; - u32 prev_idle; bool gpu_active; - u32 busy_cl[2]; - u32 busy_gl; u32 active_cl_ctx[2]; u32 active_gl_ctx[2]; /* GL jobs can only run on 2 of the 3 job slots */ spinlock_t lock; + void *platform_data; + struct kbase_device *kbdev; + + struct kbasep_pm_metrics values; + #ifdef CONFIG_MALI_BIFROST_DVFS struct hrtimer timer; bool timer_active; + struct kbasep_pm_metrics dvfs_last; + struct kbasep_pm_metrics dvfs_diff; #endif - - void *platform_data; - struct kbase_device *kbdev; }; union kbase_pm_policy_data { @@ -291,7 +300,7 @@ struct kbase_pm_backend_data { spinlock_t gpu_powered_lock; - struct kbasep_pm_metrics_data metrics; + struct kbasep_pm_metrics_state metrics; int gpu_poweroff_pending; int shader_poweroff_pending_time; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h index d7afc6173a2d..624efc833f98 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -507,9 +507,9 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume); void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend); #if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) -void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, - unsigned long *total, unsigned long *busy); -void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev); +void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, + struct kbasep_pm_metrics *last, + struct kbasep_pm_metrics *diff); #endif /* defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) */ #ifdef CONFIG_MALI_BIFROST_DVFS diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c index a7781c5cccab..aabd6541402c 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,6 +30,7 @@ #include <mali_kbase_pm.h> #include <backend/gpu/mali_kbase_pm_internal.h> #include <backend/gpu/mali_kbase_jm_rb.h> +#include <backend/gpu/mali_kbase_pm_defs.h> /* When VSync is being hit aim for utilisation between 70-90% */ #define KBASE_PM_VSYNC_MIN_UTILISATION 70 @@ -43,19 +44,15 @@ * under 11s. Exceeding this will cause overflow */ #define KBASE_PM_TIME_SHIFT 8 -/* Maximum time between sampling of utilization data, without resetting the - * counters. */ -#define MALI_UTILIZATION_MAX_PERIOD 100000 /* ns = 100ms */ - #ifdef CONFIG_MALI_BIFROST_DVFS static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) { unsigned long flags; - struct kbasep_pm_metrics_data *metrics; + struct kbasep_pm_metrics_state *metrics; KBASE_DEBUG_ASSERT(timer != NULL); - metrics = container_of(timer, struct kbasep_pm_metrics_data, timer); + metrics = container_of(timer, struct kbasep_pm_metrics_state, timer); kbase_pm_get_dvfs_action(metrics->kbdev); spin_lock_irqsave(&metrics->lock, flags); @@ -78,18 +75,17 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) kbdev->pm.backend.metrics.kbdev = kbdev; kbdev->pm.backend.metrics.time_period_start = ktime_get(); - kbdev->pm.backend.metrics.time_busy = 0; - kbdev->pm.backend.metrics.time_idle = 0; - kbdev->pm.backend.metrics.prev_busy = 0; - kbdev->pm.backend.metrics.prev_idle = 0; kbdev->pm.backend.metrics.gpu_active = false; kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; kbdev->pm.backend.metrics.active_gl_ctx[0] = 0; kbdev->pm.backend.metrics.active_gl_ctx[1] = 0; - kbdev->pm.backend.metrics.busy_cl[0] = 0; - kbdev->pm.backend.metrics.busy_cl[1] = 0; - kbdev->pm.backend.metrics.busy_gl = 0; + + kbdev->pm.backend.metrics.values.time_busy = 0; + kbdev->pm.backend.metrics.values.time_idle = 0; + kbdev->pm.backend.metrics.values.busy_cl[0] = 0; + kbdev->pm.backend.metrics.values.busy_cl[1] = 0; + kbdev->pm.backend.metrics.values.busy_gl = 0; spin_lock_init(&kbdev->pm.backend.metrics.lock); @@ -143,17 +139,17 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, if (kbdev->pm.backend.metrics.gpu_active) { u32 ns_time = (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); - kbdev->pm.backend.metrics.time_busy += ns_time; + kbdev->pm.backend.metrics.values.time_busy += ns_time; if (kbdev->pm.backend.metrics.active_cl_ctx[0]) - kbdev->pm.backend.metrics.busy_cl[0] += ns_time; + kbdev->pm.backend.metrics.values.busy_cl[0] += ns_time; if (kbdev->pm.backend.metrics.active_cl_ctx[1]) - kbdev->pm.backend.metrics.busy_cl[1] += ns_time; + kbdev->pm.backend.metrics.values.busy_cl[1] += ns_time; if (kbdev->pm.backend.metrics.active_gl_ctx[0]) - kbdev->pm.backend.metrics.busy_gl += ns_time; + kbdev->pm.backend.metrics.values.busy_gl += ns_time; if (kbdev->pm.backend.metrics.active_gl_ctx[1]) - kbdev->pm.backend.metrics.busy_gl += ns_time; + kbdev->pm.backend.metrics.values.busy_gl += ns_time; } else { - kbdev->pm.backend.metrics.time_idle += (u32) (ktime_to_ns(diff) + kbdev->pm.backend.metrics.values.time_idle += (u32) (ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); } @@ -161,160 +157,53 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, } #if defined(CONFIG_MALI_BIFROST_DEVFREQ) || defined(CONFIG_MALI_BIFROST_DVFS) -/* Caller needs to hold kbdev->pm.backend.metrics.lock before calling this - * function. - */ -static void kbase_pm_reset_dvfs_utilisation_unlocked(struct kbase_device *kbdev, - ktime_t now) -{ - /* Store previous value */ - kbdev->pm.backend.metrics.prev_idle = - kbdev->pm.backend.metrics.time_idle; - kbdev->pm.backend.metrics.prev_busy = - kbdev->pm.backend.metrics.time_busy; - - /* Reset current values */ - kbdev->pm.backend.metrics.time_period_start = now; - kbdev->pm.backend.metrics.time_idle = 0; - kbdev->pm.backend.metrics.time_busy = 0; - kbdev->pm.backend.metrics.busy_cl[0] = 0; - kbdev->pm.backend.metrics.busy_cl[1] = 0; - kbdev->pm.backend.metrics.busy_gl = 0; -} - -void kbase_pm_reset_dvfs_utilisation(struct kbase_device *kbdev) +void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, + struct kbasep_pm_metrics *last, + struct kbasep_pm_metrics *diff) { + struct kbasep_pm_metrics *cur = &kbdev->pm.backend.metrics.values; unsigned long flags; spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, ktime_get()); - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -} + kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get()); -void kbase_pm_get_dvfs_utilisation(struct kbase_device *kbdev, - unsigned long *total_out, unsigned long *busy_out) -{ - ktime_t now = ktime_get(); - unsigned long flags, busy, total; + memset(diff, 0, sizeof(*diff)); + diff->time_busy = cur->time_busy - last->time_busy; + diff->time_idle = cur->time_idle - last->time_idle; + diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; + diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; + diff->busy_gl = cur->busy_gl - last->busy_gl; - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - kbase_pm_get_dvfs_utilisation_calc(kbdev, now); - - busy = kbdev->pm.backend.metrics.time_busy; - total = busy + kbdev->pm.backend.metrics.time_idle; - - /* Reset stats if older than MALI_UTILIZATION_MAX_PERIOD (default - * 100ms) */ - if (total >= MALI_UTILIZATION_MAX_PERIOD) { - kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); - } else if (total < (MALI_UTILIZATION_MAX_PERIOD / 2)) { - total += kbdev->pm.backend.metrics.prev_idle + - kbdev->pm.backend.metrics.prev_busy; - busy += kbdev->pm.backend.metrics.prev_busy; - } + *last = *cur; - *total_out = total; - *busy_out = busy; spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); } +KBASE_EXPORT_TEST_API(kbase_pm_get_dvfs_metrics); #endif #ifdef CONFIG_MALI_BIFROST_DVFS - -/* caller needs to hold kbdev->pm.backend.metrics.lock before calling this - * function - */ -int kbase_pm_get_dvfs_utilisation_old(struct kbase_device *kbdev, - int *util_gl_share, - int util_cl_share[2], - ktime_t now) -{ - int utilisation; - int busy; - - kbase_pm_get_dvfs_utilisation_calc(kbdev, now); - - if (kbdev->pm.backend.metrics.time_idle + - kbdev->pm.backend.metrics.time_busy == 0) { - /* No data - so we return NOP */ - utilisation = -1; - if (util_gl_share) - *util_gl_share = -1; - if (util_cl_share) { - util_cl_share[0] = -1; - util_cl_share[1] = -1; - } - goto out; - } - - utilisation = (100 * kbdev->pm.backend.metrics.time_busy) / - (kbdev->pm.backend.metrics.time_idle + - kbdev->pm.backend.metrics.time_busy); - - busy = kbdev->pm.backend.metrics.busy_gl + - kbdev->pm.backend.metrics.busy_cl[0] + - kbdev->pm.backend.metrics.busy_cl[1]; - - if (busy != 0) { - if (util_gl_share) - *util_gl_share = - (100 * kbdev->pm.backend.metrics.busy_gl) / - busy; - if (util_cl_share) { - util_cl_share[0] = - (100 * kbdev->pm.backend.metrics.busy_cl[0]) / - busy; - util_cl_share[1] = - (100 * kbdev->pm.backend.metrics.busy_cl[1]) / - busy; - } - } else { - if (util_gl_share) - *util_gl_share = -1; - if (util_cl_share) { - util_cl_share[0] = -1; - util_cl_share[1] = -1; - } - } - -out: - return utilisation; -} - void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) { - unsigned long flags; int utilisation, util_gl_share; int util_cl_share[2]; - ktime_t now; + int busy; + struct kbasep_pm_metrics *diff; KBASE_DEBUG_ASSERT(kbdev != NULL); - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); - - now = ktime_get(); + diff = &kbdev->pm.backend.metrics.dvfs_diff; - utilisation = kbase_pm_get_dvfs_utilisation_old(kbdev, &util_gl_share, - util_cl_share, now); + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, diff); - if (utilisation < 0 || util_gl_share < 0 || util_cl_share[0] < 0 || - util_cl_share[1] < 0) { - utilisation = 0; - util_gl_share = 0; - util_cl_share[0] = 0; - util_cl_share[1] = 0; - goto out; - } + utilisation = (100 * diff->time_busy) / + max(diff->time_busy + diff->time_idle, 1u); -out: -#ifdef CONFIG_MALI_BIFROST_DVFS - kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, - util_cl_share); -#endif /*CONFIG_MALI_BIFROST_DVFS */ + busy = max(diff->busy_gl + diff->busy_cl[0] + diff->busy_cl[1], 1u); + util_gl_share = (100 * diff->busy_gl) / busy; + util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; + util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; - kbase_pm_reset_dvfs_utilisation_unlocked(kbdev, now); - - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); + kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); } bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp index 936d5dca8f4a..0676a2494875 100644 --- a/drivers/gpu/arm/bifrost/build.bp +++ b/drivers/gpu/arm/bifrost/build.bp @@ -3,13 +3,17 @@ * ---------------------------------------------------------------------------- * This confidential and proprietary software may be used only as authorized * by a licensing agreement from ARM Limited. - * (C) COPYRIGHT 2017 ARM Limited, ALL RIGHTS RESERVED + * (C) COPYRIGHT 2017-2018 ARM Limited, ALL RIGHTS RESERVED * The entire notice above must be reproduced on all authorized copies and * copies may only be made to the extent permitted by a licensing agreement * from ARM Limited. * ---------------------------------------------------------------------------- */ +/* Kernel-side tests may include mali_kbase's headers. Therefore any config + * options which affect the sizes of any structs (e.g. adding extra members) + * must be included in these defaults, so that the structs are consistent in + * both mali_kbase and the test modules. */ bob_defaults { name: "mali_kbase_shared_config_defaults", no_mali: { @@ -21,7 +25,32 @@ bob_defaults { mali_devfreq: { kbuild_options: ["CONFIG_MALI_BIFROST_DEVFREQ=y"], }, - + mali_midgard_dvfs: { + kbuild_options: ["CONFIG_MALI_BIFROST_DVFS=y"], + }, + mali_trace_timeline: { + kbuild_options: ["CONFIG_MALI_BIFROST_TRACE_TIMELINE=y"], + }, + mali_debug: { + kbuild_options: ["CONFIG_MALI_BIFROST_DEBUG=y"], + }, + mali_fpga_bus_logger: { + kbuild_options: ["CONFIG_MALI_FPGA_BUS_LOGGER=y"], + }, + cinstr_job_dump: { + kbuild_options: ["CONFIG_MALI_JOB_DUMP=y"], + }, + mali_gator_support: { + kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"], + }, + mali_system_trace: { + kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"], + }, + kbuild_options: [ + "MALI_UNIT_TEST={{.unit_test_code}}", + "MALI_CUSTOMER_RELEASE={{.release}}", + "MALI_KERNEL_TEST_API={{.debug}}", + ], defaults: ["kernel_defaults"], } @@ -50,46 +79,23 @@ bob_kernel_module { "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}", "MALI_KERNEL_TEST_API={{.unit_test_code}}", "MALI_MOCK_TEST={{.mali_mock_test}}", - "MALI_UNIT_TEST={{.unit_test_code}}", ], - cinstr_job_dump: { - kbuild_options: ["CONFIG_MALI_JOB_DUMP=y"], - }, - mali_debug: { - kbuild_options: ["CONFIG_MALI_BIFROST_DEBUG=y"], - }, - mali_gator_support: { - kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"], - }, - mali_system_trace: { - kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"], - }, mali_error_inject: { kbuild_options: ["CONFIG_MALI_BIFROST_ERROR_INJECT=y"], }, mali_error_inject_random: { kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"], }, - mali_trace_timeline: { - kbuild_options: ["CONFIG_MALI_BIFROST_TRACE_TIMELINE=y"], - }, mali_prfcnt_set_secondary: { kbuild_options: ["CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY=y"], }, - mali_fpga_bus_logger: { - kbuild_options: ["CONFIG_MALI_FPGA_BUS_LOGGER=y"], - }, - mali_midgard_dvfs: { - kbuild_options: ["CONFIG_MALI_BIFROST_DVFS=y"], - }, mali_2mb_alloc: { kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"], }, mali_mock_test: { srcs: ["tests/internal/src/mock/mali_kbase_pm_driver_mock.c"], }, - ump: { - extra_symbols: ["ump"], - }, defaults: ["mali_kbase_shared_config_defaults"], } + +optional_subdirs = ["tests"] diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c index d14e1e6d01fa..23ac2ec16dfb 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.c @@ -26,6 +26,7 @@ #include "mali_kbase_ipa.h" #include "mali_kbase_ipa_debugfs.h" #include "mali_kbase_ipa_simple.h" +#include "backend/gpu/mali_kbase_pm_internal.h" #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) #include <linux/pm_opp.h> @@ -318,14 +319,6 @@ int kbase_ipa_init(struct kbase_device *kbdev) /* The simple IPA model must *always* be present.*/ ops = kbase_ipa_model_ops_find(kbdev, KBASE_IPA_FALLBACK_MODEL_NAME); - if (!ops->do_utilization_scaling_in_framework) { - dev_err(kbdev->dev, - "Fallback IPA model %s should not account for utilization\n", - ops->name); - err = -EINVAL; - goto end; - } - default_model = kbase_ipa_init_model(kbdev, ops); if (!default_model) { err = -EINVAL; @@ -545,7 +538,7 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, model = kbdev->ipa.fallback_model; - err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + err = model->ops->get_dynamic_coeff(model, &power_coeff); if (!err) power = kbase_scale_dynamic_power(power_coeff, freq, voltage); @@ -564,48 +557,63 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, return power; } -int kbase_get_real_power(struct devfreq *df, u32 *power, +int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, unsigned long freq, unsigned long voltage) { struct kbase_ipa_model *model; u32 power_coeff = 0; int err = 0; - struct kbase_device *kbdev = dev_get_drvdata(&df->dev); + struct kbasep_pm_metrics diff; + u64 total_time; - mutex_lock(&kbdev->ipa.lock); + lockdep_assert_held(&kbdev->ipa.lock); + + kbase_pm_get_dvfs_metrics(kbdev, &kbdev->ipa.last_metrics, &diff); model = get_current_model(kbdev); - err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + err = model->ops->get_dynamic_coeff(model, &power_coeff); - /* If we switch to protected model between get_current_model() and - * get_dynamic_coeff(), counter reading could fail. If that happens - * (unlikely, but possible), revert to the fallback model. */ + /* If the counter model returns an error (e.g. switching back to + * protected mode and failing to read counters, or a counter sample + * with too few cycles), revert to the fallback model. + */ if (err && model != kbdev->ipa.fallback_model) { model = kbdev->ipa.fallback_model; - err = model->ops->get_dynamic_coeff(model, &power_coeff, freq); + err = model->ops->get_dynamic_coeff(model, &power_coeff); } if (err) - goto exit_unlock; + return err; *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); - if (model->ops->do_utilization_scaling_in_framework) { - struct devfreq_dev_status *status = &df->last_status; - unsigned long total_time = max(status->total_time, 1ul); - u64 busy_time = min(status->busy_time, total_time); - - *power = div_u64((u64) *power * (u64) busy_time, total_time); - } + /* time_busy / total_time cannot be >1, so assigning the 64-bit + * result of div_u64 to *power cannot overflow. + */ + total_time = diff.time_busy + (u64) diff.time_idle; + *power = div_u64(*power * (u64) diff.time_busy, + max(total_time, 1ull)); *power += get_static_power_locked(kbdev, model, voltage); -exit_unlock: + return err; +} +KBASE_EXPORT_TEST_API(kbase_get_real_power_locked); + +int kbase_get_real_power(struct devfreq *df, u32 *power, + unsigned long freq, + unsigned long voltage) +{ + int ret; + struct kbase_device *kbdev = dev_get_drvdata(&df->dev); + + mutex_lock(&kbdev->ipa.lock); + ret = kbase_get_real_power_locked(kbdev, power, freq, voltage); mutex_unlock(&kbdev->ipa.lock); - return err; + return ret; } KBASE_EXPORT_TEST_API(kbase_get_real_power); diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h index fc45f22d6a6a..844723a74bda 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa.h @@ -96,8 +96,6 @@ struct kbase_ipa_model_ops { * get_dynamic_coeff() - calculate dynamic power coefficient * @model: pointer to model * @coeffp: pointer to return value location - * @current_freq: frequency the GPU has been running at for the - * previous sampling period. * * Calculate a dynamic power coefficient, with units pW/(Hz V^2), which * is then scaled by the IPA framework according to the current OPP's @@ -105,8 +103,7 @@ struct kbase_ipa_model_ops { * * Return: 0 on success, or an error code. */ - int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp, - u32 current_freq); + int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp); /* * get_static_coeff() - calculate static power coefficient * @model: pointer to model @@ -118,11 +115,6 @@ struct kbase_ipa_model_ops { * Return: 0 on success, or an error code. */ int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); - /* If false, the model's get_dynamic_coeff() method accounts for how - * long the GPU was active over the sample period. If true, the - * framework will scale the calculated power according to the - * utilization stats recorded by devfreq in get_real_power(). */ - bool do_utilization_scaling_in_framework; }; /** @@ -208,6 +200,13 @@ extern struct kbase_ipa_model_ops kbase_tnox_ipa_model_ops; int kbase_get_real_power(struct devfreq *df, u32 *power, unsigned long freq, unsigned long voltage); + +/* Called by kbase_get_real_power() to invoke the power models. + * Must be called with kbdev->ipa.lock held. + */ +int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, + unsigned long freq, + unsigned long voltage); #endif /* MALI_UNIT_TEST */ #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c index 7dd2ae252112..e684df4a6662 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_simple.c @@ -203,8 +203,7 @@ static int model_static_coeff(struct kbase_ipa_model *model, u32 *coeffp) return 0; } -static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, - u32 current_freq) +static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) { struct kbase_ipa_model_simple_data *model_data = (struct kbase_ipa_model_simple_data *) model->model_data; @@ -347,6 +346,5 @@ struct kbase_ipa_model_ops kbase_simple_ipa_model_ops = { .term = &kbase_simple_power_model_term, .get_dynamic_coeff = &model_dynamic_coeff, .get_static_coeff = &model_static_coeff, - .do_utilization_scaling_in_framework = true, }; KBASE_EXPORT_TEST_API(kbase_simple_ipa_model_ops); diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c index 3917fb8d0ef7..4019657f41a3 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.c @@ -23,34 +23,13 @@ #include "mali_kbase_ipa_vinstr_common.h" #include "mali_kbase_ipa_debugfs.h" -#if MALI_UNIT_TEST -static ktime_t dummy_time; +#define DEFAULT_SCALING_FACTOR 5 -/* Intercept calls to the kernel function using a macro */ -#ifdef ktime_get -#undef ktime_get -#endif - -#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) -#define ktime_get() (READ_ONCE(dummy_time)) - -void kbase_ipa_set_dummy_time(ktime_t t) -{ - WRITE_ONCE(dummy_time, t); -} -KBASE_EXPORT_TEST_API(kbase_ipa_set_dummy_time); -#else -#define ktime_get() (READ_ONCE(dummy_time)) - -void kbase_ipa_set_dummy_time(ktime_t t) -{ - WRITE_ONCE(dummy_time, t); -} -KBASE_EXPORT_TEST_API(kbase_ipa_set_dummy_time); - -#endif - -#endif /* MALI_UNIT_TEST */ +/* If the value of GPU_ACTIVE is below this, use the simple model + * instead, to avoid extrapolating small amounts of counter data across + * large sample periods. + */ +#define DEFAULT_MIN_SAMPLE_CYCLES 10000 /** * read_hwcnt() - read a counter value @@ -100,10 +79,8 @@ s64 kbase_ipa_sum_all_shader_cores( core_mask >>= 1; } - /* Range: -2^54 < ret < 2^54 */ - ret *= coeff; - - return div_s64(ret, 1000000); + /* Range: -2^54 < ret * coeff < 2^54 */ + return ret * coeff; } s64 kbase_ipa_single_counter( @@ -114,10 +91,7 @@ s64 kbase_ipa_single_counter( const u32 counter_value = kbase_ipa_read_hwcnt(model_data, counter); /* Range: -2^49 < ret < 2^49 */ - const s64 multiplied = (s64) counter_value * (s64) coeff; - - /* Range: -2^29 < return < 2^29 */ - return div_s64(multiplied, 1000000); + return counter_value * (s64) coeff; } /** @@ -182,7 +156,6 @@ int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) return -1; } - model_data->last_sample_read_time = ktime_get(); kbase_vinstr_hwc_clear(model_data->vinstr_cli); kbdev->ipa.gpu_active_callback = kbase_ipa_gpu_active; @@ -214,21 +187,15 @@ void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) model_data->vinstr_buffer = NULL; } -int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, - u32 current_freq) +int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) { struct kbase_ipa_model_vinstr_data *model_data = (struct kbase_ipa_model_vinstr_data *)model->model_data; struct kbase_device *kbdev = model_data->kbdev; s64 energy = 0; size_t i; - ktime_t now = ktime_get(); - ktime_t time_since_last_sample = - ktime_sub(now, model_data->last_sample_read_time); - /* Range: 2^0 < time_since_last_sample_ms < 2^10 (1-1000ms) */ - s64 time_since_last_sample_ms = ktime_to_ms(time_since_last_sample); - u64 coeff = 0; - u64 num_cycles; + u64 coeff = 0, coeff_mul = 0; + u32 active_cycles; int err = 0; if (!kbdev->ipa.vinstr_active) @@ -239,66 +206,82 @@ int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, if (err) goto err0; - model_data->last_sample_read_time = now; + /* Range: 0 (GPU not used at all), to the max sampling interval, say + * 1s, * max GPU frequency (GPU 100% utilized). + * 0 <= active_cycles <= 1 * ~2GHz + * 0 <= active_cycles < 2^31 + */ + active_cycles = model_data->get_active_cycles(model_data); + + if (active_cycles < (u32) max(model_data->min_sample_cycles, 0)) { + err = -ENODATA; + goto err0; + } - /* Range of 'energy' is +/- 2^34 * number of IPA groups, so around - * -2^38 < energy < 2^38 */ + /* Range: 1 <= active_cycles < 2^31 */ + active_cycles = max(1u, active_cycles); + + /* Range of 'energy' is +/- 2^54 * number of IPA groups (~8), so around + * -2^57 < energy < 2^57 + */ for (i = 0; i < model_data->groups_def_num; i++) { const struct kbase_ipa_group *group = &model_data->groups_def[i]; - s32 coeff, group_energy; - - coeff = model_data->group_values[i]; - group_energy = group->op(model_data, coeff, group->counter_block_offset); + s32 coeff = model_data->group_values[i]; + s64 group_energy = group->op(model_data, coeff, + group->counter_block_offset); energy = kbase_ipa_add_saturate(energy, group_energy); } - /* Range: 0 <= coeff < 2^38 */ + /* Range: 0 <= coeff < 2^57 */ if (energy > 0) coeff = energy; - /* Scale by user-specified factor and divide by 1000. But actually - * cancel the division out, because we want the num_cycles in KHz and - * don't want to lose precision. */ - - /* Range: 0 < coeff < 2^53 */ - coeff = coeff * model_data->scaling_factor; - - if (time_since_last_sample_ms == 0) { - time_since_last_sample_ms = 1; - } else if (time_since_last_sample_ms < 0) { - err = -ERANGE; - goto err0; - } - - /* Range: 2^20 < num_cycles < 2^40 mCycles */ - num_cycles = (u64) current_freq * (u64) time_since_last_sample_ms; - /* Range: 2^10 < num_cycles < 2^30 Cycles */ - num_cycles = div_u64(num_cycles, 1000000); + /* Range: 0 <= coeff < 2^57 (because active_cycles >= 1). However, this + * can be constrained further: Counter values can only be increased by + * a theoretical maximum of about 64k per clock cycle. Beyond this, + * we'd have to sample every 1ms to avoid them overflowing at the + * lowest clock frequency (say 100MHz). Therefore, we can write the + * range of 'coeff' in terms of active_cycles: + * + * coeff = SUM(coeffN * counterN * num_cores_for_counterN) + * coeff <= SUM(coeffN * counterN) * max_num_cores + * coeff <= num_IPA_groups * max_coeff * max_counter * max_num_cores + * (substitute max_counter = 2^16 * active_cycles) + * coeff <= num_IPA_groups * max_coeff * 2^16 * active_cycles * max_num_cores + * coeff <= 2^3 * 2^22 * 2^16 * active_cycles * 2^5 + * coeff <= 2^46 * active_cycles + * + * So after the division: 0 <= coeff <= 2^46 + */ + coeff = div_u64(coeff, active_cycles); - /* num_cycles should never be 0 in _normal_ usage (because we expect - * frequencies on the order of MHz and >10ms polling intervals), but - * protect against divide-by-zero anyway. */ - if (num_cycles == 0) - num_cycles = 1; + /* Scale by user-specified factor (where unity is 1000). + * Range: 0 <= coeff_mul < 2^61 + */ + coeff_mul = coeff * model_data->scaling_factor; - /* Range: 0 < coeff < 2^43 */ - coeff = div_u64(coeff, num_cycles); + /* Range: 0 <= coeff_mul < 2^51 */ + coeff_mul = div_u64(coeff_mul, 1000u); err0: /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ - *coeffp = clamp(coeff, (u64) 0, (u64) 1 << 16); + *coeffp = clamp(coeff_mul, (u64) 0, (u64) 1 << 16); return err; } int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size) + const struct kbase_ipa_group *ipa_groups_def, + size_t ipa_group_size, + kbase_ipa_get_active_cycles_callback get_active_cycles) { int err = 0; size_t i; struct kbase_ipa_model_vinstr_data *model_data; + if (!model || !ipa_groups_def || !ipa_group_size || !get_active_cycles) + return -EINVAL; + model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); if (!model_data) return -ENOMEM; @@ -306,6 +289,7 @@ int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, model_data->kbdev = model->kbdev; model_data->groups_def = ipa_groups_def; model_data->groups_def_num = ipa_group_size; + model_data->get_active_cycles = get_active_cycles; model->model_data = (void *) model_data; @@ -320,13 +304,20 @@ int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, goto exit; } - model_data->scaling_factor = 5; + model_data->scaling_factor = DEFAULT_SCALING_FACTOR; err = kbase_ipa_model_add_param_s32(model, "scale", &model_data->scaling_factor, 1, false); if (err) goto exit; + model_data->min_sample_cycles = DEFAULT_MIN_SAMPLE_CYCLES; + err = kbase_ipa_model_add_param_s32(model, "min_sample_cycles", + &model_data->min_sample_cycles, + 1, false); + if (err) + goto exit; + err = kbase_ipa_attach_vinstr(model_data); exit: diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h index 18c30fe701a0..c9288e8c01f7 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_common.h @@ -38,28 +38,41 @@ #define KBASE_IPA_NR_BYTES_PER_BLOCK \ (KBASE_IPA_NR_CNT_PER_BLOCK * KBASE_IPA_NR_BYTES_PER_CNT) +struct kbase_ipa_model_vinstr_data; + +typedef u32 (*kbase_ipa_get_active_cycles_callback)(struct kbase_ipa_model_vinstr_data *); + /** * struct kbase_ipa_model_vinstr_data - IPA context per device * @kbdev: pointer to kbase device * @groups_def: Array of IPA groups. * @groups_def_num: Number of elements in the array of IPA groups. + * @get_active_cycles: Callback to return number of active cycles during + * counter sample period * @vinstr_cli: vinstr client handle * @vinstr_buffer: buffer to dump hardware counters onto - * @last_sample_read_time: timestamp of last vinstr buffer read * @scaling_factor: user-specified power scaling factor. This is * interpreted as a fraction where the denominator is * 1000. Range approx 0.0-32.0: * 0 < scaling_factor < 2^15 + * @min_sample_cycles: If the value of the GPU_ACTIVE counter (the number of + * cycles the GPU was working) is less than + * min_sample_cycles, the counter model will return an + * error, causing the IPA framework to approximate using + * the cached simple model results instead. This may be + * more accurate than extrapolating using a very small + * counter dump. */ struct kbase_ipa_model_vinstr_data { struct kbase_device *kbdev; s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; const struct kbase_ipa_group *groups_def; size_t groups_def_num; + kbase_ipa_get_active_cycles_callback get_active_cycles; struct kbase_vinstr_client *vinstr_cli; void *vinstr_buffer; - ktime_t last_sample_read_time; s32 scaling_factor; + s32 min_sample_cycles; }; /** @@ -88,7 +101,7 @@ struct kbase_ipa_group { * Calculate energy estimation based on hardware counter `counter' * across all shader cores. * - * Return: Sum of counter values. Range: -2^34 < ret < 2^34 + * Return: Sum of counter values. Range: -2^54 < ret < 2^54 */ s64 kbase_ipa_sum_all_shader_cores( struct kbase_ipa_model_vinstr_data *model_data, @@ -103,7 +116,7 @@ s64 kbase_ipa_sum_all_shader_cores( * * Calculate energy estimation based on hardware counter `counter'. * - * Return: Counter value. Range: -2^34 < ret < 2^34 + * Return: Counter value. Range: -2^49 < ret < 2^49 */ s64 kbase_ipa_single_counter( struct kbase_ipa_model_vinstr_data *model_data, @@ -134,9 +147,6 @@ void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); * @model: pointer to instantiated model * @coeffp: pointer to location where calculated power, in * pW/(Hz V^2), is stored. - * @current_freq: frequency the GPU has been running at over the sample - * period. In Hz. Range: 10 MHz < 1GHz, - * 2^20 < current_freq < 2^30 * * This is a GPU-agnostic implementation of the get_dynamic_coeff() * function of an IPA model. It relies on the model being populated @@ -144,8 +154,7 @@ void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); * * Return: 0 on success, or an error code. */ -int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, - u32 current_freq); +int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); /** * kbase_ipa_vinstr_common_model_init() - initialize ipa power model @@ -153,6 +162,8 @@ int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, * @ipa_groups_def: array of ipa groups which sets coefficients for * the corresponding counters used in the ipa model * @ipa_group_size: number of elements in the array @ipa_groups_def + * @get_active_cycles: callback to return the number of cycles the GPU was + * active during the counter sample period. * * This initialization function performs initialization steps common * for ipa models based on counter values. In each call, the model @@ -162,8 +173,9 @@ int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp, * Return: 0 on success, error code otherwise */ int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size); + const struct kbase_ipa_group *ipa_groups_def, + size_t ipa_group_size, + kbase_ipa_get_active_cycles_callback get_active_cycles); /** * kbase_ipa_vinstr_common_model_term() - terminate ipa power model @@ -174,17 +186,4 @@ int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, */ void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model); -#if MALI_UNIT_TEST -/** - * kbase_ipa_set_dummy_time() - set a dummy monotonic time value - * @t: a monotonic time value - * - * This is only intended for use in unit tests, to ensure that the kernel time - * values used by a power model are predictable. Deterministic behavior is - * necessary to allow validation of the dynamic power values computed by the - * model. - */ -void kbase_ipa_set_dummy_time(ktime_t t); -#endif /* MALI_UNIT_TEST */ - #endif /* _KBASE_IPA_VINSTR_COMMON_H_ */ diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c index 81137ea80454..7951b7475a18 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_vinstr_g7x.c @@ -164,6 +164,24 @@ static s64 kbase_g7x_jm_single_counter( return kbase_ipa_single_counter(model_data, coeff, counter); } +/** + * get_active_cycles() - return the GPU_ACTIVE counter + * @model_data: pointer to GPU model data. + * + * Return: the number of cycles the GPU was active during the counter sampling + * period. + */ +static u32 kbase_g7x_get_active_cycles( + struct kbase_ipa_model_vinstr_data *model_data) +{ + u32 counter = kbase_g7x_power_model_get_jm_counter(model_data, JM_GPU_ACTIVE); + + /* Counters are only 32-bit, so we can safely multiply by 1 then cast + * the 64-bit result back to a u32. + */ + return kbase_ipa_single_counter(model_data, 1, counter); +} + /** Table of IPA group definitions. * * For each IPA group, this table defines a function to access the given performance block counter (or counters, @@ -277,14 +295,14 @@ static const struct kbase_ipa_group ipa_groups_def_tnox[] = { KBASE_IPA_MAX_GROUP_DEF_NUM); \ return kbase_ipa_vinstr_common_model_init(model, \ ipa_groups_def_ ## gpu, \ - ARRAY_SIZE(ipa_groups_def_ ## gpu)); \ + ARRAY_SIZE(ipa_groups_def_ ## gpu), \ + kbase_g7x_get_active_cycles); \ } \ struct kbase_ipa_model_ops kbase_ ## gpu ## _ipa_model_ops = { \ .name = "mali-" #gpu "-power-model", \ .init = kbase_ ## gpu ## _power_model_init, \ .term = kbase_ipa_vinstr_common_model_term, \ .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ - .do_utilization_scaling_in_framework = false \ }; \ KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h index d3cfdbee5921..bcd6c5ffbb25 100644 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +++ b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h @@ -123,6 +123,7 @@ enum base_hw_issue { GPUCORE_1619, BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TMIX_8438, + BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_END }; @@ -1123,6 +1124,7 @@ static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_END }; @@ -1138,6 +1140,7 @@ static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, + BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_END }; diff --git a/drivers/gpu/arm/bifrost/mali_base_kernel.h b/drivers/gpu/arm/bifrost/mali_base_kernel.h index f0ccd789e81c..e53528e29fe0 100644 --- a/drivers/gpu/arm/bifrost/mali_base_kernel.h +++ b/drivers/gpu/arm/bifrost/mali_base_kernel.h @@ -242,7 +242,6 @@ typedef u32 base_mem_alloc_flags; * enum base_mem_import_type - Memory types supported by @a base_mem_import * * @BASE_MEM_IMPORT_TYPE_INVALID: Invalid type - * @BASE_MEM_IMPORT_TYPE_UMP: UMP import. Handle type is ump_secure_id. * @BASE_MEM_IMPORT_TYPE_UMM: UMM import. Handle type is a file descriptor (int) * @BASE_MEM_IMPORT_TYPE_USER_BUFFER: User buffer import. Handle is a * base_mem_import_user_buffer @@ -257,7 +256,9 @@ typedef u32 base_mem_alloc_flags; */ typedef enum base_mem_import_type { BASE_MEM_IMPORT_TYPE_INVALID = 0, - BASE_MEM_IMPORT_TYPE_UMP = 1, + /** + * Import type with value 1 is deprecated. + */ BASE_MEM_IMPORT_TYPE_UMM = 2, BASE_MEM_IMPORT_TYPE_USER_BUFFER = 3 } base_mem_import_type; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h index a95736c54a29..9d918a804b2f 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -151,12 +151,6 @@ enum { #define DEFAULT_3BIT_AWID_LIMIT KBASE_3BIT_AID_32 /** - * Default UMP device mapping. A UMP_DEVICE_<device>_SHIFT value which - * defines which UMP device this GPU should be mapped to. - */ -#define DEFAULT_UMP_GPU_DEVICE_SHIFT UMP_DEVICE_Z_SHIFT - -/** * Default period for DVFS sampling */ #define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 21fd399f58da..2400c5b2a386 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -69,7 +69,7 @@ #include <linux/uaccess.h> #include <linux/interrupt.h> #include <linux/mm.h> -#include <linux/compat.h> /* is_compat_task */ +#include <linux/compat.h> /* is_compat_task/in_compat_syscall */ #include <linux/mman.h> #include <linux/version.h> #include <mali_kbase_hw.h> @@ -396,7 +396,11 @@ static int kbase_open(struct inode *inode, struct file *filp) if (!kbdev) return -ENODEV; +#if (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE) + kctx = kbase_create_context(kbdev, in_compat_syscall()); +#else kctx = kbase_create_context(kbdev, is_compat_task()); +#endif /* (KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE) */ if (!kctx) { ret = -ENOMEM; goto out; @@ -2408,6 +2412,10 @@ static ssize_t kbase_show_gpuinfo(struct device *dev, .name = "Mali-G51" }, { .id = GPU_ID2_PRODUCT_TNOX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, .name = "Mali-TNOx" }, + { .id = GPU_ID2_PRODUCT_TDVX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G31" }, + { .id = GPU_ID2_PRODUCT_TGOX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, + .name = "Mali-G52" }, }; const char *product_name = "(Unknown Mali GPU)"; struct kbase_device *kbdev; @@ -3936,6 +3944,7 @@ static int kbase_platform_device_probe(struct platform_device *pdev) kbdev->mdev.name = kbdev->devname; kbdev->mdev.fops = &kbase_fops; kbdev->mdev.parent = get_device(kbdev->dev); + kbdev->mdev.mode = 0666; kbdev->inited_subsys |= inited_get_device; /* This needs to happen before registering the device with misc_register(), diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index db1b9abfb2bb..908293236ec4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -1535,6 +1535,7 @@ struct kbase_device { u64 current_core_mask; struct kbase_devfreq_opp *opp_table; int num_opps; + struct kbasep_pm_metrics last_devfreq_metrics; #ifdef CONFIG_DEVFREQ_THERMAL #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) struct devfreq_cooling_device *devfreq_cooling; @@ -1548,6 +1549,12 @@ struct kbase_device { struct kbase_ipa_model *configured_model; struct kbase_ipa_model *fallback_model; + /* Values of the PM utilization metrics from last time the + * power model was invoked. The utilization is calculated as + * the difference between last_metrics and the current values. + */ + struct kbasep_pm_metrics last_metrics; + /* * gpu_active_callback - Inform IPA that GPU is now active * @model_data: Pointer to model data @@ -1732,6 +1739,18 @@ struct jsctx_queue { * allocation mechanism. However, the 64-bit user-space client must still * reserve a JIT region using KBASE_IOCTL_MEM_JIT_INIT * + * @KCTX_PULLED_SINCE_ACTIVE_JS0: Set when the context has had an atom pulled + * from it for job slot 0. This is reset when the context first goes active or + * is re-activated on that slot. + * + * @KCTX_PULLED_SINCE_ACTIVE_JS1: Set when the context has had an atom pulled + * from it for job slot 1. This is reset when the context first goes active or + * is re-activated on that slot. + * + * @KCTX_PULLED_SINCE_ACTIVE_JS2: Set when the context has had an atom pulled + * from it for job slot 2. This is reset when the context first goes active or + * is re-activated on that slot. + * * All members need to be separate bits. This enum is intended for use in a * bitmask where multiple values get OR-ed together. */ @@ -1748,6 +1767,9 @@ enum kbase_context_flags { KCTX_DYING = 1U << 9, KCTX_NO_IMPLICIT_SYNC = 1U << 10, KCTX_FORCE_SAME_VA = 1U << 11, + KCTX_PULLED_SINCE_ACTIVE_JS0 = 1U << 12, + KCTX_PULLED_SINCE_ACTIVE_JS1 = 1U << 13, + KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, }; struct kbase_sub_alloc { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gator_hwcnt_names_ttrx.h b/drivers/gpu/arm/bifrost/mali_kbase_gator_hwcnt_names_ttrx.h index e8148a018df5..a17870d03b21 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gator_hwcnt_names_ttrx.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gator_hwcnt_names_ttrx.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2016-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_id.h b/drivers/gpu/arm/bifrost/mali_kbase_gpu_id.h index c14317087ead..218e63a61c6c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_id.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_id.h @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2015-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2015-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -110,7 +110,7 @@ #define GPU_ID2_PRODUCT_TNOX GPU_ID2_MODEL_MAKE(7, 1) #define GPU_ID2_PRODUCT_TGOX GPU_ID2_MODEL_MAKE(7, 2) #define GPU_ID2_PRODUCT_TKAX GPU_ID2_MODEL_MAKE(8, 0) -#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(8, 1) +#define GPU_ID2_PRODUCT_TTRX GPU_ID2_MODEL_MAKE(9, 0) #define GPU_ID2_PRODUCT_TBOX GPU_ID2_MODEL_MAKE(8, 2) /* Values for GPU_ID_VERSION_STATUS field for PRODUCT_ID GPU_ID_PI_T60X */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h index 64fa3028f4cc..124a2d9cf0c3 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_defs.h @@ -43,7 +43,7 @@ * @backend: GPU backend specific data for HW access layer */ struct kbase_hwaccess_data { - struct kbase_context *active_kctx; + struct kbase_context *active_kctx[BASE_JM_MAX_NR_SLOTS]; struct kbase_backend_data backend; }; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h index 706e60a7fe89..ea87913a7a39 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwaccess_jm.h @@ -86,6 +86,7 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev, * kbase_backend_use_ctx_sched() - Activate a context. * @kbdev: Device pointer * @kctx: Context pointer + * @js: Job slot to activate context on * * kbase_gpu_next_job() will pull atoms from the active context. * @@ -99,7 +100,7 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev, * not have an address space assigned) */ bool kbase_backend_use_ctx_sched(struct kbase_device *kbdev, - struct kbase_context *kctx); + struct kbase_context *kctx, int js); /** * kbase_backend_release_ctx_irq - Release a context from the GPU. This will diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h index 58edda5a0427..fcb9ad31e937 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_ioctl.h @@ -27,6 +27,7 @@ extern "C" { #endif +#include <asm-generic/ioctl.h> #include <linux/types.h> #define KBASE_IOCTL_TYPE 0x80 @@ -48,9 +49,11 @@ extern "C" { * 11.6: * - Added flags field to base_jit_alloc_info structure, which can be used to * specify pseudo chunked tiler alignment for JIT allocations. + * 11.7: + * - Removed UMP support */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 6 +#define BASE_UK_VERSION_MINOR 7 /** * struct kbase_ioctl_version_check - Check version compatibility with kernel diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_jm.c index 3b16e6f73fe7..da78a1670d9b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jm.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -44,7 +44,7 @@ static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, struct kbase_context *kctx; int i; - kctx = kbdev->hwaccess.active_kctx; + kctx = kbdev->hwaccess.active_kctx[js]; if (!kctx) return true; @@ -106,10 +106,14 @@ void kbase_jm_try_kick_all(struct kbase_device *kbdev) void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) { + int js; + lockdep_assert_held(&kbdev->hwaccess_lock); - if (kbdev->hwaccess.active_kctx == kctx) - kbdev->hwaccess.active_kctx = NULL; + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + if (kbdev->hwaccess.active_kctx[js] == kctx) + kbdev->hwaccess.active_kctx[js] = NULL; + } } struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c index 938eaa5ff608..0ef7cf67cd38 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_js.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -1201,7 +1201,8 @@ bool kbasep_js_add_job(struct kbase_context *kctx, } /* If this context is active and the atom is the first on its slot, * kick the job manager to attempt to fast-start the atom */ - if (enqueue_required && kctx == kbdev->hwaccess.active_kctx) + if (enqueue_required && kctx == + kbdev->hwaccess.active_kctx[atom->slot_nr]) kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -1486,8 +1487,10 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( kbase_backend_release_ctx_irq(kbdev, kctx); - if (kbdev->hwaccess.active_kctx == kctx) - kbdev->hwaccess.active_kctx = NULL; + for (slot = 0; slot < num_slots; slot++) { + if (kbdev->hwaccess.active_kctx[slot] == kctx) + kbdev->hwaccess.active_kctx[slot] = NULL; + } /* Ctx Attribute handling * @@ -1679,7 +1682,8 @@ void kbase_js_set_timeouts(struct kbase_device *kbdev) } static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, - struct kbase_context *kctx) + struct kbase_context *kctx, + int js) { struct kbasep_js_device_data *js_devdata; struct kbasep_js_kctx_info *js_kctx_info; @@ -1755,7 +1759,7 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, return false; } - kbdev->hwaccess.active_kctx = kctx; + kbdev->hwaccess.active_kctx[js] = kctx; #if defined(CONFIG_MALI_BIFROST_GATOR_SUPPORT) kbase_trace_mali_mmu_as_in_use(kctx->as_nr); @@ -1788,6 +1792,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, kctx_suspended = true; } + kbase_ctx_flag_clear(kctx, KCTX_PULLED_SINCE_ACTIVE_JS0 << js); + /* Transaction complete */ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); mutex_unlock(&kbdev->mmu_hw_mutex); @@ -1812,23 +1818,27 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, } static bool kbase_js_use_ctx(struct kbase_device *kbdev, - struct kbase_context *kctx) + struct kbase_context *kctx, + int js) { unsigned long flags; spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_backend_use_ctx_sched(kbdev, kctx)) { + kbase_backend_use_ctx_sched(kbdev, kctx, js)) { /* Context already has ASID - mark as active */ - kbdev->hwaccess.active_kctx = kctx; + if (kbdev->hwaccess.active_kctx[js] != kctx) { + kbdev->hwaccess.active_kctx[js] = kctx; + kbase_ctx_flag_clear(kctx, + KCTX_PULLED_SINCE_ACTIVE_JS0 << js); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); return true; /* Context already scheduled */ } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return kbasep_js_schedule_ctx(kbdev, kctx); + return kbasep_js_schedule_ctx(kbdev, kctx, js); } void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, @@ -2200,6 +2210,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) } kbase_ctx_flag_set(kctx, KCTX_PULLED); + kbase_ctx_flag_set(kctx, (KCTX_PULLED_SINCE_ACTIVE_JS0 << js)); pulled = atomic_inc_return(&kctx->atoms_pulled); if (pulled == 1 && !kctx->slots_pullable) { @@ -2501,20 +2512,22 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, void kbase_js_sched(struct kbase_device *kbdev, int js_mask) { struct kbasep_js_device_data *js_devdata; - struct kbase_context *last_active; + struct kbase_context *last_active[BASE_JM_MAX_NR_SLOTS]; bool timer_sync = false; - bool ctx_waiting = false; + bool ctx_waiting[BASE_JM_MAX_NR_SLOTS]; + int js; js_devdata = &kbdev->js_data; down(&js_devdata->schedule_sem); mutex_lock(&js_devdata->queue_mutex); - last_active = kbdev->hwaccess.active_kctx; + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + last_active[js] = kbdev->hwaccess.active_kctx[js]; + ctx_waiting[js] = false; + } while (js_mask) { - int js; - js = ffs(js_mask) - 1; while (1) { @@ -2551,7 +2564,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) kbase_ctx_flag_set(kctx, KCTX_ACTIVE); } - if (!kbase_js_use_ctx(kbdev, kctx)) { + if (!kbase_js_use_ctx(kbdev, kctx, js)) { mutex_lock( &kctx->jctx.sched_info.ctx.jsctx_mutex); /* Context can not be used at this time */ @@ -2596,7 +2609,10 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) * Unless this context is already 'active', in * which case it's effectively already scheduled * so push it to the back of the list. */ - if (pullable && kctx == last_active) + if (pullable && kctx == last_active[js] && + kbase_ctx_flag(kctx, + (KCTX_PULLED_SINCE_ACTIVE_JS0 << + js))) timer_sync |= kbase_js_ctx_list_add_pullable_nolock( kctx->kbdev, @@ -2618,10 +2634,10 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) * marker to prevent it from submitting atoms in * the IRQ handler, which would prevent this * context from making progress. */ - if (last_active && kctx != last_active && - kbase_js_ctx_pullable( - last_active, js, true)) - ctx_waiting = true; + if (last_active[js] && kctx != last_active[js] + && kbase_js_ctx_pullable( + last_active[js], js, true)) + ctx_waiting[js] = true; if (context_idle) { kbase_jm_idle_ctx(kbdev, kctx); @@ -2661,8 +2677,11 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) if (timer_sync) kbase_js_sync_timers(kbdev); - if (kbdev->hwaccess.active_kctx == last_active && ctx_waiting) - kbdev->hwaccess.active_kctx = NULL; + for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { + if (kbdev->hwaccess.active_kctx[js] == last_active[js] && + ctx_waiting[js]) + kbdev->hwaccess.active_kctx[js] = NULL; + } mutex_unlock(&js_devdata->queue_mutex); up(&js_devdata->schedule_sem); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c index e5acab915bd2..6ce3a53a8bcf 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -29,9 +29,6 @@ #ifdef CONFIG_DMA_SHARED_BUFFER #include <linux/dma-buf.h> #endif /* CONFIG_DMA_SHARED_BUFFER */ -#ifdef CONFIG_UMP -#include <linux/ump.h> -#endif /* CONFIG_UMP */ #include <linux/kernel.h> #include <linux/bug.h> #include <linux/compat.h> @@ -1688,7 +1685,8 @@ invalid_request: struct tagged_addr *kbase_alloc_phy_pages_helper_locked( struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, - size_t nr_pages_requested) + size_t nr_pages_requested, + struct kbase_sub_alloc **prealloc_sa) { int new_page_count __maybe_unused; size_t nr_left = nr_pages_requested; @@ -1786,16 +1784,9 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked( if (np) { int i; - struct kbase_sub_alloc *sa; + struct kbase_sub_alloc *const sa = *prealloc_sa; struct page *p; - sa = kmalloc(sizeof(*sa), GFP_KERNEL); - if (!sa) { - kbase_mem_pool_free_locked(pool, np, - false); - goto alloc_failed; - } - /* store pointers back to the control struct */ np->lru.next = (void *)sa; for (p = np; p < np + SZ_2M / SZ_4K; p++) @@ -1811,6 +1802,10 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked( bitmap_set(sa->sub_pages, 0, nr_left); nr_left = 0; + /* Indicate to user that we'll free this memory + * later. + */ + *prealloc_sa = NULL; /* expose for later use */ list_add(&sa->link, &kctx->mem_partials); @@ -2127,11 +2122,6 @@ void kbase_mem_kref_free(struct kref *kref) case KBASE_MEM_TYPE_RAW: /* raw pages, external cleanup */ break; - #ifdef CONFIG_UMP - case KBASE_MEM_TYPE_IMPORTED_UMP: - ump_dd_release(alloc->imported.ump_handle); - break; -#endif #ifdef CONFIG_DMA_SHARED_BUFFER case KBASE_MEM_TYPE_IMPORTED_UMM: dma_buf_detach(alloc->imported.umm.dma_buf, @@ -2649,6 +2639,8 @@ static int kbase_jit_grow(struct kbase_context *kctx, struct kbase_mem_pool *pool; int ret = -ENOMEM; struct tagged_addr *gpu_pages; + struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; + int i; if (info->commit_pages > reg->nr_pages) { /* Attempted to grow larger than maximum size */ @@ -2672,6 +2664,14 @@ static int kbase_jit_grow(struct kbase_context *kctx, pages_required = delta; #ifdef CONFIG_MALI_2MB_ALLOC + /* Preallocate memory for the sub-allocation structs */ + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { + prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), + GFP_KERNEL); + if (!prealloc_sas[i]) + goto update_failed; + } + if (pages_required >= (SZ_2M / SZ_4K)) { pool = &kctx->lp_mem_pool; /* Round up to number of 2 MB pages required */ @@ -2711,7 +2711,7 @@ static int kbase_jit_grow(struct kbase_context *kctx, } gpu_pages = kbase_alloc_phy_pages_helper_locked(reg->gpu_alloc, pool, - delta); + delta, &prealloc_sas[0]); if (!gpu_pages) { kbase_mem_pool_unlock(pool); mutex_unlock(&kctx->mem_partials_lock); @@ -2722,7 +2722,7 @@ static int kbase_jit_grow(struct kbase_context *kctx, struct tagged_addr *cpu_pages; cpu_pages = kbase_alloc_phy_pages_helper_locked(reg->cpu_alloc, - pool, delta); + pool, delta, &prealloc_sas[1]); if (!cpu_pages) { kbase_free_phy_pages_helper_locked(reg->gpu_alloc, pool, gpu_pages, delta); @@ -2753,6 +2753,9 @@ done: update_failed: kbase_gpu_vm_unlock(kctx); update_failed_unlocked: + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) + kfree(prealloc_sas[i]); + return ret; } @@ -3012,6 +3015,7 @@ bool kbase_jit_evict(struct kbase_context *kctx) reg = list_entry(kctx->jit_pool_head.prev, struct kbase_va_region, jit_node); list_del(®->jit_node); + list_del_init(®->gpu_alloc->evict_node); } mutex_unlock(&kctx->jit_evict_lock); @@ -3029,12 +3033,6 @@ void kbase_jit_term(struct kbase_context *kctx) /* Free all allocations for this context */ - /* - * Flush the freeing of allocations whose backing has been freed - * (i.e. everything in jit_destroy_head). - */ - cancel_work_sync(&kctx->jit_work); - kbase_gpu_vm_lock(kctx); mutex_lock(&kctx->jit_evict_lock); /* Free all allocations from the pool */ @@ -3042,6 +3040,7 @@ void kbase_jit_term(struct kbase_context *kctx) walker = list_first_entry(&kctx->jit_pool_head, struct kbase_va_region, jit_node); list_del(&walker->jit_node); + list_del_init(&walker->gpu_alloc->evict_node); mutex_unlock(&kctx->jit_evict_lock); walker->flags &= ~KBASE_REG_JIT; kbase_mem_free_region(kctx, walker); @@ -3053,6 +3052,7 @@ void kbase_jit_term(struct kbase_context *kctx) walker = list_first_entry(&kctx->jit_active_head, struct kbase_va_region, jit_node); list_del(&walker->jit_node); + list_del_init(&walker->gpu_alloc->evict_node); mutex_unlock(&kctx->jit_evict_lock); walker->flags &= ~KBASE_REG_JIT; kbase_mem_free_region(kctx, walker); @@ -3060,6 +3060,12 @@ void kbase_jit_term(struct kbase_context *kctx) } mutex_unlock(&kctx->jit_evict_lock); kbase_gpu_vm_unlock(kctx); + + /* + * Flush the freeing of allocations whose backing has been freed + * (i.e. everything in jit_destroy_head). + */ + cancel_work_sync(&kctx->jit_work); } static int kbase_jd_user_buf_map(struct kbase_context *kctx, @@ -3335,9 +3341,6 @@ struct kbase_mem_phy_alloc *kbase_map_external_resource( } } break; - case KBASE_MEM_TYPE_IMPORTED_UMP: { - break; - } #ifdef CONFIG_DMA_SHARED_BUFFER case KBASE_MEM_TYPE_IMPORTED_UMM: { reg->gpu_alloc->imported.umm.current_mapping_usage_count++; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h index b456752b417b..36ed41c543e4 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -35,9 +35,6 @@ #endif #include <linux/kref.h> -#ifdef CONFIG_UMP -#include <linux/ump.h> -#endif /* CONFIG_UMP */ #include "mali_base_kernel.h" #include <mali_kbase_hw.h> #include "mali_kbase_pm.h" @@ -77,7 +74,6 @@ struct kbase_cpu_mapping { enum kbase_memory_type { KBASE_MEM_TYPE_NATIVE, - KBASE_MEM_TYPE_IMPORTED_UMP, KBASE_MEM_TYPE_IMPORTED_UMM, KBASE_MEM_TYPE_IMPORTED_USER_BUF, KBASE_MEM_TYPE_ALIAS, @@ -133,9 +129,6 @@ struct kbase_mem_phy_alloc { /* member in union valid based on @a type */ union { -#ifdef CONFIG_UMP - ump_dd_handle ump_handle; -#endif /* CONFIG_UMP */ #if defined(CONFIG_DMA_SHARED_BUFFER) struct { struct dma_buf *dma_buf; @@ -204,8 +197,7 @@ static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc * */ static inline bool kbase_mem_is_imported(enum kbase_memory_type type) { - return (type == KBASE_MEM_TYPE_IMPORTED_UMP) || - (type == KBASE_MEM_TYPE_IMPORTED_UMM) || + return (type == KBASE_MEM_TYPE_IMPORTED_UMM) || (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF); } @@ -1063,6 +1055,8 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, * @alloc: allocation object to add pages to * @pool: Memory pool to allocate from * @nr_pages_requested: number of physical pages to allocate + * @prealloc_sa: Information about the partial allocation if the amount + * of memory requested is not a multiple of 2MB. * * Allocates \a nr_pages_requested and updates the alloc object. This function * does not allocate new pages from the kernel, and therefore will never trigger @@ -1092,13 +1086,16 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, * @pool must be alloc->imported.kctx->lp_mem_pool. Otherwise it must be * alloc->imported.kctx->mem_pool. * + * @prealloc_sa shall be set to NULL if it has been consumed by this function. + * * Return: Pointer to array of allocated pages. NULL on failure. * * Note : Caller must hold pool->pool_lock */ struct tagged_addr *kbase_alloc_phy_pages_helper_locked( struct kbase_mem_phy_alloc *alloc, struct kbase_mem_pool *pool, - size_t nr_pages_requested); + size_t nr_pages_requested, + struct kbase_sub_alloc **prealloc_sa); /** * @brief Free physical pages. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c index 33021142e6d5..59cc03538f1b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -616,8 +616,7 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in } /* limit to imported memory */ - if ((reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMP) && - (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM)) + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) goto out_unlock; /* no change? */ @@ -632,28 +631,17 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in reg->flags |= real_flags; /* Currently supporting only imported memory */ - switch (reg->gpu_alloc->type) { -#ifdef CONFIG_UMP - case KBASE_MEM_TYPE_IMPORTED_UMP: - ret = kbase_mmu_update_pages(kctx, reg->start_pfn, - kbase_get_gpu_phy_pages(reg), - reg->gpu_alloc->nents, reg->flags); - break; -#endif #ifdef CONFIG_DMA_SHARED_BUFFER - case KBASE_MEM_TYPE_IMPORTED_UMM: + if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { /* Future use will use the new flags, existing mapping will NOT be updated * as memory should not be in use by the GPU when updating the flags. */ ret = 0; WARN_ON(reg->gpu_alloc->imported.umm.current_mapping_usage_count); - break; -#endif - default: - break; } +#endif /* CONFIG_DMA_SHARED_BUFFER */ - /* roll back on error, i.e. not UMP */ + /* roll back on error */ if (ret) reg->flags = prev_flags; @@ -666,128 +654,6 @@ out: #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) -#ifdef CONFIG_UMP -static struct kbase_va_region *kbase_mem_from_ump(struct kbase_context *kctx, ump_secure_id id, u64 *va_pages, u64 *flags) -{ - struct kbase_va_region *reg; - ump_dd_handle umph; - u64 block_count; - const ump_dd_physical_block_64 *block_array; - u64 i, j; - int page = 0; - ump_alloc_flags ump_flags; - ump_alloc_flags cpu_flags; - ump_alloc_flags gpu_flags; - - if (*flags & BASE_MEM_SECURE) - goto bad_flags; - - umph = ump_dd_from_secure_id(id); - if (UMP_DD_INVALID_MEMORY_HANDLE == umph) - goto bad_id; - - ump_flags = ump_dd_allocation_flags_get(umph); - cpu_flags = (ump_flags >> UMP_DEVICE_CPU_SHIFT) & UMP_DEVICE_MASK; - gpu_flags = (ump_flags >> DEFAULT_UMP_GPU_DEVICE_SHIFT) & - UMP_DEVICE_MASK; - - *va_pages = ump_dd_size_get_64(umph); - *va_pages >>= PAGE_SHIFT; - - if (!*va_pages) - goto bad_size; - - if (*va_pages > (U64_MAX / PAGE_SIZE)) - /* 64-bit address range is the max */ - goto bad_size; - - if (*flags & BASE_MEM_SAME_VA) - reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_SAME_VA); - else - reg = kbase_alloc_free_region(kctx, 0, *va_pages, KBASE_REG_ZONE_CUSTOM_VA); - - if (!reg) - goto no_region; - - /* we've got pages to map now, and support SAME_VA */ - *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; - - reg->gpu_alloc = kbase_alloc_create(*va_pages, KBASE_MEM_TYPE_IMPORTED_UMP); - if (IS_ERR_OR_NULL(reg->gpu_alloc)) - goto no_alloc_obj; - - reg->cpu_alloc = kbase_mem_phy_alloc_get(reg->gpu_alloc); - - reg->gpu_alloc->imported.ump_handle = umph; - - reg->flags &= ~KBASE_REG_FREE; - reg->flags |= KBASE_REG_GPU_NX; /* UMP is always No eXecute */ - reg->flags &= ~KBASE_REG_GROWABLE; /* UMP cannot be grown */ - - /* Override import flags based on UMP flags */ - *flags &= ~(BASE_MEM_CACHED_CPU); - *flags &= ~(BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_CPU_WR); - *flags &= ~(BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR); - - if ((cpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == - (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) { - reg->flags |= KBASE_REG_CPU_CACHED; - *flags |= BASE_MEM_CACHED_CPU; - } - - if (cpu_flags & UMP_PROT_CPU_WR) { - reg->flags |= KBASE_REG_CPU_WR; - *flags |= BASE_MEM_PROT_CPU_WR; - } - - if (cpu_flags & UMP_PROT_CPU_RD) { - reg->flags |= KBASE_REG_CPU_RD; - *flags |= BASE_MEM_PROT_CPU_RD; - } - - if ((gpu_flags & (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) == - (UMP_HINT_DEVICE_RD | UMP_HINT_DEVICE_WR)) - reg->flags |= KBASE_REG_GPU_CACHED; - - if (gpu_flags & UMP_PROT_DEVICE_WR) { - reg->flags |= KBASE_REG_GPU_WR; - *flags |= BASE_MEM_PROT_GPU_WR; - } - - if (gpu_flags & UMP_PROT_DEVICE_RD) { - reg->flags |= KBASE_REG_GPU_RD; - *flags |= BASE_MEM_PROT_GPU_RD; - } - - /* ump phys block query */ - ump_dd_phys_blocks_get_64(umph, &block_count, &block_array); - - for (i = 0; i < block_count; i++) { - for (j = 0; j < (block_array[i].size >> PAGE_SHIFT); j++) { - struct tagged_addr tagged; - - tagged = as_tagged(block_array[i].addr + - (j << PAGE_SHIFT)); - reg->gpu_alloc->pages[page] = tagged; - page++; - } - } - reg->gpu_alloc->nents = *va_pages; - reg->extent = 0; - - return reg; - -no_alloc_obj: - kfree(reg); -no_region: -bad_size: - ump_dd_release(umph); -bad_id: -bad_flags: - return NULL; -} -#endif /* CONFIG_UMP */ - #ifdef CONFIG_DMA_SHARED_BUFFER static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, u64 *flags, u32 padding) @@ -981,7 +847,11 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( user_buf->address = address; user_buf->nr_pages = *va_pages; user_buf->mm = current->mm; +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE atomic_inc(¤t->mm->mm_count); +#else + mmgrab(current->mm); +#endif user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), GFP_KERNEL); @@ -1308,17 +1178,6 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, } switch (type) { -#ifdef CONFIG_UMP - case BASE_MEM_IMPORT_TYPE_UMP: { - ump_secure_id id; - - if (get_user(id, (ump_secure_id __user *)phandle)) - reg = NULL; - else - reg = kbase_mem_from_ump(kctx, id, va_pages, flags); - } - break; -#endif /* CONFIG_UMP */ #ifdef CONFIG_DMA_SHARED_BUFFER case BASE_MEM_IMPORT_TYPE_UMM: { int fd; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mali_kbase_mmu.c index 4abab34cf155..7ec579d6dc6c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mmu.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mmu.c @@ -1,6 +1,6 @@ /* * - * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -314,6 +314,8 @@ void page_fault_worker(struct work_struct *data) struct kbase_mem_pool *pool; int pages_to_grow; struct tagged_addr *gpu_pages, *cpu_pages; + struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; + int i; faulting_as = container_of(data, struct kbase_as, work_pagefault); fault_pfn = faulting_as->fault_addr >> PAGE_SHIFT; @@ -399,6 +401,22 @@ void page_fault_worker(struct work_struct *data) } page_fault_retry: +#ifdef CONFIG_MALI_2MB_ALLOC + /* Preallocate memory for the sub-allocation structs if necessary */ + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { + if (!prealloc_sas[i]) { + prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), + GFP_KERNEL); + if (!prealloc_sas[i]) { + kbase_mmu_report_fault_and_kill( + kctx, faulting_as, + "Failed pre-allocating memory for sub-allocations' metadata"); + goto fault_done; + } + } + } +#endif /* CONFIG_MALI_2MB_ALLOC */ + /* so we have a translation fault, let's see if it is for growable * memory */ kbase_gpu_vm_lock(kctx); @@ -520,13 +538,14 @@ page_fault_retry: */ if (kbase_mem_pool_size(pool) >= min_pool_size) { gpu_pages = kbase_alloc_phy_pages_helper_locked( - region->gpu_alloc, pool, new_pages); + region->gpu_alloc, pool, new_pages, + &prealloc_sas[0]); if (gpu_pages) { if (region->gpu_alloc != region->cpu_alloc) { cpu_pages = kbase_alloc_phy_pages_helper_locked( region->cpu_alloc, pool, - new_pages); + new_pages, &prealloc_sas[1]); if (cpu_pages) { grown = true; @@ -656,6 +675,9 @@ page_fault_retry: } fault_done: + for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) + kfree(prealloc_sas[i]); + /* * By this point, the fault was handled in some way, * so release the ctx refcount diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c index 83b83fe7533d..3f87ac08eec6 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -691,13 +691,6 @@ static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) ret = 0; break; } - case KBASE_MEM_TYPE_IMPORTED_UMP: - { - dev_warn(katom->kctx->kbdev->dev, - "UMP is not supported for debug_copy jobs\n"); - ret = -EINVAL; - goto out_unlock; - } default: /* Nothing to be done. */ break; @@ -971,7 +964,8 @@ fail: static u8 kbase_jit_free_get_id(struct kbase_jd_atom *katom) { - if (WARN_ON(katom->core_req != BASE_JD_REQ_SOFT_JIT_FREE)) + if (WARN_ON((katom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) != + BASE_JD_REQ_SOFT_JIT_FREE)) return 0; return (u8) katom->jc; @@ -1017,7 +1011,8 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) list_for_each_entry(jit_atom, &kctx->jit_atoms_head, jit_node) { if (jit_atom == katom) break; - if (jit_atom->core_req == BASE_JD_REQ_SOFT_JIT_FREE) { + if ((jit_atom->core_req & BASE_JD_REQ_SOFT_JOB_TYPE) == + BASE_JD_REQ_SOFT_JIT_FREE) { u8 free_id = kbase_jit_free_get_id(jit_atom); if (free_id && kctx->jit_alloc[free_id]) { diff --git a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c index 324cf75df6a2..d2bfa12f4a96 100755 --- a/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c +++ b/drivers/gpu/arm/bifrost/platform/rk/mali_kbase_config_rk.c @@ -14,6 +14,7 @@ #include <mali_kbase_defs.h> #include <mali_kbase_config.h> #include <backend/gpu/mali_kbase_pm_internal.h> +#include <backend/gpu/mali_kbase_pm_defs.h> #include <linux/pm_runtime.h> #include <linux/suspend.h> @@ -398,22 +399,25 @@ static ssize_t utilisation_show(struct device *dev, struct rk_context *platform = get_rk_context(kbdev); ssize_t ret = 0; unsigned long period_in_us = platform->utilisation_period * 1000; - unsigned long total_time; - unsigned long busy_time; - unsigned long utilisation; - - kbase_pm_reset_dvfs_utilisation(kbdev); + u32 utilisation; + struct kbasep_pm_metrics metrics_when_start; + struct kbasep_pm_metrics metrics_diff; /* between start and end. */ + u32 total_time = 0; + u32 busy_time = 0; + + /* get current metrics data. */ + kbase_pm_get_dvfs_metrics(kbdev, &metrics_when_start, &metrics_diff); + /* sleep for 'period_in_us'. */ usleep_range(period_in_us, period_in_us + 100); - kbase_pm_get_dvfs_utilisation(kbdev, &total_time, &busy_time); - /* 'devfreq_dev_profile' instance registered to devfreq - * also uses kbase_pm_reset_dvfs_utilisation - * and kbase_pm_get_dvfs_utilisation. - * it's better to cat this file when DVFS is disabled. - */ - D("total_time : %lu, busy_time : %lu.", total_time, busy_time); + /* get metrics data between start and end. */ + kbase_pm_get_dvfs_metrics(kbdev, &metrics_when_start, &metrics_diff); + + total_time = metrics_diff.time_busy + metrics_diff.time_idle; + busy_time = metrics_diff.time_busy; + D("total_time : %u, busy_time : %u.", total_time, busy_time); utilisation = busy_time * 100 / total_time; - ret += snprintf(buf, PAGE_SIZE, "%ld\n", utilisation); + ret += snprintf(buf, PAGE_SIZE, "%d\n", utilisation); return ret; } diff --git a/drivers/gpu/arm/bifrost/sconscript b/drivers/gpu/arm/bifrost/sconscript index eae28f4011e9..4c38f2a07045 100644 --- a/drivers/gpu/arm/bifrost/sconscript +++ b/drivers/gpu/arm/bifrost/sconscript @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -47,7 +47,6 @@ if Glob('#kernel/drivers/gpu/arm/midgard/tests/internal/src/mock') and env['unit make_args = env.kernel_get_config_defines(ret_list = True) + [ 'PLATFORM=%s' % env['platform'], - 'MALI_ERROR_INJECT_ON=%s' % env['error_inject'], 'MALI_KERNEL_TEST_API=%s' % env['debug'], 'MALI_UNIT_TEST=%s' % env['unit'], 'MALI_RELEASE_NAME=%s' % env['mali_release_name'], @@ -59,10 +58,6 @@ make_args = env.kernel_get_config_defines(ret_list = True) + [ kbase = env.BuildKernelModule('$STATIC_LIB_PATH/mali_kbase.ko', kbase_src, make_args = make_args) -# need Module.symvers from ump.ko build -if int(env['ump']) == 1: - env.Depends(kbase, '$STATIC_LIB_PATH/ump.ko') - if 'smc_protected_mode_switcher' in env: env.Depends('$STATIC_LIB_PATH/mali_kbase.ko', '$STATIC_LIB_PATH/smc_protected_mode_switcher.ko') diff --git a/drivers/gpu/arm/bifrost/tests/Mconfig b/drivers/gpu/arm/bifrost/tests/Mconfig new file mode 100644 index 000000000000..35f85139cf12 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/Mconfig @@ -0,0 +1,22 @@ +# +# (C) COPYRIGHT 2018 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU licence. +# +# A copy of the licence is included with the program, and can also be obtained +# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +# + +config UNIT_TEST_KERNEL_MODULES + bool + default y if UNIT_TEST_CODE && BUILD_KERNEL_MODULES + default n + +config BUILD_IPA_TESTS + bool + default y if UNIT_TEST_KERNEL_MODULES && MALI_BIFROST_DEVFREQ + default n diff --git a/drivers/gpu/arm/bifrost/tests/build.bp b/drivers/gpu/arm/bifrost/tests/build.bp new file mode 100644 index 000000000000..28a756b16dc1 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/build.bp @@ -0,0 +1,36 @@ +/* + * Copyright: + * ---------------------------------------------------------------------------- + * This confidential and proprietary software may be used only as authorized + * by a licensing agreement from ARM Limited. + * (C) COPYRIGHT 2018 ARM Limited, ALL RIGHTS RESERVED + * The entire notice above must be reproduced on all authorized copies and + * copies may only be made to the extent permitted by a licensing agreement + * from ARM Limited. + * ---------------------------------------------------------------------------- + */ + +bob_defaults { + name: "kernel_test_module_defaults", + defaults: ["mali_kbase_shared_config_defaults"], + include_dirs: [ + "kernel/drivers/gpu/arm", + "kernel/drivers/gpu/arm/midgard", + "kernel/drivers/gpu/arm/midgard/backend/gpu", + "kernel/drivers/gpu/arm/midgard/tests/include", + ], + extra_symbols: ["kutf"], +} + +subdirs = [ + "kutf", + "mali_kutf_irq_test", +] + +optional_subdirs = [ + "kutf_test", + "kutf_test_runner", + "mali_kutf_ipa_test", + "mali_kutf_ipa_unit_test", + "mali_kutf_vinstr_test", +] diff --git a/drivers/gpu/arm/bifrost/tests/kutf/build.bp b/drivers/gpu/arm/bifrost/tests/kutf/build.bp new file mode 100644 index 000000000000..f6d4c3fc7e15 --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/kutf/build.bp @@ -0,0 +1,31 @@ +/* + * Copyright: + * ---------------------------------------------------------------------------- + * This confidential and proprietary software may be used only as authorized + * by a licensing agreement from ARM Limited. + * (C) COPYRIGHT 2018 ARM Limited, ALL RIGHTS RESERVED + * The entire notice above must be reproduced on all authorized copies and + * copies may only be made to the extent permitted by a licensing agreement + * from ARM Limited. + * ---------------------------------------------------------------------------- + */ + +bob_kernel_module { + name: "kutf", + defaults: ["kernel_defaults"], + srcs: [ + "Kbuild", + "kutf_helpers.c", + "kutf_helpers_user.c", + "kutf_mem.c", + "kutf_resultset.c", + "kutf_suite.c", + "kutf_utils.c", + ], + kbuild_options: ["CONFIG_MALI_KUTF=m"], + include_dirs: ["kernel/drivers/gpu/arm/midgard/tests/include"], + enabled: false, + unit_test_kernel_modules: { + enabled: true, + }, +} diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile index d7dd5880f418..40df1179b86b 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/Makefile @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2015, 2017-2018 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -29,12 +29,7 @@ $(error Must specify KDIR to point to the kernel to target)) endif TEST_CCFLAGS := \ - -DMALI_DEBUG=$(MALI_BIFROST_DEBUG) \ - -DMALI_BACKEND_KERNEL=$(MALI_BACKEND_KERNEL) \ - -DMALI_NO_MALI=$(MALI_BIFROST_NO_MALI) \ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ - -DMALI_USE_UMP=$(MALI_USE_UMP) \ - -DMALI_ERROR_INJECT_ON=$(MALI_ERROR_INJECT_ON) \ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ $(SCONS_CFLAGS) \ -I$(CURDIR)/../include \ diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp new file mode 100644 index 000000000000..e1f77b0c5d7b --- /dev/null +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/build.bp @@ -0,0 +1,27 @@ +/* + * Copyright: + * ---------------------------------------------------------------------------- + * This confidential and proprietary software may be used only as authorized + * by a licensing agreement from ARM Limited. + * (C) COPYRIGHT 2018 ARM Limited, ALL RIGHTS RESERVED + * The entire notice above must be reproduced on all authorized copies and + * copies may only be made to the extent permitted by a licensing agreement + * from ARM Limited. + * ---------------------------------------------------------------------------- + */ + +bob_kernel_module { + name: "mali_kutf_irq_test", + defaults: ["kernel_test_module_defaults"], + srcs: [ + "Kbuild", + "mali_kutf_irq_test_main.c", + ], + extra_symbols: ["mali_kbase"], + install_group: "IG_tests", + enabled: false, + unit_test_kernel_modules: { + enabled: true, + kbuild_options: ["CONFIG_MALI_IRQ_LATENCY=m"], + }, +} diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/sconscript b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/sconscript index 2be566b29abe..0ec5ce7e3632 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/sconscript +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_irq_test/sconscript @@ -1,5 +1,5 @@ # -# (C) COPYRIGHT 2015, 2017 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2015-2018 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -29,7 +29,7 @@ if env.GetOption('clean') : cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, []) env.KernelObjTarget('mali_kutf_irq_test', cmd) else: - makeAction=Action("cd ${SOURCE.dir} && make MALI_BIFROST_DEBUG=${debug} MALI_BACKEND_KERNEL=1 MALI_ERROR_INJECT_ON=${error_inject} MALI_BIFROST_NO_MALI=${no_mali} MALI_UNIT_TEST=${unit} MALI_USE_UMP=${ump} MALI_CUSTOMER_RELEASE=${release} %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % env.kernel_get_config_defines(), '$MAKECOMSTR') + makeAction=Action("cd ${SOURCE.dir} && make MALI_UNIT_TEST=${unit} MALI_CUSTOMER_RELEASE=${release} %s && ( ( [ -f mali_kutf_irq_test.ko ] && cp mali_kutf_irq_test.ko $STATIC_LIB_PATH/ ) || touch $STATIC_LIB_PATH/mali_kutf_irq_test.ko)" % env.kernel_get_config_defines(), '$MAKECOMSTR') cmd = env.Command('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', src, [makeAction]) env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/kutf.ko') env.Depends('$STATIC_LIB_PATH/mali_kutf_irq_test.ko', '$STATIC_LIB_PATH/mali_kbase.ko') |