12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052 |
- /*
- * Copyright (c) 2013-2022, ARM Limited and Contributors. All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
- #include <assert.h>
- #include <string.h>
- #include <arch.h>
- #include <arch_helpers.h>
- #include <common/bl_common.h>
- #include <common/debug.h>
- #include <context.h>
- #include <drivers/delay_timer.h>
- #include <lib/el3_runtime/context_mgmt.h>
- #include <lib/utils.h>
- #include <plat/common/platform.h>
- #include "psci_private.h"
- /*
- * SPD power management operations, expected to be supplied by the registered
- * SPD on successful SP initialization
- */
- const spd_pm_ops_t *psci_spd_pm;
- /*
- * PSCI requested local power state map. This array is used to store the local
- * power states requested by a CPU for power levels from level 1 to
- * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power
- * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a
- * CPU are the same.
- *
- * During state coordination, the platform is passed an array containing the
- * local states requested for a particular non cpu power domain by each cpu
- * within the domain.
- *
- * TODO: Dense packing of the requested states will cause cache thrashing
- * when multiple power domains write to it. If we allocate the requested
- * states at each power level in a cache-line aligned per-domain memory,
- * the cache thrashing can be avoided.
- */
- static plat_local_state_t
- psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT];
- unsigned int psci_plat_core_count;
- /*******************************************************************************
- * Arrays that hold the platform's power domain tree information for state
- * management of power domains.
- * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain
- * which is an ancestor of a CPU power domain.
- * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain
- ******************************************************************************/
- non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS]
- #if USE_COHERENT_MEM
- __section(".tzfw_coherent_mem")
- #endif
- ;
- /* Lock for PSCI state coordination */
- DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]);
- cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT];
- /*******************************************************************************
- * Pointer to functions exported by the platform to complete power mgmt. ops
- ******************************************************************************/
- const plat_psci_ops_t *psci_plat_pm_ops;
- /******************************************************************************
- * Check that the maximum power level supported by the platform makes sense
- *****************************************************************************/
- CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) &&
- (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL),
- assert_platform_max_pwrlvl_check);
- /*
- * The plat_local_state used by the platform is one of these types: RUN,
- * RETENTION and OFF. The platform can define further sub-states for each type
- * apart from RUN. This categorization is done to verify the sanity of the
- * psci_power_state passed by the platform and to print debug information. The
- * categorization is done on the basis of the following conditions:
- *
- * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN.
- *
- * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is
- * STATE_TYPE_RETN.
- *
- * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is
- * STATE_TYPE_OFF.
- */
- typedef enum plat_local_state_type {
- STATE_TYPE_RUN = 0,
- STATE_TYPE_RETN,
- STATE_TYPE_OFF
- } plat_local_state_type_t;
- /* Function used to categorize plat_local_state. */
- static plat_local_state_type_t find_local_state_type(plat_local_state_t state)
- {
- if (state != 0U) {
- if (state > PLAT_MAX_RET_STATE) {
- return STATE_TYPE_OFF;
- } else {
- return STATE_TYPE_RETN;
- }
- } else {
- return STATE_TYPE_RUN;
- }
- }
- /******************************************************************************
- * Check that the maximum retention level supported by the platform is less
- * than the maximum off level.
- *****************************************************************************/
- CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE,
- assert_platform_max_off_and_retn_state_check);
- /******************************************************************************
- * This function ensures that the power state parameter in a CPU_SUSPEND request
- * is valid. If so, it returns the requested states for each power level.
- *****************************************************************************/
- int psci_validate_power_state(unsigned int power_state,
- psci_power_state_t *state_info)
- {
- /* Check SBZ bits in power state are zero */
- if (psci_check_power_state(power_state) != 0U)
- return PSCI_E_INVALID_PARAMS;
- assert(psci_plat_pm_ops->validate_power_state != NULL);
- /* Validate the power_state using platform pm_ops */
- return psci_plat_pm_ops->validate_power_state(power_state, state_info);
- }
- /******************************************************************************
- * This function retrieves the `psci_power_state_t` for system suspend from
- * the platform.
- *****************************************************************************/
- void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info)
- {
- /*
- * Assert that the required pm_ops hook is implemented to ensure that
- * the capability detected during psci_setup() is valid.
- */
- assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL);
- /*
- * Query the platform for the power_state required for system suspend
- */
- psci_plat_pm_ops->get_sys_suspend_power_state(state_info);
- }
- /*******************************************************************************
- * This function verifies that the all the other cores in the system have been
- * turned OFF and the current CPU is the last running CPU in the system.
- * Returns true, if the current CPU is the last ON CPU or false otherwise.
- ******************************************************************************/
- bool psci_is_last_on_cpu(void)
- {
- unsigned int cpu_idx, my_idx = plat_my_core_pos();
- for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) {
- if (cpu_idx == my_idx) {
- assert(psci_get_aff_info_state() == AFF_STATE_ON);
- continue;
- }
- if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) {
- VERBOSE("core=%u other than current core=%u %s\n",
- cpu_idx, my_idx, "running in the system");
- return false;
- }
- }
- return true;
- }
- /*******************************************************************************
- * Routine to return the maximum power level to traverse to after a cpu has
- * been physically powered up. It is expected to be called immediately after
- * reset from assembler code.
- ******************************************************************************/
- static unsigned int get_power_on_target_pwrlvl(void)
- {
- unsigned int pwrlvl;
- /*
- * Assume that this cpu was suspended and retrieve its target power
- * level. If it is invalid then it could only have been turned off
- * earlier. PLAT_MAX_PWR_LVL will be the highest power level a
- * cpu can be turned off to.
- */
- pwrlvl = psci_get_suspend_pwrlvl();
- if (pwrlvl == PSCI_INVALID_PWR_LVL)
- pwrlvl = PLAT_MAX_PWR_LVL;
- assert(pwrlvl < PSCI_INVALID_PWR_LVL);
- return pwrlvl;
- }
- /******************************************************************************
- * Helper function to update the requested local power state array. This array
- * does not store the requested state for the CPU power level. Hence an
- * assertion is added to prevent us from accessing the CPU power level.
- *****************************************************************************/
- static void psci_set_req_local_pwr_state(unsigned int pwrlvl,
- unsigned int cpu_idx,
- plat_local_state_t req_pwr_state)
- {
- assert(pwrlvl > PSCI_CPU_PWR_LVL);
- if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) &&
- (cpu_idx < psci_plat_core_count)) {
- psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state;
- }
- }
- /******************************************************************************
- * This function initializes the psci_req_local_pwr_states.
- *****************************************************************************/
- void __init psci_init_req_local_pwr_states(void)
- {
- /* Initialize the requested state of all non CPU power domains as OFF */
- unsigned int pwrlvl;
- unsigned int core;
- for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) {
- for (core = 0; core < psci_plat_core_count; core++) {
- psci_req_local_pwr_states[pwrlvl][core] =
- PLAT_MAX_OFF_STATE;
- }
- }
- }
- /******************************************************************************
- * Helper function to return a reference to an array containing the local power
- * states requested by each cpu for a power domain at 'pwrlvl'. The size of the
- * array will be the number of cpu power domains of which this power domain is
- * an ancestor. These requested states will be used to determine a suitable
- * target state for this power domain during psci state coordination. An
- * assertion is added to prevent us from accessing the CPU power level.
- *****************************************************************************/
- static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl,
- unsigned int cpu_idx)
- {
- assert(pwrlvl > PSCI_CPU_PWR_LVL);
- if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) &&
- (cpu_idx < psci_plat_core_count)) {
- return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx];
- } else
- return NULL;
- }
- /*
- * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent
- * memory.
- *
- * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory,
- * it's accessed by both cached and non-cached participants. To serve the common
- * minimum, perform a cache flush before read and after write so that non-cached
- * participants operate on latest data in main memory.
- *
- * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent
- * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent.
- * In both cases, no cache operations are required.
- */
- /*
- * Retrieve local state of non-CPU power domain node from a non-cached CPU,
- * after any required cache maintenance operation.
- */
- static plat_local_state_t get_non_cpu_pd_node_local_state(
- unsigned int parent_idx)
- {
- #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
- flush_dcache_range(
- (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx],
- sizeof(psci_non_cpu_pd_nodes[parent_idx]));
- #endif
- return psci_non_cpu_pd_nodes[parent_idx].local_state;
- }
- /*
- * Update local state of non-CPU power domain node from a cached CPU; perform
- * any required cache maintenance operation afterwards.
- */
- static void set_non_cpu_pd_node_local_state(unsigned int parent_idx,
- plat_local_state_t state)
- {
- psci_non_cpu_pd_nodes[parent_idx].local_state = state;
- #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY)
- flush_dcache_range(
- (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx],
- sizeof(psci_non_cpu_pd_nodes[parent_idx]));
- #endif
- }
- /******************************************************************************
- * Helper function to return the current local power state of each power domain
- * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This
- * function will be called after a cpu is powered on to find the local state
- * each power domain has emerged from.
- *****************************************************************************/
- void psci_get_target_local_pwr_states(unsigned int end_pwrlvl,
- psci_power_state_t *target_state)
- {
- unsigned int parent_idx, lvl;
- plat_local_state_t *pd_state = target_state->pwr_domain_state;
- pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state();
- parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
- /* Copy the local power state from node to state_info */
- for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
- pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx);
- parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
- }
- /* Set the the higher levels to RUN */
- for (; lvl <= PLAT_MAX_PWR_LVL; lvl++)
- target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN;
- }
- /******************************************************************************
- * Helper function to set the target local power state that each power domain
- * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will
- * enter. This function will be called after coordination of requested power
- * states has been done for each power level.
- *****************************************************************************/
- static void psci_set_target_local_pwr_states(unsigned int end_pwrlvl,
- const psci_power_state_t *target_state)
- {
- unsigned int parent_idx, lvl;
- const plat_local_state_t *pd_state = target_state->pwr_domain_state;
- psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]);
- /*
- * Need to flush as local_state might be accessed with Data Cache
- * disabled during power on
- */
- psci_flush_cpu_data(psci_svc_cpu_data.local_state);
- parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node;
- /* Copy the local_state from state_info */
- for (lvl = 1U; lvl <= end_pwrlvl; lvl++) {
- set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]);
- parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
- }
- }
- /*******************************************************************************
- * PSCI helper function to get the parent nodes corresponding to a cpu_index.
- ******************************************************************************/
- void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx,
- unsigned int end_lvl,
- unsigned int *node_index)
- {
- unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node;
- unsigned int i;
- unsigned int *node = node_index;
- for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) {
- *node = parent_node;
- node++;
- parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node;
- }
- }
- /******************************************************************************
- * This function is invoked post CPU power up and initialization. It sets the
- * affinity info state, target power state and requested power state for the
- * current CPU and all its ancestor power domains to RUN.
- *****************************************************************************/
- void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl)
- {
- unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl;
- parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
- /* Reset the local_state to RUN for the non cpu power domains. */
- for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
- set_non_cpu_pd_node_local_state(parent_idx,
- PSCI_LOCAL_STATE_RUN);
- psci_set_req_local_pwr_state(lvl,
- cpu_idx,
- PSCI_LOCAL_STATE_RUN);
- parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
- }
- /* Set the affinity info state to ON */
- psci_set_aff_info_state(AFF_STATE_ON);
- psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN);
- psci_flush_cpu_data(psci_svc_cpu_data);
- }
- /******************************************************************************
- * This function is passed the local power states requested for each power
- * domain (state_info) between the current CPU domain and its ancestors until
- * the target power level (end_pwrlvl). It updates the array of requested power
- * states with this information.
- *
- * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it
- * retrieves the states requested by all the cpus of which the power domain at
- * that level is an ancestor. It passes this information to the platform to
- * coordinate and return the target power state. If the target state for a level
- * is RUN then subsequent levels are not considered. At the CPU level, state
- * coordination is not required. Hence, the requested and the target states are
- * the same.
- *
- * The 'state_info' is updated with the target state for each level between the
- * CPU and the 'end_pwrlvl' and returned to the caller.
- *
- * This function will only be invoked with data cache enabled and while
- * powering down a core.
- *****************************************************************************/
- void psci_do_state_coordination(unsigned int end_pwrlvl,
- psci_power_state_t *state_info)
- {
- unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos();
- unsigned int start_idx;
- unsigned int ncpus;
- plat_local_state_t target_state, *req_states;
- assert(end_pwrlvl <= PLAT_MAX_PWR_LVL);
- parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node;
- /* For level 0, the requested state will be equivalent
- to target state */
- for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) {
- /* First update the requested power state */
- psci_set_req_local_pwr_state(lvl, cpu_idx,
- state_info->pwr_domain_state[lvl]);
- /* Get the requested power states for this power level */
- start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx;
- req_states = psci_get_req_local_pwr_states(lvl, start_idx);
- /*
- * Let the platform coordinate amongst the requested states at
- * this power level and return the target local power state.
- */
- ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus;
- target_state = plat_get_target_pwr_state(lvl,
- req_states,
- ncpus);
- state_info->pwr_domain_state[lvl] = target_state;
- /* Break early if the negotiated target power state is RUN */
- if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0)
- break;
- parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node;
- }
- /*
- * This is for cases when we break out of the above loop early because
- * the target power state is RUN at a power level < end_pwlvl.
- * We update the requested power state from state_info and then
- * set the target state as RUN.
- */
- for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) {
- psci_set_req_local_pwr_state(lvl, cpu_idx,
- state_info->pwr_domain_state[lvl]);
- state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN;
- }
- /* Update the target state in the power domain nodes */
- psci_set_target_local_pwr_states(end_pwrlvl, state_info);
- }
- /******************************************************************************
- * This function validates a suspend request by making sure that if a standby
- * state is requested then no power level is turned off and the highest power
- * level is placed in a standby/retention state.
- *
- * It also ensures that the state level X will enter is not shallower than the
- * state level X + 1 will enter.
- *
- * This validation will be enabled only for DEBUG builds as the platform is
- * expected to perform these validations as well.
- *****************************************************************************/
- int psci_validate_suspend_req(const psci_power_state_t *state_info,
- unsigned int is_power_down_state)
- {
- unsigned int max_off_lvl, target_lvl, max_retn_lvl;
- plat_local_state_t state;
- plat_local_state_type_t req_state_type, deepest_state_type;
- int i;
- /* Find the target suspend power level */
- target_lvl = psci_find_target_suspend_lvl(state_info);
- if (target_lvl == PSCI_INVALID_PWR_LVL)
- return PSCI_E_INVALID_PARAMS;
- /* All power domain levels are in a RUN state to begin with */
- deepest_state_type = STATE_TYPE_RUN;
- for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) {
- state = state_info->pwr_domain_state[i];
- req_state_type = find_local_state_type(state);
- /*
- * While traversing from the highest power level to the lowest,
- * the state requested for lower levels has to be the same or
- * deeper i.e. equal to or greater than the state at the higher
- * levels. If this condition is true, then the requested state
- * becomes the deepest state encountered so far.
- */
- if (req_state_type < deepest_state_type)
- return PSCI_E_INVALID_PARAMS;
- deepest_state_type = req_state_type;
- }
- /* Find the highest off power level */
- max_off_lvl = psci_find_max_off_lvl(state_info);
- /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */
- max_retn_lvl = PSCI_INVALID_PWR_LVL;
- if (target_lvl != max_off_lvl)
- max_retn_lvl = target_lvl;
- /*
- * If this is not a request for a power down state then max off level
- * has to be invalid and max retention level has to be a valid power
- * level.
- */
- if ((is_power_down_state == 0U) &&
- ((max_off_lvl != PSCI_INVALID_PWR_LVL) ||
- (max_retn_lvl == PSCI_INVALID_PWR_LVL)))
- return PSCI_E_INVALID_PARAMS;
- return PSCI_E_SUCCESS;
- }
- /******************************************************************************
- * This function finds the highest power level which will be powered down
- * amongst all the power levels specified in the 'state_info' structure
- *****************************************************************************/
- unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info)
- {
- int i;
- for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) {
- if (is_local_state_off(state_info->pwr_domain_state[i]) != 0)
- return (unsigned int) i;
- }
- return PSCI_INVALID_PWR_LVL;
- }
- /******************************************************************************
- * This functions finds the level of the highest power domain which will be
- * placed in a low power state during a suspend operation.
- *****************************************************************************/
- unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info)
- {
- int i;
- for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) {
- if (is_local_state_run(state_info->pwr_domain_state[i]) == 0)
- return (unsigned int) i;
- }
- return PSCI_INVALID_PWR_LVL;
- }
- /*******************************************************************************
- * This function is passed the highest level in the topology tree that the
- * operation should be applied to and a list of node indexes. It picks up locks
- * from the node index list in order of increasing power domain level in the
- * range specified.
- ******************************************************************************/
- void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl,
- const unsigned int *parent_nodes)
- {
- unsigned int parent_idx;
- unsigned int level;
- /* No locking required for level 0. Hence start locking from level 1 */
- for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) {
- parent_idx = parent_nodes[level - 1U];
- psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]);
- }
- }
- /*******************************************************************************
- * This function is passed the highest level in the topology tree that the
- * operation should be applied to and a list of node indexes. It releases the
- * locks in order of decreasing power domain level in the range specified.
- ******************************************************************************/
- void psci_release_pwr_domain_locks(unsigned int end_pwrlvl,
- const unsigned int *parent_nodes)
- {
- unsigned int parent_idx;
- unsigned int level;
- /* Unlock top down. No unlocking required for level 0. */
- for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) {
- parent_idx = parent_nodes[level - 1U];
- psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]);
- }
- }
- /*******************************************************************************
- * Simple routine to determine whether a mpidr is valid or not.
- ******************************************************************************/
- int psci_validate_mpidr(u_register_t mpidr)
- {
- if (plat_core_pos_by_mpidr(mpidr) < 0)
- return PSCI_E_INVALID_PARAMS;
- return PSCI_E_SUCCESS;
- }
- /*******************************************************************************
- * This function determines the full entrypoint information for the requested
- * PSCI entrypoint on power on/resume and returns it.
- ******************************************************************************/
- #ifdef __aarch64__
- static int psci_get_ns_ep_info(entry_point_info_t *ep,
- uintptr_t entrypoint,
- u_register_t context_id)
- {
- u_register_t ep_attr, sctlr;
- unsigned int daif, ee, mode;
- u_register_t ns_scr_el3 = read_scr_el3();
- u_register_t ns_sctlr_el1 = read_sctlr_el1();
- sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ?
- read_sctlr_el2() : ns_sctlr_el1;
- ee = 0;
- ep_attr = NON_SECURE | EP_ST_DISABLE;
- if ((sctlr & SCTLR_EE_BIT) != 0U) {
- ep_attr |= EP_EE_BIG;
- ee = 1;
- }
- SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr);
- ep->pc = entrypoint;
- zeromem(&ep->args, sizeof(ep->args));
- ep->args.arg0 = context_id;
- /*
- * Figure out whether the cpu enters the non-secure address space
- * in aarch32 or aarch64
- */
- if ((ns_scr_el3 & SCR_RW_BIT) != 0U) {
- /*
- * Check whether a Thumb entry point has been provided for an
- * aarch64 EL
- */
- if ((entrypoint & 0x1UL) != 0UL)
- return PSCI_E_INVALID_ADDRESS;
- mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1;
- ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX,
- DISABLE_ALL_EXCEPTIONS);
- } else {
- mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ?
- MODE32_hyp : MODE32_svc;
- /*
- * TODO: Choose async. exception bits if HYP mode is not
- * implemented according to the values of SCR.{AW, FW} bits
- */
- daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT;
- ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee,
- daif);
- }
- return PSCI_E_SUCCESS;
- }
- #else /* !__aarch64__ */
- static int psci_get_ns_ep_info(entry_point_info_t *ep,
- uintptr_t entrypoint,
- u_register_t context_id)
- {
- u_register_t ep_attr;
- unsigned int aif, ee, mode;
- u_register_t scr = read_scr();
- u_register_t ns_sctlr, sctlr;
- /* Switch to non secure state */
- write_scr(scr | SCR_NS_BIT);
- isb();
- ns_sctlr = read_sctlr();
- sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr;
- /* Return to original state */
- write_scr(scr);
- isb();
- ee = 0;
- ep_attr = NON_SECURE | EP_ST_DISABLE;
- if (sctlr & SCTLR_EE_BIT) {
- ep_attr |= EP_EE_BIG;
- ee = 1;
- }
- SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr);
- ep->pc = entrypoint;
- zeromem(&ep->args, sizeof(ep->args));
- ep->args.arg0 = context_id;
- mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc;
- /*
- * TODO: Choose async. exception bits if HYP mode is not
- * implemented according to the values of SCR.{AW, FW} bits
- */
- aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT;
- ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif);
- return PSCI_E_SUCCESS;
- }
- #endif /* __aarch64__ */
- /*******************************************************************************
- * This function validates the entrypoint with the platform layer if the
- * appropriate pm_ops hook is exported by the platform and returns the
- * 'entry_point_info'.
- ******************************************************************************/
- int psci_validate_entry_point(entry_point_info_t *ep,
- uintptr_t entrypoint,
- u_register_t context_id)
- {
- int rc;
- /* Validate the entrypoint using platform psci_ops */
- if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) {
- rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint);
- if (rc != PSCI_E_SUCCESS)
- return PSCI_E_INVALID_ADDRESS;
- }
- /*
- * Verify and derive the re-entry information for
- * the non-secure world from the non-secure state from
- * where this call originated.
- */
- rc = psci_get_ns_ep_info(ep, entrypoint, context_id);
- return rc;
- }
- /*******************************************************************************
- * Generic handler which is called when a cpu is physically powered on. It
- * traverses the node information and finds the highest power level powered
- * off and performs generic, architectural, platform setup and state management
- * to power on that power level and power levels below it.
- * e.g. For a cpu that's been powered on, it will call the platform specific
- * code to enable the gic cpu interface and for a cluster it will enable
- * coherency at the interconnect level in addition to gic cpu interface.
- ******************************************************************************/
- void psci_warmboot_entrypoint(void)
- {
- unsigned int end_pwrlvl;
- unsigned int cpu_idx = plat_my_core_pos();
- unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
- psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} };
- /*
- * Verify that we have been explicitly turned ON or resumed from
- * suspend.
- */
- if (psci_get_aff_info_state() == AFF_STATE_OFF) {
- ERROR("Unexpected affinity info state.\n");
- panic();
- }
- /*
- * Get the maximum power domain level to traverse to after this cpu
- * has been physically powered up.
- */
- end_pwrlvl = get_power_on_target_pwrlvl();
- /* Get the parent nodes */
- psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes);
- /*
- * This function acquires the lock corresponding to each power level so
- * that by the time all locks are taken, the system topology is snapshot
- * and state management can be done safely.
- */
- psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes);
- psci_get_target_local_pwr_states(end_pwrlvl, &state_info);
- #if ENABLE_PSCI_STAT
- plat_psci_stat_accounting_stop(&state_info);
- #endif
- /*
- * This CPU could be resuming from suspend or it could have just been
- * turned on. To distinguish between these 2 cases, we examine the
- * affinity state of the CPU:
- * - If the affinity state is ON_PENDING then it has just been
- * turned on.
- * - Else it is resuming from suspend.
- *
- * Depending on the type of warm reset identified, choose the right set
- * of power management handler and perform the generic, architecture
- * and platform specific handling.
- */
- if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING)
- psci_cpu_on_finish(cpu_idx, &state_info);
- else
- psci_cpu_suspend_finish(cpu_idx, &state_info);
- /*
- * Set the requested and target state of this CPU and all the higher
- * power domains which are ancestors of this CPU to run.
- */
- psci_set_pwr_domains_to_run(end_pwrlvl);
- #if ENABLE_PSCI_STAT
- /*
- * Update PSCI stats.
- * Caches are off when writing stats data on the power down path.
- * Since caches are now enabled, it's necessary to do cache
- * maintenance before reading that same data.
- */
- psci_stats_update_pwr_up(end_pwrlvl, &state_info);
- #endif
- /*
- * This loop releases the lock corresponding to each power level
- * in the reverse order to which they were acquired.
- */
- psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes);
- }
- /*******************************************************************************
- * This function initializes the set of hooks that PSCI invokes as part of power
- * management operation. The power management hooks are expected to be provided
- * by the SPD, after it finishes all its initialization
- ******************************************************************************/
- void psci_register_spd_pm_hook(const spd_pm_ops_t *pm)
- {
- assert(pm != NULL);
- psci_spd_pm = pm;
- if (pm->svc_migrate != NULL)
- psci_caps |= define_psci_cap(PSCI_MIG_AARCH64);
- if (pm->svc_migrate_info != NULL)
- psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64)
- | define_psci_cap(PSCI_MIG_INFO_TYPE);
- }
- /*******************************************************************************
- * This function invokes the migrate info hook in the spd_pm_ops. It performs
- * the necessary return value validation. If the Secure Payload is UP and
- * migrate capable, it returns the mpidr of the CPU on which the Secure payload
- * is resident through the mpidr parameter. Else the value of the parameter on
- * return is undefined.
- ******************************************************************************/
- int psci_spd_migrate_info(u_register_t *mpidr)
- {
- int rc;
- if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL))
- return PSCI_E_NOT_SUPPORTED;
- rc = psci_spd_pm->svc_migrate_info(mpidr);
- assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) ||
- (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED));
- return rc;
- }
- /*******************************************************************************
- * This function prints the state of all power domains present in the
- * system
- ******************************************************************************/
- void psci_print_power_domain_map(void)
- {
- #if LOG_LEVEL >= LOG_LEVEL_INFO
- unsigned int idx;
- plat_local_state_t state;
- plat_local_state_type_t state_type;
- /* This array maps to the PSCI_STATE_X definitions in psci.h */
- static const char * const psci_state_type_str[] = {
- "ON",
- "RETENTION",
- "OFF",
- };
- INFO("PSCI Power Domain Map:\n");
- for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count);
- idx++) {
- state_type = find_local_state_type(
- psci_non_cpu_pd_nodes[idx].local_state);
- INFO(" Domain Node : Level %u, parent_node %u,"
- " State %s (0x%x)\n",
- psci_non_cpu_pd_nodes[idx].level,
- psci_non_cpu_pd_nodes[idx].parent_node,
- psci_state_type_str[state_type],
- psci_non_cpu_pd_nodes[idx].local_state);
- }
- for (idx = 0; idx < psci_plat_core_count; idx++) {
- state = psci_get_cpu_local_state_by_idx(idx);
- state_type = find_local_state_type(state);
- INFO(" CPU Node : MPID 0x%llx, parent_node %u,"
- " State %s (0x%x)\n",
- (unsigned long long)psci_cpu_pd_nodes[idx].mpidr,
- psci_cpu_pd_nodes[idx].parent_node,
- psci_state_type_str[state_type],
- psci_get_cpu_local_state_by_idx(idx));
- }
- #endif
- }
- /******************************************************************************
- * Return whether any secondaries were powered up with CPU_ON call. A CPU that
- * have ever been powered up would have set its MPDIR value to something other
- * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to
- * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is
- * meaningful only when called on the primary CPU during early boot.
- *****************************************************************************/
- int psci_secondaries_brought_up(void)
- {
- unsigned int idx, n_valid = 0U;
- for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) {
- if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR)
- n_valid++;
- }
- assert(n_valid > 0U);
- return (n_valid > 1U) ? 1 : 0;
- }
- /*******************************************************************************
- * Initiate power down sequence, by calling power down operations registered for
- * this CPU.
- ******************************************************************************/
- void psci_pwrdown_cpu(unsigned int power_level)
- {
- #if HW_ASSISTED_COHERENCY
- /*
- * With hardware-assisted coherency, the CPU drivers only initiate the
- * power down sequence, without performing cache-maintenance operations
- * in software. Data caches enabled both before and after this call.
- */
- prepare_cpu_pwr_dwn(power_level);
- #else
- /*
- * Without hardware-assisted coherency, the CPU drivers disable data
- * caches, then perform cache-maintenance operations in software.
- *
- * This also calls prepare_cpu_pwr_dwn() to initiate power down
- * sequence, but that function will return with data caches disabled.
- * We must ensure that the stack memory is flushed out to memory before
- * we start popping from it again.
- */
- psci_do_pwrdown_cache_maintenance(power_level);
- #endif
- }
- /*******************************************************************************
- * This function invokes the callback 'stop_func()' with the 'mpidr' of each
- * online PE. Caller can pass suitable method to stop a remote core.
- *
- * 'wait_ms' is the timeout value in milliseconds for the other cores to
- * transition to power down state. Passing '0' makes it non-blocking.
- *
- * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the
- * given timeout.
- ******************************************************************************/
- int psci_stop_other_cores(unsigned int wait_ms,
- void (*stop_func)(u_register_t mpidr))
- {
- unsigned int idx, this_cpu_idx;
- this_cpu_idx = plat_my_core_pos();
- /* Invoke stop_func for each core */
- for (idx = 0U; idx < psci_plat_core_count; idx++) {
- /* skip current CPU */
- if (idx == this_cpu_idx) {
- continue;
- }
- /* Check if the CPU is ON */
- if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) {
- (*stop_func)(psci_cpu_pd_nodes[idx].mpidr);
- }
- }
- /* Need to wait for other cores to shutdown */
- if (wait_ms != 0U) {
- while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu())) {
- mdelay(1U);
- }
- if (!psci_is_last_on_cpu()) {
- WARN("Failed to stop all cores!\n");
- psci_print_power_domain_map();
- return PSCI_E_DENIED;
- }
- }
- return PSCI_E_SUCCESS;
- }
- /*******************************************************************************
- * This function verifies that all the other cores in the system have been
- * turned OFF and the current CPU is the last running CPU in the system.
- * Returns true if the current CPU is the last ON CPU or false otherwise.
- *
- * This API has following differences with psci_is_last_on_cpu
- * 1. PSCI states are locked
- ******************************************************************************/
- bool psci_is_last_on_cpu_safe(void)
- {
- unsigned int this_core = plat_my_core_pos();
- unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0};
- psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes);
- psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
- if (!psci_is_last_on_cpu()) {
- psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
- return false;
- }
- psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes);
- return true;
- }
|