/* * Copyright (c) 2013-2024, Arm Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "psci_private.h" /* * SPD power management operations, expected to be supplied by the registered * SPD on successful SP initialization */ const spd_pm_ops_t *psci_spd_pm; /* * PSCI requested local power state map. This array is used to store the local * power states requested by a CPU for power levels from level 1 to * PLAT_MAX_PWR_LVL. It does not store the requested local power state for power * level 0 (PSCI_CPU_PWR_LVL) as the requested and the target power state for a * CPU are the same. * * During state coordination, the platform is passed an array containing the * local states requested for a particular non cpu power domain by each cpu * within the domain. * * TODO: Dense packing of the requested states will cause cache thrashing * when multiple power domains write to it. If we allocate the requested * states at each power level in a cache-line aligned per-domain memory, * the cache thrashing can be avoided. */ static plat_local_state_t psci_req_local_pwr_states[PLAT_MAX_PWR_LVL][PLATFORM_CORE_COUNT]; unsigned int psci_plat_core_count; /******************************************************************************* * Arrays that hold the platform's power domain tree information for state * management of power domains. * Each node in the array 'psci_non_cpu_pd_nodes' corresponds to a power domain * which is an ancestor of a CPU power domain. * Each node in the array 'psci_cpu_pd_nodes' corresponds to a cpu power domain ******************************************************************************/ non_cpu_pd_node_t psci_non_cpu_pd_nodes[PSCI_NUM_NON_CPU_PWR_DOMAINS] #if USE_COHERENT_MEM __section(".tzfw_coherent_mem") #endif ; /* Lock for PSCI state coordination */ DEFINE_PSCI_LOCK(psci_locks[PSCI_NUM_NON_CPU_PWR_DOMAINS]); cpu_pd_node_t psci_cpu_pd_nodes[PLATFORM_CORE_COUNT]; /******************************************************************************* * Pointer to functions exported by the platform to complete power mgmt. ops ******************************************************************************/ const plat_psci_ops_t *psci_plat_pm_ops; /****************************************************************************** * Check that the maximum power level supported by the platform makes sense *****************************************************************************/ CASSERT((PLAT_MAX_PWR_LVL <= PSCI_MAX_PWR_LVL) && (PLAT_MAX_PWR_LVL >= PSCI_CPU_PWR_LVL), assert_platform_max_pwrlvl_check); #if PSCI_OS_INIT_MODE /******************************************************************************* * The power state coordination mode used in CPU_SUSPEND. * Defaults to platform-coordinated mode. ******************************************************************************/ suspend_mode_t psci_suspend_mode = PLAT_COORD; #endif /* * The plat_local_state used by the platform is one of these types: RUN, * RETENTION and OFF. The platform can define further sub-states for each type * apart from RUN. This categorization is done to verify the sanity of the * psci_power_state passed by the platform and to print debug information. The * categorization is done on the basis of the following conditions: * * 1. If (plat_local_state == 0) then the category is STATE_TYPE_RUN. * * 2. If (0 < plat_local_state <= PLAT_MAX_RET_STATE), then the category is * STATE_TYPE_RETN. * * 3. If (plat_local_state > PLAT_MAX_RET_STATE), then the category is * STATE_TYPE_OFF. */ typedef enum plat_local_state_type { STATE_TYPE_RUN = 0, STATE_TYPE_RETN, STATE_TYPE_OFF } plat_local_state_type_t; /* Function used to categorize plat_local_state. */ static plat_local_state_type_t find_local_state_type(plat_local_state_t state) { if (state != 0U) { if (state > PLAT_MAX_RET_STATE) { return STATE_TYPE_OFF; } else { return STATE_TYPE_RETN; } } else { return STATE_TYPE_RUN; } } /****************************************************************************** * Check that the maximum retention level supported by the platform is less * than the maximum off level. *****************************************************************************/ CASSERT(PLAT_MAX_RET_STATE < PLAT_MAX_OFF_STATE, assert_platform_max_off_and_retn_state_check); /****************************************************************************** * This function ensures that the power state parameter in a CPU_SUSPEND request * is valid. If so, it returns the requested states for each power level. *****************************************************************************/ int psci_validate_power_state(unsigned int power_state, psci_power_state_t *state_info) { /* Check SBZ bits in power state are zero */ if (psci_check_power_state(power_state) != 0U) return PSCI_E_INVALID_PARAMS; assert(psci_plat_pm_ops->validate_power_state != NULL); /* Validate the power_state using platform pm_ops */ return psci_plat_pm_ops->validate_power_state(power_state, state_info); } /****************************************************************************** * This function retrieves the `psci_power_state_t` for system suspend from * the platform. *****************************************************************************/ void psci_query_sys_suspend_pwrstate(psci_power_state_t *state_info) { /* * Assert that the required pm_ops hook is implemented to ensure that * the capability detected during psci_setup() is valid. */ assert(psci_plat_pm_ops->get_sys_suspend_power_state != NULL); /* * Query the platform for the power_state required for system suspend */ psci_plat_pm_ops->get_sys_suspend_power_state(state_info); } #if PSCI_OS_INIT_MODE /******************************************************************************* * This function verifies that all the other cores at the 'end_pwrlvl' have been * idled and the current CPU is the last running CPU at the 'end_pwrlvl'. * Returns 1 (true) if the current CPU is the last ON CPU or 0 (false) * otherwise. ******************************************************************************/ static bool psci_is_last_cpu_to_idle_at_pwrlvl(unsigned int end_pwrlvl) { unsigned int my_idx, lvl; unsigned int parent_idx = 0; unsigned int cpu_start_idx, ncpus, cpu_idx; plat_local_state_t local_state; if (end_pwrlvl == PSCI_CPU_PWR_LVL) { return true; } my_idx = plat_my_core_pos(); parent_idx = psci_cpu_pd_nodes[my_idx].parent_node; for (lvl = PSCI_CPU_PWR_LVL + U(1); lvl < end_pwrlvl; lvl++) { parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; } cpu_start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; for (cpu_idx = cpu_start_idx; cpu_idx < cpu_start_idx + ncpus; cpu_idx++) { local_state = psci_get_cpu_local_state_by_idx(cpu_idx); if (cpu_idx == my_idx) { assert(is_local_state_run(local_state) != 0); continue; } if (is_local_state_run(local_state) != 0) { return false; } } return true; } #endif /******************************************************************************* * This function verifies that all the other cores in the system have been * turned OFF and the current CPU is the last running CPU in the system. * Returns true, if the current CPU is the last ON CPU or false otherwise. ******************************************************************************/ bool psci_is_last_on_cpu(void) { unsigned int cpu_idx, my_idx = plat_my_core_pos(); for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) { if (cpu_idx == my_idx) { assert(psci_get_aff_info_state() == AFF_STATE_ON); continue; } if (psci_get_aff_info_state_by_idx(cpu_idx) != AFF_STATE_OFF) { VERBOSE("core=%u other than current core=%u %s\n", cpu_idx, my_idx, "running in the system"); return false; } } return true; } /******************************************************************************* * This function verifies that all cores in the system have been turned ON. * Returns true, if all CPUs are ON or false otherwise. ******************************************************************************/ static bool psci_are_all_cpus_on(void) { unsigned int cpu_idx; for (cpu_idx = 0; cpu_idx < psci_plat_core_count; cpu_idx++) { if (psci_get_aff_info_state_by_idx(cpu_idx) == AFF_STATE_OFF) { return false; } } return true; } /******************************************************************************* * Routine to return the maximum power level to traverse to after a cpu has * been physically powered up. It is expected to be called immediately after * reset from assembler code. ******************************************************************************/ static unsigned int get_power_on_target_pwrlvl(void) { unsigned int pwrlvl; /* * Assume that this cpu was suspended and retrieve its target power * level. If it is invalid then it could only have been turned off * earlier. PLAT_MAX_PWR_LVL will be the highest power level a * cpu can be turned off to. */ pwrlvl = psci_get_suspend_pwrlvl(); if (pwrlvl == PSCI_INVALID_PWR_LVL) pwrlvl = PLAT_MAX_PWR_LVL; assert(pwrlvl < PSCI_INVALID_PWR_LVL); return pwrlvl; } /****************************************************************************** * Helper function to update the requested local power state array. This array * does not store the requested state for the CPU power level. Hence an * assertion is added to prevent us from accessing the CPU power level. *****************************************************************************/ static void psci_set_req_local_pwr_state(unsigned int pwrlvl, unsigned int cpu_idx, plat_local_state_t req_pwr_state) { assert(pwrlvl > PSCI_CPU_PWR_LVL); if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && (cpu_idx < psci_plat_core_count)) { psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx] = req_pwr_state; } } /****************************************************************************** * This function initializes the psci_req_local_pwr_states. *****************************************************************************/ void __init psci_init_req_local_pwr_states(void) { /* Initialize the requested state of all non CPU power domains as OFF */ unsigned int pwrlvl; unsigned int core; for (pwrlvl = 0U; pwrlvl < PLAT_MAX_PWR_LVL; pwrlvl++) { for (core = 0; core < psci_plat_core_count; core++) { psci_req_local_pwr_states[pwrlvl][core] = PLAT_MAX_OFF_STATE; } } } /****************************************************************************** * Helper function to return a reference to an array containing the local power * states requested by each cpu for a power domain at 'pwrlvl'. The size of the * array will be the number of cpu power domains of which this power domain is * an ancestor. These requested states will be used to determine a suitable * target state for this power domain during psci state coordination. An * assertion is added to prevent us from accessing the CPU power level. *****************************************************************************/ static plat_local_state_t *psci_get_req_local_pwr_states(unsigned int pwrlvl, unsigned int cpu_idx) { assert(pwrlvl > PSCI_CPU_PWR_LVL); if ((pwrlvl > PSCI_CPU_PWR_LVL) && (pwrlvl <= PLAT_MAX_PWR_LVL) && (cpu_idx < psci_plat_core_count)) { return &psci_req_local_pwr_states[pwrlvl - 1U][cpu_idx]; } else return NULL; } #if PSCI_OS_INIT_MODE /****************************************************************************** * Helper function to save a copy of the psci_req_local_pwr_states (prev) for a * CPU (cpu_idx), and update psci_req_local_pwr_states with the new requested * local power states (state_info). *****************************************************************************/ void psci_update_req_local_pwr_states(unsigned int end_pwrlvl, unsigned int cpu_idx, psci_power_state_t *state_info, plat_local_state_t *prev) { unsigned int lvl; #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; #else unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL; #endif plat_local_state_t req_state; for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) { /* Save the previous requested local power state */ prev[lvl - 1U] = *psci_get_req_local_pwr_states(lvl, cpu_idx); /* Update the new requested local power state */ if (lvl <= end_pwrlvl) { req_state = state_info->pwr_domain_state[lvl]; } else { req_state = state_info->pwr_domain_state[end_pwrlvl]; } psci_set_req_local_pwr_state(lvl, cpu_idx, req_state); } } /****************************************************************************** * Helper function to restore the previously saved requested local power states * (prev) for a CPU (cpu_idx) to psci_req_local_pwr_states. *****************************************************************************/ void psci_restore_req_local_pwr_states(unsigned int cpu_idx, plat_local_state_t *prev) { unsigned int lvl; #ifdef PLAT_MAX_CPU_SUSPEND_PWR_LVL unsigned int max_pwrlvl = PLAT_MAX_CPU_SUSPEND_PWR_LVL; #else unsigned int max_pwrlvl = PLAT_MAX_PWR_LVL; #endif for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= max_pwrlvl; lvl++) { /* Restore the previous requested local power state */ psci_set_req_local_pwr_state(lvl, cpu_idx, prev[lvl - 1U]); } } #endif /* * psci_non_cpu_pd_nodes can be placed either in normal memory or coherent * memory. * * With !USE_COHERENT_MEM, psci_non_cpu_pd_nodes is placed in normal memory, * it's accessed by both cached and non-cached participants. To serve the common * minimum, perform a cache flush before read and after write so that non-cached * participants operate on latest data in main memory. * * When USE_COHERENT_MEM is used, psci_non_cpu_pd_nodes is placed in coherent * memory. With HW_ASSISTED_COHERENCY, all PSCI participants are cache-coherent. * In both cases, no cache operations are required. */ /* * Retrieve local state of non-CPU power domain node from a non-cached CPU, * after any required cache maintenance operation. */ static plat_local_state_t get_non_cpu_pd_node_local_state( unsigned int parent_idx) { #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) flush_dcache_range( (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], sizeof(psci_non_cpu_pd_nodes[parent_idx])); #endif return psci_non_cpu_pd_nodes[parent_idx].local_state; } /* * Update local state of non-CPU power domain node from a cached CPU; perform * any required cache maintenance operation afterwards. */ static void set_non_cpu_pd_node_local_state(unsigned int parent_idx, plat_local_state_t state) { psci_non_cpu_pd_nodes[parent_idx].local_state = state; #if !(USE_COHERENT_MEM || HW_ASSISTED_COHERENCY || WARMBOOT_ENABLE_DCACHE_EARLY) flush_dcache_range( (uintptr_t) &psci_non_cpu_pd_nodes[parent_idx], sizeof(psci_non_cpu_pd_nodes[parent_idx])); #endif } /****************************************************************************** * Helper function to return the current local power state of each power domain * from the current cpu power domain to its ancestor at the 'end_pwrlvl'. This * function will be called after a cpu is powered on to find the local state * each power domain has emerged from. *****************************************************************************/ void psci_get_target_local_pwr_states(unsigned int end_pwrlvl, psci_power_state_t *target_state) { unsigned int parent_idx, lvl; plat_local_state_t *pd_state = target_state->pwr_domain_state; pd_state[PSCI_CPU_PWR_LVL] = psci_get_cpu_local_state(); parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; /* Copy the local power state from node to state_info */ for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { pd_state[lvl] = get_non_cpu_pd_node_local_state(parent_idx); parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; } /* Set the the higher levels to RUN */ for (; lvl <= PLAT_MAX_PWR_LVL; lvl++) target_state->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; } /****************************************************************************** * Helper function to set the target local power state that each power domain * from the current cpu power domain to its ancestor at the 'end_pwrlvl' will * enter. This function will be called after coordination of requested power * states has been done for each power level. *****************************************************************************/ void psci_set_target_local_pwr_states(unsigned int end_pwrlvl, const psci_power_state_t *target_state) { unsigned int parent_idx, lvl; const plat_local_state_t *pd_state = target_state->pwr_domain_state; psci_set_cpu_local_state(pd_state[PSCI_CPU_PWR_LVL]); /* * Need to flush as local_state might be accessed with Data Cache * disabled during power on */ psci_flush_cpu_data(psci_svc_cpu_data.local_state); parent_idx = psci_cpu_pd_nodes[plat_my_core_pos()].parent_node; /* Copy the local_state from state_info */ for (lvl = 1U; lvl <= end_pwrlvl; lvl++) { set_non_cpu_pd_node_local_state(parent_idx, pd_state[lvl]); parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; } } /******************************************************************************* * PSCI helper function to get the parent nodes corresponding to a cpu_index. ******************************************************************************/ void psci_get_parent_pwr_domain_nodes(unsigned int cpu_idx, unsigned int end_lvl, unsigned int *node_index) { unsigned int parent_node = psci_cpu_pd_nodes[cpu_idx].parent_node; unsigned int i; unsigned int *node = node_index; for (i = PSCI_CPU_PWR_LVL + 1U; i <= end_lvl; i++) { *node = parent_node; node++; parent_node = psci_non_cpu_pd_nodes[parent_node].parent_node; } } /****************************************************************************** * This function is invoked post CPU power up and initialization. It sets the * affinity info state, target power state and requested power state for the * current CPU and all its ancestor power domains to RUN. *****************************************************************************/ void psci_set_pwr_domains_to_run(unsigned int end_pwrlvl) { unsigned int parent_idx, cpu_idx = plat_my_core_pos(), lvl; parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; /* Reset the local_state to RUN for the non cpu power domains. */ for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { set_non_cpu_pd_node_local_state(parent_idx, PSCI_LOCAL_STATE_RUN); psci_set_req_local_pwr_state(lvl, cpu_idx, PSCI_LOCAL_STATE_RUN); parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; } /* Set the affinity info state to ON */ psci_set_aff_info_state(AFF_STATE_ON); psci_set_cpu_local_state(PSCI_LOCAL_STATE_RUN); psci_flush_cpu_data(psci_svc_cpu_data); } /****************************************************************************** * This function is used in platform-coordinated mode. * * This function is passed the local power states requested for each power * domain (state_info) between the current CPU domain and its ancestors until * the target power level (end_pwrlvl). It updates the array of requested power * states with this information. * * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it * retrieves the states requested by all the cpus of which the power domain at * that level is an ancestor. It passes this information to the platform to * coordinate and return the target power state. If the target state for a level * is RUN then subsequent levels are not considered. At the CPU level, state * coordination is not required. Hence, the requested and the target states are * the same. * * The 'state_info' is updated with the target state for each level between the * CPU and the 'end_pwrlvl' and returned to the caller. * * This function will only be invoked with data cache enabled and while * powering down a core. *****************************************************************************/ void psci_do_state_coordination(unsigned int end_pwrlvl, psci_power_state_t *state_info) { unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); unsigned int start_idx; unsigned int ncpus; plat_local_state_t target_state, *req_states; assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; /* For level 0, the requested state will be equivalent to target state */ for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { /* First update the requested power state */ psci_set_req_local_pwr_state(lvl, cpu_idx, state_info->pwr_domain_state[lvl]); /* Get the requested power states for this power level */ start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; req_states = psci_get_req_local_pwr_states(lvl, start_idx); /* * Let the platform coordinate amongst the requested states at * this power level and return the target local power state. */ ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; target_state = plat_get_target_pwr_state(lvl, req_states, ncpus); state_info->pwr_domain_state[lvl] = target_state; /* Break early if the negotiated target power state is RUN */ if (is_local_state_run(state_info->pwr_domain_state[lvl]) != 0) break; parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; } /* * This is for cases when we break out of the above loop early because * the target power state is RUN at a power level < end_pwlvl. * We update the requested power state from state_info and then * set the target state as RUN. */ for (lvl = lvl + 1U; lvl <= end_pwrlvl; lvl++) { psci_set_req_local_pwr_state(lvl, cpu_idx, state_info->pwr_domain_state[lvl]); state_info->pwr_domain_state[lvl] = PSCI_LOCAL_STATE_RUN; } } #if PSCI_OS_INIT_MODE /****************************************************************************** * This function is used in OS-initiated mode. * * This function is passed the local power states requested for each power * domain (state_info) between the current CPU domain and its ancestors until * the target power level (end_pwrlvl), and ensures the requested power states * are valid. It updates the array of requested power states with this * information. * * Then, for each level (apart from the CPU level) until the 'end_pwrlvl', it * retrieves the states requested by all the cpus of which the power domain at * that level is an ancestor. It passes this information to the platform to * coordinate and return the target power state. If the requested state does * not match the target state, the request is denied. * * The 'state_info' is not modified. * * This function will only be invoked with data cache enabled and while * powering down a core. *****************************************************************************/ int psci_validate_state_coordination(unsigned int end_pwrlvl, psci_power_state_t *state_info) { int rc = PSCI_E_SUCCESS; unsigned int lvl, parent_idx, cpu_idx = plat_my_core_pos(); unsigned int start_idx; unsigned int ncpus; plat_local_state_t target_state, *req_states; plat_local_state_t prev[PLAT_MAX_PWR_LVL]; assert(end_pwrlvl <= PLAT_MAX_PWR_LVL); parent_idx = psci_cpu_pd_nodes[cpu_idx].parent_node; /* * Save a copy of the previous requested local power states and update * the new requested local power states. */ psci_update_req_local_pwr_states(end_pwrlvl, cpu_idx, state_info, prev); for (lvl = PSCI_CPU_PWR_LVL + 1U; lvl <= end_pwrlvl; lvl++) { /* Get the requested power states for this power level */ start_idx = psci_non_cpu_pd_nodes[parent_idx].cpu_start_idx; req_states = psci_get_req_local_pwr_states(lvl, start_idx); /* * Let the platform coordinate amongst the requested states at * this power level and return the target local power state. */ ncpus = psci_non_cpu_pd_nodes[parent_idx].ncpus; target_state = plat_get_target_pwr_state(lvl, req_states, ncpus); /* * Verify that the requested power state matches the target * local power state. */ if (state_info->pwr_domain_state[lvl] != target_state) { if (target_state == PSCI_LOCAL_STATE_RUN) { rc = PSCI_E_DENIED; } else { rc = PSCI_E_INVALID_PARAMS; } goto exit; } parent_idx = psci_non_cpu_pd_nodes[parent_idx].parent_node; } /* * Verify that the current core is the last running core at the * specified power level. */ lvl = state_info->last_at_pwrlvl; if (!psci_is_last_cpu_to_idle_at_pwrlvl(lvl)) { rc = PSCI_E_DENIED; } exit: if (rc != PSCI_E_SUCCESS) { /* Restore the previous requested local power states. */ psci_restore_req_local_pwr_states(cpu_idx, prev); return rc; } return rc; } #endif /****************************************************************************** * This function validates a suspend request by making sure that if a standby * state is requested then no power level is turned off and the highest power * level is placed in a standby/retention state. * * It also ensures that the state level X will enter is not shallower than the * state level X + 1 will enter. * * This validation will be enabled only for DEBUG builds as the platform is * expected to perform these validations as well. *****************************************************************************/ int psci_validate_suspend_req(const psci_power_state_t *state_info, unsigned int is_power_down_state) { unsigned int max_off_lvl, target_lvl, max_retn_lvl; plat_local_state_t state; plat_local_state_type_t req_state_type, deepest_state_type; int i; /* Find the target suspend power level */ target_lvl = psci_find_target_suspend_lvl(state_info); if (target_lvl == PSCI_INVALID_PWR_LVL) return PSCI_E_INVALID_PARAMS; /* All power domain levels are in a RUN state to begin with */ deepest_state_type = STATE_TYPE_RUN; for (i = (int) target_lvl; i >= (int) PSCI_CPU_PWR_LVL; i--) { state = state_info->pwr_domain_state[i]; req_state_type = find_local_state_type(state); /* * While traversing from the highest power level to the lowest, * the state requested for lower levels has to be the same or * deeper i.e. equal to or greater than the state at the higher * levels. If this condition is true, then the requested state * becomes the deepest state encountered so far. */ if (req_state_type < deepest_state_type) return PSCI_E_INVALID_PARAMS; deepest_state_type = req_state_type; } /* Find the highest off power level */ max_off_lvl = psci_find_max_off_lvl(state_info); /* The target_lvl is either equal to the max_off_lvl or max_retn_lvl */ max_retn_lvl = PSCI_INVALID_PWR_LVL; if (target_lvl != max_off_lvl) max_retn_lvl = target_lvl; /* * If this is not a request for a power down state then max off level * has to be invalid and max retention level has to be a valid power * level. */ if ((is_power_down_state == 0U) && ((max_off_lvl != PSCI_INVALID_PWR_LVL) || (max_retn_lvl == PSCI_INVALID_PWR_LVL))) return PSCI_E_INVALID_PARAMS; return PSCI_E_SUCCESS; } /****************************************************************************** * This function finds the highest power level which will be powered down * amongst all the power levels specified in the 'state_info' structure *****************************************************************************/ unsigned int psci_find_max_off_lvl(const psci_power_state_t *state_info) { int i; for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { if (is_local_state_off(state_info->pwr_domain_state[i]) != 0) return (unsigned int) i; } return PSCI_INVALID_PWR_LVL; } /****************************************************************************** * This functions finds the level of the highest power domain which will be * placed in a low power state during a suspend operation. *****************************************************************************/ unsigned int psci_find_target_suspend_lvl(const psci_power_state_t *state_info) { int i; for (i = (int) PLAT_MAX_PWR_LVL; i >= (int) PSCI_CPU_PWR_LVL; i--) { if (is_local_state_run(state_info->pwr_domain_state[i]) == 0) return (unsigned int) i; } return PSCI_INVALID_PWR_LVL; } /******************************************************************************* * This function is passed the highest level in the topology tree that the * operation should be applied to and a list of node indexes. It picks up locks * from the node index list in order of increasing power domain level in the * range specified. ******************************************************************************/ void psci_acquire_pwr_domain_locks(unsigned int end_pwrlvl, const unsigned int *parent_nodes) { unsigned int parent_idx; unsigned int level; /* No locking required for level 0. Hence start locking from level 1 */ for (level = PSCI_CPU_PWR_LVL + 1U; level <= end_pwrlvl; level++) { parent_idx = parent_nodes[level - 1U]; psci_lock_get(&psci_non_cpu_pd_nodes[parent_idx]); } } /******************************************************************************* * This function is passed the highest level in the topology tree that the * operation should be applied to and a list of node indexes. It releases the * locks in order of decreasing power domain level in the range specified. ******************************************************************************/ void psci_release_pwr_domain_locks(unsigned int end_pwrlvl, const unsigned int *parent_nodes) { unsigned int parent_idx; unsigned int level; /* Unlock top down. No unlocking required for level 0. */ for (level = end_pwrlvl; level >= (PSCI_CPU_PWR_LVL + 1U); level--) { parent_idx = parent_nodes[level - 1U]; psci_lock_release(&psci_non_cpu_pd_nodes[parent_idx]); } } /******************************************************************************* * This function determines the full entrypoint information for the requested * PSCI entrypoint on power on/resume and returns it. ******************************************************************************/ #ifdef __aarch64__ static int psci_get_ns_ep_info(entry_point_info_t *ep, uintptr_t entrypoint, u_register_t context_id) { u_register_t ep_attr, sctlr; unsigned int daif, ee, mode; u_register_t ns_scr_el3 = read_scr_el3(); u_register_t ns_sctlr_el1 = read_sctlr_el1(); sctlr = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? read_sctlr_el2() : ns_sctlr_el1; ee = 0; ep_attr = NON_SECURE | EP_ST_DISABLE; if ((sctlr & SCTLR_EE_BIT) != 0U) { ep_attr |= EP_EE_BIG; ee = 1; } SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); ep->pc = entrypoint; zeromem(&ep->args, sizeof(ep->args)); ep->args.arg0 = context_id; /* * Figure out whether the cpu enters the non-secure address space * in aarch32 or aarch64 */ if ((ns_scr_el3 & SCR_RW_BIT) != 0U) { /* * Check whether a Thumb entry point has been provided for an * aarch64 EL */ if ((entrypoint & 0x1UL) != 0UL) return PSCI_E_INVALID_ADDRESS; mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE_EL2 : MODE_EL1; ep->spsr = SPSR_64((uint64_t)mode, MODE_SP_ELX, DISABLE_ALL_EXCEPTIONS); } else { mode = ((ns_scr_el3 & SCR_HCE_BIT) != 0U) ? MODE32_hyp : MODE32_svc; /* * TODO: Choose async. exception bits if HYP mode is not * implemented according to the values of SCR.{AW, FW} bits */ daif = DAIF_ABT_BIT | DAIF_IRQ_BIT | DAIF_FIQ_BIT; ep->spsr = SPSR_MODE32((uint64_t)mode, entrypoint & 0x1, ee, daif); } return PSCI_E_SUCCESS; } #else /* !__aarch64__ */ static int psci_get_ns_ep_info(entry_point_info_t *ep, uintptr_t entrypoint, u_register_t context_id) { u_register_t ep_attr; unsigned int aif, ee, mode; u_register_t scr = read_scr(); u_register_t ns_sctlr, sctlr; /* Switch to non secure state */ write_scr(scr | SCR_NS_BIT); isb(); ns_sctlr = read_sctlr(); sctlr = scr & SCR_HCE_BIT ? read_hsctlr() : ns_sctlr; /* Return to original state */ write_scr(scr); isb(); ee = 0; ep_attr = NON_SECURE | EP_ST_DISABLE; if (sctlr & SCTLR_EE_BIT) { ep_attr |= EP_EE_BIG; ee = 1; } SET_PARAM_HEAD(ep, PARAM_EP, VERSION_1, ep_attr); ep->pc = entrypoint; zeromem(&ep->args, sizeof(ep->args)); ep->args.arg0 = context_id; mode = scr & SCR_HCE_BIT ? MODE32_hyp : MODE32_svc; /* * TODO: Choose async. exception bits if HYP mode is not * implemented according to the values of SCR.{AW, FW} bits */ aif = SPSR_ABT_BIT | SPSR_IRQ_BIT | SPSR_FIQ_BIT; ep->spsr = SPSR_MODE32(mode, entrypoint & 0x1, ee, aif); return PSCI_E_SUCCESS; } #endif /* __aarch64__ */ /******************************************************************************* * This function validates the entrypoint with the platform layer if the * appropriate pm_ops hook is exported by the platform and returns the * 'entry_point_info'. ******************************************************************************/ int psci_validate_entry_point(entry_point_info_t *ep, uintptr_t entrypoint, u_register_t context_id) { int rc; /* Validate the entrypoint using platform psci_ops */ if (psci_plat_pm_ops->validate_ns_entrypoint != NULL) { rc = psci_plat_pm_ops->validate_ns_entrypoint(entrypoint); if (rc != PSCI_E_SUCCESS) return PSCI_E_INVALID_ADDRESS; } /* * Verify and derive the re-entry information for * the non-secure world from the non-secure state from * where this call originated. */ rc = psci_get_ns_ep_info(ep, entrypoint, context_id); return rc; } /******************************************************************************* * Generic handler which is called when a cpu is physically powered on. It * traverses the node information and finds the highest power level powered * off and performs generic, architectural, platform setup and state management * to power on that power level and power levels below it. * e.g. For a cpu that's been powered on, it will call the platform specific * code to enable the gic cpu interface and for a cluster it will enable * coherency at the interconnect level in addition to gic cpu interface. ******************************************************************************/ void psci_warmboot_entrypoint(void) { unsigned int end_pwrlvl; unsigned int cpu_idx = plat_my_core_pos(); unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; psci_power_state_t state_info = { {PSCI_LOCAL_STATE_RUN} }; /* Init registers that never change for the lifetime of TF-A */ cm_manage_extensions_el3(); /* * Verify that we have been explicitly turned ON or resumed from * suspend. */ if (psci_get_aff_info_state() == AFF_STATE_OFF) { ERROR("Unexpected affinity info state.\n"); panic(); } /* * Get the maximum power domain level to traverse to after this cpu * has been physically powered up. */ end_pwrlvl = get_power_on_target_pwrlvl(); /* Get the parent nodes */ psci_get_parent_pwr_domain_nodes(cpu_idx, end_pwrlvl, parent_nodes); /* * This function acquires the lock corresponding to each power level so * that by the time all locks are taken, the system topology is snapshot * and state management can be done safely. */ psci_acquire_pwr_domain_locks(end_pwrlvl, parent_nodes); psci_get_target_local_pwr_states(end_pwrlvl, &state_info); #if ENABLE_PSCI_STAT plat_psci_stat_accounting_stop(&state_info); #endif /* * This CPU could be resuming from suspend or it could have just been * turned on. To distinguish between these 2 cases, we examine the * affinity state of the CPU: * - If the affinity state is ON_PENDING then it has just been * turned on. * - Else it is resuming from suspend. * * Depending on the type of warm reset identified, choose the right set * of power management handler and perform the generic, architecture * and platform specific handling. */ if (psci_get_aff_info_state() == AFF_STATE_ON_PENDING) psci_cpu_on_finish(cpu_idx, &state_info); else psci_cpu_suspend_finish(cpu_idx, &state_info); /* * Generic management: Now we just need to retrieve the * information that we had stashed away during the cpu_on * call to set this cpu on its way. */ cm_prepare_el3_exit_ns(); /* * Set the requested and target state of this CPU and all the higher * power domains which are ancestors of this CPU to run. */ psci_set_pwr_domains_to_run(end_pwrlvl); #if ENABLE_PSCI_STAT /* * Update PSCI stats. * Caches are off when writing stats data on the power down path. * Since caches are now enabled, it's necessary to do cache * maintenance before reading that same data. */ psci_stats_update_pwr_up(end_pwrlvl, &state_info); #endif /* * This loop releases the lock corresponding to each power level * in the reverse order to which they were acquired. */ psci_release_pwr_domain_locks(end_pwrlvl, parent_nodes); } /******************************************************************************* * This function initializes the set of hooks that PSCI invokes as part of power * management operation. The power management hooks are expected to be provided * by the SPD, after it finishes all its initialization ******************************************************************************/ void psci_register_spd_pm_hook(const spd_pm_ops_t *pm) { assert(pm != NULL); psci_spd_pm = pm; if (pm->svc_migrate != NULL) psci_caps |= define_psci_cap(PSCI_MIG_AARCH64); if (pm->svc_migrate_info != NULL) psci_caps |= define_psci_cap(PSCI_MIG_INFO_UP_CPU_AARCH64) | define_psci_cap(PSCI_MIG_INFO_TYPE); } /******************************************************************************* * This function invokes the migrate info hook in the spd_pm_ops. It performs * the necessary return value validation. If the Secure Payload is UP and * migrate capable, it returns the mpidr of the CPU on which the Secure payload * is resident through the mpidr parameter. Else the value of the parameter on * return is undefined. ******************************************************************************/ int psci_spd_migrate_info(u_register_t *mpidr) { int rc; if ((psci_spd_pm == NULL) || (psci_spd_pm->svc_migrate_info == NULL)) return PSCI_E_NOT_SUPPORTED; rc = psci_spd_pm->svc_migrate_info(mpidr); assert((rc == PSCI_TOS_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_UP_MIG_CAP) || (rc == PSCI_TOS_NOT_PRESENT_MP) || (rc == PSCI_E_NOT_SUPPORTED)); return rc; } /******************************************************************************* * This function prints the state of all power domains present in the * system ******************************************************************************/ void psci_print_power_domain_map(void) { #if LOG_LEVEL >= LOG_LEVEL_INFO unsigned int idx; plat_local_state_t state; plat_local_state_type_t state_type; /* This array maps to the PSCI_STATE_X definitions in psci.h */ static const char * const psci_state_type_str[] = { "ON", "RETENTION", "OFF", }; INFO("PSCI Power Domain Map:\n"); for (idx = 0; idx < (PSCI_NUM_PWR_DOMAINS - psci_plat_core_count); idx++) { state_type = find_local_state_type( psci_non_cpu_pd_nodes[idx].local_state); INFO(" Domain Node : Level %u, parent_node %u," " State %s (0x%x)\n", psci_non_cpu_pd_nodes[idx].level, psci_non_cpu_pd_nodes[idx].parent_node, psci_state_type_str[state_type], psci_non_cpu_pd_nodes[idx].local_state); } for (idx = 0; idx < psci_plat_core_count; idx++) { state = psci_get_cpu_local_state_by_idx(idx); state_type = find_local_state_type(state); INFO(" CPU Node : MPID 0x%llx, parent_node %u," " State %s (0x%x)\n", (unsigned long long)psci_cpu_pd_nodes[idx].mpidr, psci_cpu_pd_nodes[idx].parent_node, psci_state_type_str[state_type], psci_get_cpu_local_state_by_idx(idx)); } #endif } /****************************************************************************** * Return whether any secondaries were powered up with CPU_ON call. A CPU that * have ever been powered up would have set its MPDIR value to something other * than PSCI_INVALID_MPIDR. Note that MPDIR isn't reset back to * PSCI_INVALID_MPIDR when a CPU is powered down later, so the return value is * meaningful only when called on the primary CPU during early boot. *****************************************************************************/ int psci_secondaries_brought_up(void) { unsigned int idx, n_valid = 0U; for (idx = 0U; idx < ARRAY_SIZE(psci_cpu_pd_nodes); idx++) { if (psci_cpu_pd_nodes[idx].mpidr != PSCI_INVALID_MPIDR) n_valid++; } assert(n_valid > 0U); return (n_valid > 1U) ? 1 : 0; } /******************************************************************************* * Initiate power down sequence, by calling power down operations registered for * this CPU. ******************************************************************************/ void psci_pwrdown_cpu(unsigned int power_level) { psci_do_manage_extensions(); #if HW_ASSISTED_COHERENCY /* * With hardware-assisted coherency, the CPU drivers only initiate the * power down sequence, without performing cache-maintenance operations * in software. Data caches enabled both before and after this call. */ prepare_cpu_pwr_dwn(power_level); #else /* * Without hardware-assisted coherency, the CPU drivers disable data * caches, then perform cache-maintenance operations in software. * * This also calls prepare_cpu_pwr_dwn() to initiate power down * sequence, but that function will return with data caches disabled. * We must ensure that the stack memory is flushed out to memory before * we start popping from it again. */ psci_do_pwrdown_cache_maintenance(power_level); #endif } /******************************************************************************* * This function invokes the callback 'stop_func()' with the 'mpidr' of each * online PE. Caller can pass suitable method to stop a remote core. * * 'wait_ms' is the timeout value in milliseconds for the other cores to * transition to power down state. Passing '0' makes it non-blocking. * * The function returns 'PSCI_E_DENIED' if some cores failed to stop within the * given timeout. ******************************************************************************/ int psci_stop_other_cores(unsigned int wait_ms, void (*stop_func)(u_register_t mpidr)) { unsigned int idx, this_cpu_idx; this_cpu_idx = plat_my_core_pos(); /* Invoke stop_func for each core */ for (idx = 0U; idx < psci_plat_core_count; idx++) { /* skip current CPU */ if (idx == this_cpu_idx) { continue; } /* Check if the CPU is ON */ if (psci_get_aff_info_state_by_idx(idx) == AFF_STATE_ON) { (*stop_func)(psci_cpu_pd_nodes[idx].mpidr); } } /* Need to wait for other cores to shutdown */ if (wait_ms != 0U) { while ((wait_ms-- != 0U) && (!psci_is_last_on_cpu())) { mdelay(1U); } if (!psci_is_last_on_cpu()) { WARN("Failed to stop all cores!\n"); psci_print_power_domain_map(); return PSCI_E_DENIED; } } return PSCI_E_SUCCESS; } /******************************************************************************* * This function verifies that all the other cores in the system have been * turned OFF and the current CPU is the last running CPU in the system. * Returns true if the current CPU is the last ON CPU or false otherwise. * * This API has following differences with psci_is_last_on_cpu * 1. PSCI states are locked ******************************************************************************/ bool psci_is_last_on_cpu_safe(void) { unsigned int this_core = plat_my_core_pos(); unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); if (!psci_is_last_on_cpu()) { psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); return false; } psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); return true; } /******************************************************************************* * This function verifies that all cores in the system have been turned ON. * Returns true, if all CPUs are ON or false otherwise. * * This API has following differences with psci_are_all_cpus_on * 1. PSCI states are locked ******************************************************************************/ bool psci_are_all_cpus_on_safe(void) { unsigned int this_core = plat_my_core_pos(); unsigned int parent_nodes[PLAT_MAX_PWR_LVL] = {0}; psci_get_parent_pwr_domain_nodes(this_core, PLAT_MAX_PWR_LVL, parent_nodes); psci_acquire_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); if (!psci_are_all_cpus_on()) { psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); return false; } psci_release_pwr_domain_locks(PLAT_MAX_PWR_LVL, parent_nodes); return true; } /******************************************************************************* * This function performs architectural feature specific management. * It ensures the architectural features are disabled during cpu * power off/suspend operations. ******************************************************************************/ void psci_do_manage_extensions(void) { /* * On power down we need to disable statistical profiling extensions * before exiting coherency. */ if (is_feat_spe_supported()) { spe_stop(); } }