| // Copyright (c) 2022 Intel Corporation |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "cpuinfo_utils.hpp" |
| #include "speed_select.hpp" |
| |
| #include <iostream> |
| |
| namespace cpu_info |
| { |
| namespace sst |
| { |
| |
| /** |
| * Convenience RAII object for Wake-On-PECI (WOP) management, since PECI Config |
| * Local accesses to the OS Mailbox require the package to pop up to PC2. Also |
| * provides PCode OS Mailbox routine. |
| * |
| * Since multiple applications may be modifing WOP, we'll use this algorithm: |
| * Whenever a PECI command fails with associated error code, set WOP bit and |
| * retry command. Upon manager destruction, clear WOP bit only if we previously |
| * set it. |
| */ |
| struct PECIManager |
| { |
| uint8_t peciAddress; |
| bool peciWoken; |
| CPUModel cpuModel; |
| uint8_t mbBus; |
| WakePolicy wakePolicy; |
| |
| PECIManager(uint8_t address, CPUModel model, WakePolicy wakePolicy_) : |
| peciAddress(address), peciWoken(false), cpuModel(model), |
| wakePolicy(wakePolicy_) |
| { |
| mbBus = (model == iceLake) ? mbBusIceLake : mbBusOther; |
| } |
| |
| ~PECIManager() |
| { |
| // If we're being destroyed due to a PECIError, try to clear the mode |
| // bit, but catch and ignore any duplicate error it might raise to |
| // prevent termination. |
| try |
| { |
| if (peciWoken) |
| { |
| setWakeOnPECI(false); |
| } |
| } |
| catch (const PECIError& err) |
| {} |
| } |
| |
| static bool isSleeping(EPECIStatus libStatus, uint8_t completionCode) |
| { |
| // PECI completion code defined in peci-ioctl.h which is not available |
| // for us to include. |
| constexpr int PECI_DEV_CC_UNAVAIL_RESOURCE = 0x82; |
| // Observed library returning DRIVER_ERR for reads and TIMEOUT for |
| // writes while PECI is sleeping. Either way, the completion code from |
| // PECI client should be reliable indicator of need to set WOP. |
| return libStatus != PECI_CC_SUCCESS && |
| completionCode == PECI_DEV_CC_UNAVAIL_RESOURCE; |
| } |
| |
| /** |
| * Send a single PECI PCS write to modify the Wake-On-PECI mode bit |
| */ |
| void setWakeOnPECI(bool enable) |
| { |
| uint8_t completionCode; |
| EPECIStatus libStatus = peci_WrPkgConfig(peciAddress, 5, enable ? 1 : 0, |
| 0, sizeof(uint32_t), |
| &completionCode); |
| if (!checkPECIStatus(libStatus, completionCode)) |
| { |
| throw PECIError("Failed to set Wake-On-PECI mode bit"); |
| } |
| |
| if (enable) |
| { |
| peciWoken = true; |
| } |
| } |
| |
| // PCode OS Mailbox interface register locations |
| static constexpr int mbBusIceLake = 14; |
| static constexpr int mbBusOther = 31; |
| static constexpr int mbSegment = 0; |
| static constexpr int mbDevice = 30; |
| static constexpr int mbFunction = 1; |
| static constexpr int mbDataReg = 0xA0; |
| static constexpr int mbInterfaceReg = 0xA4; |
| static constexpr int mbRegSize = sizeof(uint32_t); |
| |
| enum class MailboxStatus |
| { |
| NoError = 0x0, |
| InvalidCommand = 0x1, |
| IllegalData = 0x16 |
| }; |
| |
| /** |
| * Send a single Write PCI Config Local command, targeting the PCU CR1 |
| * register block. |
| * |
| * @param[in] regAddress PCI Offset of register. |
| * @param[in] data Data to write. |
| */ |
| void wrMailboxReg(uint16_t regAddress, uint32_t data) |
| { |
| uint8_t completionCode; |
| bool tryWaking = (wakePolicy == wakeAllowed); |
| while (true) |
| { |
| EPECIStatus libStatus = peci_WrEndPointPCIConfigLocal( |
| peciAddress, mbSegment, mbBus, mbDevice, mbFunction, regAddress, |
| mbRegSize, data, &completionCode); |
| if (tryWaking && isSleeping(libStatus, completionCode)) |
| { |
| setWakeOnPECI(true); |
| tryWaking = false; |
| continue; |
| } |
| else if (!checkPECIStatus(libStatus, completionCode)) |
| { |
| throw PECIError("Failed to write mailbox reg"); |
| } |
| break; |
| } |
| } |
| |
| /** |
| * Send a single Read PCI Config Local command, targeting the PCU CR1 |
| * register block. |
| * |
| * @param[in] regAddress PCI offset of register. |
| * |
| * @return Register value |
| */ |
| uint32_t rdMailboxReg(uint16_t regAddress) |
| { |
| uint8_t completionCode; |
| uint32_t outputData; |
| bool tryWaking = (wakePolicy == wakeAllowed); |
| while (true) |
| { |
| EPECIStatus libStatus = peci_RdEndPointConfigPciLocal( |
| peciAddress, mbSegment, mbBus, mbDevice, mbFunction, regAddress, |
| mbRegSize, reinterpret_cast<uint8_t*>(&outputData), |
| &completionCode); |
| if (tryWaking && isSleeping(libStatus, completionCode)) |
| { |
| setWakeOnPECI(true); |
| tryWaking = false; |
| continue; |
| } |
| if (!checkPECIStatus(libStatus, completionCode)) |
| { |
| throw PECIError("Failed to read mailbox reg"); |
| } |
| break; |
| } |
| return outputData; |
| } |
| |
| /** |
| * Send command on PCode OS Mailbox interface. |
| * |
| * @param[in] command Main command ID. |
| * @param[in] subCommand Sub command ID. |
| * @param[in] inputData Data to put in mailbox. Is always written, but |
| * will be ignored by PCode if command is a |
| * "getter". |
| * @param[out] responseCode Optional parameter to receive the |
| * mailbox-level response status. If null, a |
| * PECIError will be thrown for error status. |
| * |
| * @return Data returned in mailbox. Value is undefined if command is a |
| * "setter". |
| */ |
| uint32_t sendPECIOSMailboxCmd(uint8_t command, uint8_t subCommand, |
| uint32_t inputData = 0, |
| MailboxStatus* responseCode = nullptr) |
| { |
| // The simple mailbox algorithm just says to wait until the busy bit |
| // is clear, but we'll give up after 10 tries. It's arbitrary but that's |
| // quite long wall clock time. |
| constexpr int mbRetries = 10; |
| constexpr uint32_t mbBusyBit = bit(31); |
| |
| // Wait until RUN_BUSY == 0 |
| int attempts = mbRetries; |
| while ((rdMailboxReg(mbInterfaceReg) & mbBusyBit) != 0 && |
| --attempts > 0) |
| ; |
| if (attempts == 0) |
| { |
| throw PECIError("OS Mailbox failed to become free"); |
| } |
| |
| // Write required command specific input data to data register |
| wrMailboxReg(mbDataReg, inputData); |
| |
| // Write required command specific command/sub-command values and set |
| // RUN_BUSY bit in interface register. |
| uint32_t interfaceReg = |
| mbBusyBit | (static_cast<uint32_t>(subCommand) << 8) | command; |
| wrMailboxReg(mbInterfaceReg, interfaceReg); |
| |
| // Wait until RUN_BUSY == 0 |
| attempts = mbRetries; |
| do |
| { |
| interfaceReg = rdMailboxReg(mbInterfaceReg); |
| } while ((interfaceReg & mbBusyBit) != 0 && --attempts > 0); |
| if (attempts == 0) |
| { |
| throw PECIError("OS Mailbox failed to return"); |
| } |
| |
| // Read command return status or error code from interface register |
| auto status = static_cast<MailboxStatus>(interfaceReg & 0xFF); |
| if (responseCode != nullptr) |
| { |
| *responseCode = status; |
| } |
| else if (status != MailboxStatus::NoError) |
| { |
| throw PECIError(std::string("OS Mailbox returned with error: ") + |
| std::to_string(static_cast<int>(status))); |
| } |
| |
| // Read command return data from the data register |
| return rdMailboxReg(mbDataReg); |
| } |
| }; |
| |
| /** |
| * Base class for set of PECI OS Mailbox commands. |
| * Constructing it runs the command and stores the value for use by derived |
| * class accessor methods. |
| */ |
| template <uint8_t subcommand> |
| struct OsMailboxCommand |
| { |
| enum ErrorPolicy |
| { |
| Throw, |
| NoThrow |
| }; |
| |
| uint32_t value; |
| PECIManager::MailboxStatus status = PECIManager::MailboxStatus::NoError; |
| /** |
| * Construct the command object with required PECI address and up to 4 |
| * optional 1-byte input data parameters. |
| */ |
| OsMailboxCommand(PECIManager& pm, uint8_t param1 = 0, uint8_t param2 = 0, |
| uint8_t param3 = 0, uint8_t param4 = 0) : |
| OsMailboxCommand(pm, ErrorPolicy::Throw, param1, param2, param3, param4) |
| {} |
| |
| OsMailboxCommand(PECIManager& pm, ErrorPolicy errorPolicy, |
| uint8_t param1 = 0, uint8_t param2 = 0, uint8_t param3 = 0, |
| uint8_t param4 = 0) |
| { |
| DEBUG_PRINT << "Running OS Mailbox command " |
| << static_cast<int>(subcommand) << '\n'; |
| PECIManager::MailboxStatus* callStatus = errorPolicy == Throw ? nullptr |
| : &status; |
| uint32_t param = (static_cast<uint32_t>(param4) << 24) | |
| (static_cast<uint32_t>(param3) << 16) | |
| (static_cast<uint32_t>(param2) << 8) | param1; |
| value = pm.sendPECIOSMailboxCmd(0x7F, subcommand, param, callStatus); |
| } |
| |
| /** Return whether the mailbox status indicated success or not. */ |
| bool success() const |
| { |
| return status == PECIManager::MailboxStatus::NoError; |
| } |
| }; |
| |
| /** |
| * Macro to define a derived class accessor method. |
| * |
| * @param[in] type Return type of accessor method. |
| * @param[in] name Name of accessor method. |
| * @param[in] hibit Most significant bit of field to access. |
| * @param[in] lobit Least significant bit of field to access. |
| */ |
| #define FIELD(type, name, hibit, lobit) \ |
| type name() const \ |
| { \ |
| return (value >> lobit) & (bit(hibit - lobit + 1) - 1); \ |
| } |
| |
| struct GetLevelsInfo : OsMailboxCommand<0x0> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(bool, enabled, 31, 31) |
| FIELD(bool, lock, 24, 24) |
| FIELD(unsigned, currentConfigTdpLevel, 23, 16) |
| FIELD(unsigned, configTdpLevels, 15, 8) |
| FIELD(unsigned, version, 7, 0) |
| }; |
| |
| struct GetConfigTdpControl : OsMailboxCommand<0x1> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(bool, pbfEnabled, 17, 17); |
| FIELD(bool, factEnabled, 16, 16); |
| FIELD(bool, pbfSupport, 1, 1); |
| FIELD(bool, factSupport, 0, 0); |
| }; |
| |
| struct SetConfigTdpControl : OsMailboxCommand<0x2> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| }; |
| |
| struct GetTdpInfo : OsMailboxCommand<0x3> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(unsigned, tdpRatio, 23, 16); |
| FIELD(unsigned, pkgTdp, 14, 0); |
| }; |
| |
| struct GetCoreMask : OsMailboxCommand<0x6> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(uint32_t, coresMask, 31, 0); |
| }; |
| |
| struct GetTurboLimitRatios : OsMailboxCommand<0x7> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| }; |
| |
| struct SetLevel : OsMailboxCommand<0x8> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| }; |
| |
| struct GetRatioInfo : OsMailboxCommand<0xC> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(unsigned, pm, 31, 24); |
| FIELD(unsigned, pn, 23, 16); |
| FIELD(unsigned, p1, 15, 8); |
| FIELD(unsigned, p0, 7, 0); |
| }; |
| |
| struct GetTjmaxInfo : OsMailboxCommand<0x5> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(unsigned, tProchot, 7, 0); |
| }; |
| |
| struct PbfGetCoreMaskInfo : OsMailboxCommand<0x20> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(uint32_t, p1HiCoreMask, 31, 0); |
| }; |
| |
| struct PbfGetP1HiP1LoInfo : OsMailboxCommand<0x21> |
| { |
| using OsMailboxCommand::OsMailboxCommand; |
| FIELD(unsigned, p1Hi, 15, 8); |
| FIELD(unsigned, p1Lo, 7, 0); |
| }; |
| |
| /** |
| * Implementation of SSTInterface based on OS Mailbox interface supported on ICX |
| * and SPR processors. |
| * It's expected that an instance of this class will be created for each |
| * "atomic" set of operations. |
| */ |
| class SSTMailbox : public SSTInterface |
| { |
| private: |
| uint8_t address; |
| CPUModel model; |
| PECIManager pm; |
| |
| static constexpr int mhzPerRatio = 100; |
| |
| public: |
| SSTMailbox(uint8_t _address, CPUModel _model, WakePolicy wakePolicy) : |
| address(_address), model(_model), |
| pm(static_cast<uint8_t>(address), model, wakePolicy) |
| {} |
| ~SSTMailbox() {} |
| |
| bool ready() override |
| { |
| return true; |
| } |
| |
| bool supportsControl() override |
| { |
| switch (model) |
| { |
| case sapphireRapids: |
| case emeraldRapids: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| unsigned int currentLevel() override |
| { |
| return GetLevelsInfo(pm).currentConfigTdpLevel(); |
| } |
| unsigned int maxLevel() override |
| { |
| return GetLevelsInfo(pm).configTdpLevels(); |
| } |
| bool ppEnabled() override |
| { |
| return GetLevelsInfo(pm).enabled(); |
| } |
| |
| bool levelSupported(unsigned int level) override |
| { |
| GetConfigTdpControl tdpControl( |
| pm, GetConfigTdpControl::ErrorPolicy::NoThrow, |
| static_cast<uint8_t>(level)); |
| return tdpControl.success(); |
| } |
| bool bfSupported(unsigned int level) override |
| { |
| return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) |
| .pbfSupport(); |
| } |
| bool tfSupported(unsigned int level) override |
| { |
| return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) |
| .factSupport(); |
| } |
| bool bfEnabled(unsigned int level) override |
| { |
| return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) |
| .pbfEnabled(); |
| } |
| bool tfEnabled(unsigned int level) override |
| { |
| return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) |
| .factEnabled(); |
| } |
| unsigned int tdp(unsigned int level) override |
| { |
| return GetTdpInfo(pm, static_cast<uint8_t>(level)).pkgTdp(); |
| } |
| unsigned int coreCount(unsigned int level) override |
| { |
| return enabledCoreList(level).size(); |
| } |
| std::vector<unsigned int> enabledCoreList(unsigned int level) override |
| { |
| uint64_t coreMaskLo = |
| GetCoreMask(pm, static_cast<uint8_t>(level), 0).coresMask(); |
| uint64_t coreMaskHi = |
| GetCoreMask(pm, static_cast<uint8_t>(level), 1).coresMask(); |
| std::bitset<64> coreMask = (coreMaskHi << 32 | coreMaskLo); |
| return convertMaskToList(coreMask); |
| } |
| std::vector<TurboEntry> sseTurboProfile(unsigned int level) override |
| { |
| // Read the Turbo Ratio Limit Cores MSR which is used to generate the |
| // Turbo Profile for each profile. This is a package scope MSR, so just |
| // read thread 0. |
| uint64_t trlCores; |
| uint8_t cc; |
| EPECIStatus status = peci_RdIAMSR(static_cast<uint8_t>(address), 0, |
| 0x1AE, &trlCores, &cc); |
| if (!checkPECIStatus(status, cc)) |
| { |
| throw PECIError("Failed to read TRL MSR"); |
| } |
| |
| std::vector<TurboEntry> turboSpeeds; |
| uint64_t limitRatioLo = |
| GetTurboLimitRatios(pm, static_cast<uint8_t>(level), 0, 0).value; |
| uint64_t limitRatioHi = |
| GetTurboLimitRatios(pm, static_cast<uint8_t>(level), 1, 0).value; |
| uint64_t limitRatios = (limitRatioHi << 32) | limitRatioLo; |
| |
| constexpr int maxTFBuckets = 8; |
| for (int i = 0; i < maxTFBuckets; ++i) |
| { |
| size_t bucketCount = trlCores & 0xFF; |
| int bucketSpeed = limitRatios & 0xFF; |
| if (bucketCount != 0 && bucketSpeed != 0) |
| { |
| turboSpeeds.push_back({bucketSpeed * mhzPerRatio, bucketCount}); |
| } |
| |
| trlCores >>= 8; |
| limitRatios >>= 8; |
| } |
| return turboSpeeds; |
| } |
| unsigned int p1Freq(unsigned int level) override |
| { |
| return GetRatioInfo(pm, static_cast<uint8_t>(level)).p1() * mhzPerRatio; |
| } |
| unsigned int p0Freq(unsigned int level) override |
| { |
| return GetRatioInfo(pm, static_cast<uint8_t>(level)).p0() * mhzPerRatio; |
| } |
| unsigned int prochotTemp(unsigned int level) override |
| { |
| return GetTjmaxInfo(pm, static_cast<uint8_t>(level)).tProchot(); |
| } |
| std::vector<unsigned int> |
| bfHighPriorityCoreList(unsigned int level) override |
| { |
| uint64_t coreMaskLo = PbfGetCoreMaskInfo(pm, |
| static_cast<uint8_t>(level), 0) |
| .p1HiCoreMask(); |
| uint64_t coreMaskHi = PbfGetCoreMaskInfo(pm, |
| static_cast<uint8_t>(level), 1) |
| .p1HiCoreMask(); |
| std::bitset<64> hiFreqCoreList = (coreMaskHi << 32) | coreMaskLo; |
| return convertMaskToList(hiFreqCoreList); |
| } |
| unsigned int bfHighPriorityFreq(unsigned int level) override |
| { |
| return PbfGetP1HiP1LoInfo(pm, static_cast<uint8_t>(level)).p1Hi() * |
| mhzPerRatio; |
| } |
| unsigned int bfLowPriorityFreq(unsigned int level) override |
| { |
| return PbfGetP1HiP1LoInfo(pm, static_cast<uint8_t>(level)).p1Lo() * |
| mhzPerRatio; |
| } |
| |
| void setBfEnabled(bool enable) override |
| { |
| GetConfigTdpControl getTDPControl(pm); |
| bool tfEnabled = false; |
| uint8_t param = (enable ? bit(1) : 0) | (tfEnabled ? bit(0) : 0); |
| SetConfigTdpControl(pm, 0, 0, param); |
| } |
| void setTfEnabled(bool enable) override |
| { |
| // TODO: use cached BF value |
| bool bfEnabled = false; |
| uint8_t param = (bfEnabled ? bit(1) : 0) | (enable ? bit(0) : 0); |
| SetConfigTdpControl(pm, 0, 0, param); |
| } |
| void setCurrentLevel(unsigned int level) override |
| { |
| SetLevel(pm, static_cast<uint8_t>(level)); |
| } |
| }; |
| |
| static std::unique_ptr<SSTInterface> |
| createMailbox(uint8_t address, CPUModel model, WakePolicy wakePolicy) |
| { |
| DEBUG_PRINT << "createMailbox\n"; |
| switch (model) |
| { |
| case iceLake: |
| case iceLakeD: |
| case sapphireRapids: |
| case emeraldRapids: |
| return std::make_unique<SSTMailbox>(address, model, wakePolicy); |
| default: |
| return nullptr; |
| } |
| } |
| |
| SSTProviderRegistration(createMailbox); |
| |
| } // namespace sst |
| } // namespace cpu_info |