Commit f46c7062 authored by Ondrej Vysocky's avatar Ondrej Vysocky
Browse files

NEW, FIX added support for RAPL energy measurement using libmsr, and found a...

NEW, FIX added support for RAPL energy measurement using libmsr, and found a bug in x86adapt measurement in missing register mask #59 #4 #34
parent 289cead7
......@@ -25,7 +25,7 @@ static std::map<std::string, std::string> counterLabel = {
};
}
////////////////////////////////////////////////////////////////////////////////
/**
* to add new PERFEVENT counter:
* add here counter description
......@@ -50,7 +50,7 @@ static std::map<std::string, std::string> counterLabel = {
};
}
////////////////////////////////////////////////////////////////////////////////
/**
* to add new RAPL counter:
* add here counter name as it is in "x86a_read -n"
......@@ -59,7 +59,7 @@ static std::map<std::string, std::string> counterLabel = {
* WARNING: RAPL counters are for each socket - in output file counters label will have a suffix
* WARNING: all added counters must be ENERGY counters, RAPL time and RAPL power has different unit and RAPL::init() must be changed accordingly. Further explanation at the top of raplwrapper.cpp.
**/
namespace RAPL_COUNTERS {
namespace X86ADAPT_RAPL_COUNTERS {
const std::string RAPL_PCKG_ENERGY = "Intel_RAPL_Pckg_Energy";
const std::string RAPL_RAM_ENERGY = "Intel_RAPL_RAM_Energy";
......@@ -82,6 +82,41 @@ static std::map<std::string, std::string> counterLabel = {
{ "Energy summary [J]", "Energy summary [J]" }
};
}
////////////////////////////////////////////////////////////////////////////////
namespace MSR_RAPL_COUNTERS {
const std::string RAPL_PCKG_ENERGY = "MSR_PKG_ENERGY_STATUS";
const std::string RAPL_RAM_ENERGY = "MSR_DRAM_ENERGY_STATUS";
const std::string RAPL_SUM_ENERGY = "Energy summary [J]"; //PCKG+RAM = not a counter to measure
/*
const std::string RAPL_PP0_ENERGY = "MSR_PP0_ENERGY_STATUS"; //not available
const std::string RAPL_PP1_ENERGY = "MSR_PP1_ENERGY_STATUS";
*/
static std::map<std::string, std::string> counterLabel = {
{ "MSR_PKG_ENERGY_STATUS", "RAPL_PCKG [J]" },
{ "MSR_DRAM_ENERGY_STATUS", "RAPL_RAM [J]" },
{ "MSR_PKG_ENERGY_STATUS_0", "RAPL_PCKG_0 [J]" },
{ "MSR_PKG_ENERGY_STATUS_1", "RAPL_PCKG_1 [J]" },
{ "MSR_DRAM_ENERGY_STATUS_0", "RAPL_RAM_0 [J]" },
{ "MSR_DRAM_ENERGY_STATUS_1", "RAPL_RAM_1 [J]" },
{ "Energy summary [J]", "Energy summary [J]" }
};
#ifdef HAVE_MSR_CORE_H
#include <master.h>
static std::map<std::string, uint64_t> counterDefines = {
{ "MSR_PKG_ENERGY_STATUS", MSR_PKG_ENERGY_STATUS },
{ "MSR_DRAM_ENERGY_STATUS", MSR_DRAM_ENERGY_STATUS },
{ "MSR_PKG_ENERGY_STATUS_0", MSR_PKG_ENERGY_STATUS },
{ "MSR_PKG_ENERGY_STATUS_1", MSR_PKG_ENERGY_STATUS },
{ "MSR_DRAM_ENERGY_STATUS_0", MSR_DRAM_ENERGY_STATUS },
{ "MSR_DRAM_ENERGY_STATUS_1", MSR_DRAM_ENERGY_STATUS },
{ "Energy summary [J]", 0x0 }
};
#endif
}
#endif /* SRC_WRAPPER_COUNTERS_H_ */
......@@ -19,6 +19,7 @@ systemLimits Environment::FreqLimits [] = {
[SYS_OTHER] = { -1, -1, -1, -1, -1, -1 }
};
int Environment::NODE_SIZE_THREAD = 1;
int Environment::NODE_SIZE_CPU = 0;
int Environment::NODE_SIZE_SOCKET = 0;
int Environment::SOCKET_SIZE_CPU = 0;
......@@ -63,16 +64,30 @@ void Environment::threadNumber(int nthreads)
void Environment::initHWSizes()
{
#ifdef HAVE_X86_ADAPT_H // and adapt must be initialized which makes it
NODE_SIZE_CPU = x86_adapt_get_nr_avaible_devices(X86_ADAPT_CPU);
#ifdef HAVE_MSR_CORE_H
int Hthreads;
uint64_t cs = 0;
uint64_t ts = 0;
uint64_t ss = 0;
core_config(&cs, &ts, &ss, &Hthreads);
NODE_SIZE_SOCKET = ss;
SOCKET_SIZE_CPU = cs;
NODE_SIZE_CPU = SOCKET_SIZE_CPU * NODE_SIZE_SOCKET;
NODE_SIZE_THREAD = ts * NODE_SIZE_CPU;
MERIC_INFO << "HThreads: " << ((Hthreads == 0) ? "inactive" : "active") << std::endl;
#elif defined HAVE_X86_ADAPT_H // and adapt must be initialized which makes it
NODE_SIZE_CPU = x86_adapt_get_nr_avaible_devices(X86_ADAPT_CPU);
NODE_SIZE_SOCKET = x86_adapt_get_nr_avaible_devices(X86_ADAPT_DIE);
//NODE_SIZE_THREAD = 1; TODO get real number of avail threads per core - not used now
SOCKET_SIZE_CPU = NODE_SIZE_CPU / NODE_SIZE_SOCKET;
#else
NODE_SIZE_CPU = sysconf(_SC_NPROCESSORS_ONLN);
NODE_SIZE_CPU = sysconf(_SC_NPROCESSORS_ONLN);
NODE_SIZE_SOCKET = numa_node_of_cpu(NODE_SIZE_CPU-1)+1;
//NODE_SIZE_THREAD = 1; TODO
if (NODE_SIZE_SOCKET <= 0)
NODE_SIZE_SOCKET=1;
#endif
SOCKET_SIZE_CPU = NODE_SIZE_CPU / NODE_SIZE_SOCKET;
#endif
MERIC_INFO << "CPUs " << NODE_SIZE_CPU << " SOCKETS " << NODE_SIZE_SOCKET << std::endl;
}
......@@ -147,6 +162,14 @@ void Environment::init(int system, RegionParameters defaultParameter, int rank)
// exit(1);
}
#endif
#ifdef HAVE_MSR_CORE_H
if (init_msr())
{
libmsr_error_handler("Unable to initialize libmsr", LIBMSR_ERROR_MSR_INIT, getenv("HOSTNAME"), __FILE__, __LINE__);
// exit(1);
}
#endif
if (SOCKET_SIZE_CPU <= 0)
initHWSizes();
MERIC_INFO << "SOCKET SIZE is " << SOCKET_SIZE_CPU <<std::endl;
......@@ -460,6 +483,9 @@ void Environment::close()
#ifdef HAVE_X86_ADAPT_H
x86_adapt_finalize();
#endif
#ifdef HAVE_MSR_CORE_H
finalize_msr();
#endif
}
......@@ -15,6 +15,13 @@
#include <numa.h>
#endif
#ifdef HAVE_MSR_CORE_H
#include <msr_core.h>
#include <msr_rapl.h>
#include <msr_clocks.h>
#include <libmsr_error.h>
#endif
#ifdef HAVE_CILK_CILK_API_H
#include <cilk/cilk_api.h>
#endif
......@@ -58,6 +65,7 @@ const int X86ADAPT_MAX_FREQ = 33;
class Environment {
public:
static int NODE_SIZE_THREAD;
static int NODE_SIZE_CPU;
static int NODE_SIZE_SOCKET;
static int SOCKET_SIZE_CPU;
......
#ifdef HAVE_X86_ADAPT_H
#include "x86_adapt.h"
#include "raplwrapper.h"
#include "../basis/utils.h"
/* RAPL description
/**
* RAPL description
*
* Intel_RAPL_Pckg_Energy - all energy consumtions
* Intel_RAPL_Pckg_Energy - all energy consumptions
* Intel_RAPL_PP0_Energy - CPU
* Intel_RAPL_PP1_Energy - may reflect to uncore devices
* Intel_RAPL_RAM_Energy - RAM
......@@ -12,45 +13,106 @@
* Intel_RAPL_Pckg_Energy ~ Intel_RAPL_PP0_Energy + Intel_RAPL_RAM_Energy
*
*
* MSR_RAPL_POWER_UNIT Register
* MSR_RAPL_POWER_UNIT Register ***********************************************
* Power Units (bits 3:0):
Power related information (in Watts) is based on the multiplier, 1/2^PU; where PU is an unsigned integer represented by bits 3:0. Default value is 0011b, indicating power unit is in 1/8 Watts
increment.
* Energy Status Units (bits 12:8):
Energy related information (in Joules) is based on the multiplier, 1/2^ESU; where ESU is an unsigned integer represented by bits 12:8. Default value is 10000b, indicating energy status unit is in 15.3 micro-Joules increment.
* Time Units (bits 19:16):
Time related information (in Seconds) is based on the multiplier, 1/ 2^TU; where TU is an unsigned integer represented by bits 19:16. Default value is 1010b, indicating time unit is in 976 micro-seconds increment.
Time related information (in Seconds) is based on the multiplier, 1/ 2^TU; where TU is an unsigned integer represented by bits 19:16. Default value is 1010b, indicating time unit is in 976 micro-seconds increment.
TIME ENERGY POWER
1010|000|01110|0000|0011
9876 543 21098 7654 3210
*/
const uint64_t RAPLpowerMask = 0xF;
const uint64_t RAPLenergyMask = 0x1F00;
const uint64_t RAPLenergyShift= 8;
const uint64_t RAPLtimeMask = 0xF0000;
const uint64_t RAPLtimeShift = 16;
#endif
* MSR_PKG_ENERGY_STATUS register *********************************************
* Total energy consumed (bits 31:0):
The unsigned integer value represents the total amount of energy consumed since that last time this register is cleared. The unit of this field is specified by the "Energy Status Units" field of MSR_RAPL_POWER_UNIT.
* Reserved (bits 63:32)
* MSR_PP0_ENERGY_STATUS/MSR_PP1_ENERGY_STATUS register ***********************
* Total Energy Consumed (bits 31:0):
The unsigned integer value represents the total amount of energy consumed since the last time this register was cleared. The unit of this field is specified by the "Energy Status Units" field of MSR_RAPL_POWER_UNIT.
* Reserved (bits 63:32)
#include "raplwrapper.h"
#include "../basis/utils.h"
* MSR_DRAM_ENERGY_STATUS register ********************************************
* Total Energy Consumed (bits 31:0):
The unsigned integer value represents the total amount of energy consumed since that last time this register is cleared. The unit of this field is specified by the "Energy Status Units" field of MSR_RAPL_POWER_UNIT.
* Reserved (bits 63:32)
*
**/
const uint64_t MSR_RAPL_POWER_UNIT_power_mask = 0xF;
const uint64_t MSR_RAPL_POWER_UNIT_energy_mask = 0x1F00;
const uint64_t MSR_RAPL_POWER_UNIT_energy_shift = 8;
const uint64_t MSR_RAPL_POWER_UNIT_time_mask = 0xF;
const uint64_t MSR_RAPL_POWER_UNIT_time_shift = 16;
//TODO to simplyfy the code, only PKG mask is used in the following code since
//it is the same as DRAM/PP0/PP1 mask
const uint64_t MSR_PKG_ENERGY_STATUS_mask = 0xFFFFFFFF;
//const uint64_t MSR_DRAM_ENERGY_STATUS_mask = 0xFFFFFFFF;
//const uint64_t MSR_PP0_ENERGY_STATUS_mask = 0xFFFFFFFF;
//const uint64_t MSR_PP1_ENERGY_STATUS_mask = 0xFFFFFFFF;
#ifdef HAVE_X86_ADAPT_H
#include "x86_adapt.h"
using namespace X86ADAPT_RAPL_COUNTERS;
#elif defined (HAVE_MSR_CORE_H)
#include <msr_core.h>
#include <msr_rapl.h>
// #include <master.h>
// #include <libmsr_error.h>
using namespace MSR_RAPL_COUNTERS;
#else
using namespace X86ADAPT_RAPL_COUNTERS;
#endif
using namespace meric;
std::string RAPL::suffix = "_";
int RAPL::numOfDevices = 1;
int RAPL::NODE_SIZE_SOCKET = 1;
int RAPL::NODE_SIZE_CPU = 1;
int RAPL::SOCKET_SIZE_CPU = 1;
bool RAPL::detailedMode = false;
unsigned long long int RAPL::COUNTERMAX = pow(2,32);
#ifdef HAVE_X86_ADAPT_H
#if defined (HAVE_X86_ADAPT_H) || defined (HAVE_MSR_CORE_H)
std::map<std::string, unsigned long long int> RAPL::counters;
std::map<std::string, double> RAPL::energyUnit;
// solve counter overflow and add baseline energy consumption
unsigned long long RAPL::getResultValue(unsigned long long int startValue, unsigned long long int stopValue, double runtime, std::string counter)
{
if (counter == RAPL_SUM_ENERGY)
return 0;
std::string counterName;
if (detailedMode)
counterName = counter.substr(0,counter.length() - suffix.length() - 1);
else
counterName = counter;
if (stopValue < startValue)
{
if (energyUnit.count(counterName))
{
MERIC_INFO << "RAPL OVERFLOW, MAX VALUE: " <<COUNTERMAX << "*"<< energyUnit[counterName] <<" " <<counterName << std::endl;
MERIC_INFO << "RAPL REPAIR " << startValue << " .. " << stopValue << " = " << ((COUNTERMAX - startValue) + stopValue) * energyUnit[counterName] << std::endl;
return ((COUNTERMAX - startValue) + stopValue) * energyUnit[counterName];
} //else: unit modifier is missing
return 0;
}
else
return (stopValue - startValue) * energyUnit[counterName];
}
#endif
#ifdef HAVE_X86_ADAPT_H
// set counters units
void RAPL::init()
{
numOfDevices = x86_adapt_get_nr_avaible_devices(X86_ADAPT_DIE);
NODE_SIZE_SOCKET = x86_adapt_get_nr_avaible_devices(X86_ADAPT_DIE);
initMap(counters);
//set unit modifier for Intel_RAPL_Pckg_Energy
......@@ -65,8 +127,8 @@ void RAPL::init()
}
const int itemId = x86_adapt_lookup_ci_name(X86_ADAPT_DIE, "Intel_RAPL_Power_Unit");
x86_adapt_get_setting(fd, itemId, &unit);
unit &= RAPLenergyMask;
unit >>= RAPLenergyShift;
unit &= MSR_RAPL_POWER_UNIT_energy_mask;
unit >>= MSR_RAPL_POWER_UNIT_energy_shift;
if (x86_adapt_put_device(X86_ADAPT_DIE, 0))
std::cerr << "X86ADAPT RAPL INIT error while closing file descriptor\n";
......@@ -82,16 +144,16 @@ void RAPL::init()
}
// read counters values
void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & record)
void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & record)
{
for(std::map<std::string, unsigned long long int>::iterator iterator = counters.begin(); iterator != counters.end(); iterator++)
record[iterator->first] = 0;
record[RAPL_COUNTERS::RAPL_SUM_ENERGY] = 0; //will be filled when the measurement is over
record[RAPL_SUM_ENERGY] = 0; //will be filled when the measurement is over
if (detailedMode)
{
for (int die=0; die<numOfDevices; die++)
for (int die=0; die<NODE_SIZE_SOCKET; die++)
{
uint64_t dieEnergy = 0;
const int fd = x86_adapt_get_device_ro(X86_ADAPT_DIE, die);
......@@ -103,10 +165,10 @@ void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & reco
for(std::map<std::string, unsigned long long int>::iterator iterator = counters.begin(); iterator != counters.end(); iterator++)
{
//TODO this version doesn't count with more than 10 sockets per node (should be enough for this century)
//TODO this version doesn't expect more than 10 sockets per node (should be enough for this century)
if ((iterator->first.back()-'0') != die)
continue;
std::string itemName = iterator->first.substr(0,iterator->first.length() - suffix.length() - 1);
std::string itemName = iterator->first.substr(0,iterator->first.length() - suffix.length() - 1);
const int itemId = x86_adapt_lookup_ci_name(X86_ADAPT_DIE, itemName.c_str());
if (itemId < 0) //== -ENXIO
{
......@@ -119,14 +181,14 @@ void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & reco
continue;
}
x86_adapt_get_setting(fd, itemId, &dieEnergy);
record [itemName+suffix+std::to_string(die)] = dieEnergy;
record [itemName+suffix+std::to_string(die)] = (dieEnergy & MSR_PKG_ENERGY_STATUS_mask);
}
if (x86_adapt_put_device(X86_ADAPT_DIE, die))
std::cerr << "X86ADAPT RAPL error while closing file descriptor\n";
}
}else
{
for (int die=0; die<numOfDevices; die++)
for (int die=0; die<NODE_SIZE_SOCKET; die++)
{
uint64_t dieEnergy = 0;
const int fd = x86_adapt_get_device_ro(X86_ADAPT_DIE, die);
......@@ -139,7 +201,7 @@ void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & reco
{
const int itemId = x86_adapt_lookup_ci_name(X86_ADAPT_DIE, iterator->first.c_str());
x86_adapt_get_setting(fd, itemId, &dieEnergy);
record [iterator->first] += dieEnergy;
record [iterator->first] += (dieEnergy & MSR_PKG_ENERGY_STATUS_mask);
}
if (x86_adapt_put_device(X86_ADAPT_DIE, die))
std::cerr << "X86ADAPT RAPL error while closing file descriptor\n";
......@@ -147,33 +209,59 @@ void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & reco
}
}
// solve counter overflow and add baseline energy consumption
unsigned long long RAPL::getResultValue(unsigned long long int startValue, unsigned long long int stopValue, double runtime, std::string counter)
////////////////////////////////////////////////////////////////////////////////
#elif defined HAVE_MSR_CORE_H
// set counters units
void RAPL::init()
{
if (counter == RAPL_COUNTERS::RAPL_SUM_ENERGY)
return 0;
std::string counterName;
uint64_t cs = 0;
uint64_t ss = 0;
core_config(&cs, NULL, &ss, NULL);
NODE_SIZE_SOCKET = ss;
SOCKET_SIZE_CPU = cs;
NODE_SIZE_CPU = SOCKET_SIZE_CPU * NODE_SIZE_SOCKET;
initMap(counters);
uint64_t unit;
read_msr_by_idx(0, MSR_RAPL_POWER_UNIT, &unit);
energyUnit[RAPL_PCKG_ENERGY] = pow(2,((unit & MSR_RAPL_POWER_UNIT_energy_mask) >> MSR_RAPL_POWER_UNIT_energy_shift));
energyUnit[RAPL_RAM_ENERGY] = energyUnit[RAPL_PCKG_ENERGY];
}
void RAPL::getCounterValues(std::map<std::string, unsigned long long int> & record)
{
for(std::map<std::string, unsigned long long int>::iterator iterator = counters.begin(); iterator != counters.end(); iterator++)
record[iterator->first] = 0;
record[RAPL_SUM_ENERGY] = 0;
if (detailedMode)
counterName = counter.substr(0,counter.length() - suffix.length() - 1);
else
counterName = counter;
if (stopValue < startValue)
{
if (energyUnit.count(counterName))
{
for (int die=0; die<NODE_SIZE_SOCKET; die++)
{
MERIC_INFO << "RAPL OVERFLOW, MAX VALUE: " <<COUNTERMAX << "*"<< energyUnit[counterName] <<" " <<counterName << std::endl;
MERIC_INFO << "RAPL REPAIR " << startValue << " .. " << stopValue << " = " << ((COUNTERMAX - startValue) + stopValue) * energyUnit[counterName] << std::endl;
return ((COUNTERMAX - startValue) + stopValue) * energyUnit[counterName];
} //else: unit modifier is missing
return 0;
uint64_t dieEnergy = 0;
for(std::map<std::string, unsigned long long int>::iterator iterator = counters.begin(); iterator != counters.end(); iterator++)
{
if ((iterator->first.back()-'0') != die)
continue;
std::string itemName = iterator->first.substr(0,iterator->first.length() - suffix.length() - 1);
read_msr_by_idx(die * SOCKET_SIZE_CPU, counterDefines[itemName], &dieEnergy);
record [itemName+suffix+std::to_string(die)] = (dieEnergy & MSR_PKG_ENERGY_STATUS_mask);
}
}
}else
{
for (int die=0; die<NODE_SIZE_SOCKET; die++)
{
uint64_t dieEnergy = 0;
for(std::map<std::string, unsigned long long int>::iterator iterator = counters.begin(); iterator != counters.end(); iterator++)
{
read_msr_by_idx(die * SOCKET_SIZE_CPU, counterDefines[iterator->first], &dieEnergy);
record [iterator->first] += (dieEnergy & MSR_PKG_ENERGY_STATUS_mask);
}
}
}
else
return (stopValue - startValue) * energyUnit[counterName];
}
#else
#endif
std::string RAPL::getCountersLabel()
......@@ -183,8 +271,8 @@ std::string RAPL::getCountersLabel()
std::string RAPL::getCounterLabel(std::string counter)
{
std::map<std::string, std::string>::iterator it = RAPL_COUNTERS::counterLabel.find(counter);
if (it != RAPL_COUNTERS::counterLabel.end())
std::map<std::string, std::string>::iterator it = counterLabel.find(counter);
if (it != counterLabel.end())
return it->second;
else
return counter;
......@@ -192,7 +280,7 @@ std::string RAPL::getCounterLabel(std::string counter)
std::string RAPL::getSummaryCounterName()
{
return RAPL_COUNTERS::RAPL_SUM_ENERGY;
return RAPL_SUM_ENERGY;
}
void RAPL::getSummaryValue(double time, unsigned long long int * counterSummary)
{
......@@ -204,8 +292,8 @@ void RAPL::initMap(std::map<std::string, unsigned long long int> & dict)
{
// fill this vector
std::vector <std::string> names;
names.push_back(RAPL_COUNTERS::RAPL_PCKG_ENERGY);
names.push_back(RAPL_COUNTERS::RAPL_RAM_ENERGY);
names.push_back(RAPL_PCKG_ENERGY);
names.push_back(RAPL_RAM_ENERGY);
// in detailed mode RAPL gives us values for each cpu
// this is a reason for adding a suffix with number of a socket for each counter
......@@ -216,7 +304,7 @@ void RAPL::initMap(std::map<std::string, unsigned long long int> & dict)
if (detailedMode)
{
for(int die = 0; die != numOfDevices; die++)
for(int die = 0; die != NODE_SIZE_SOCKET; die++)
{
for(std::vector<std::string>::iterator it = names.begin(); it != names.end(); ++it)
{
......
......@@ -13,9 +13,9 @@
namespace meric {
class RAPL {
#ifdef HAVE_X86_ADAPT_H
#if defined (HAVE_X86_ADAPT_H) || defined (HAVE_MSR_CORE_H)
public:
static const int RAPL_BASELINE = 0; // ~70 W should be added in the RADAR config file
static const int RAPL_BASELINE = 0; // NOT USED at the MERIC side now
static void init();
static void close() {};
......@@ -33,7 +33,9 @@ private:
static std::map<std::string, unsigned long long int> counters;
static std::map<std::string, double> energyUnit;
static std::string suffix;
static int numOfDevices;
static int NODE_SIZE_SOCKET;
static int NODE_SIZE_CPU;
static int SOCKET_SIZE_CPU;
static bool detailedMode;
static unsigned long long int COUNTERMAX;
#else
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment