Skip to content

ROCM-26020: read render-node-opening GPU config once at create time; … #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sarat-k merged 1 commit into from
Jul 21, 2026
Merged

ROCM-26020: read render-node-opening GPU config once at create time; … #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions sw/nic/gpuagent/api/gpu.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ gpu_entry::update_handler(api_params_base *api_params) {
return ret;
}
// update the stashed spec if the gpu update goes through
memcpy(spec, &spec_, sizeof(aga_gpu_spec_t));
memcpy(&spec_, spec, sizeof(aga_gpu_spec_t));
return ret;
}

Expand Down Expand Up @@ -180,13 +180,13 @@ gpu_entry::fill_status_(aga_gpu_spec_t *spec, aga_gpu_status_t *status) {
}

void
gpu_entry::init_immutable_attrs(void) {
gpu_entry::init_attrs(void) {
// stash information already available
status_.index = id_;
status_.handle = handle_;
status_.partition_id = partition_id_;
// fill other immutable attributes that we can get from API calls
smi_gpu_init_immutable_attrs(handle_, &key_, &spec_, &status_);
// fill initial attributes that we can get from API calls
smi_gpu_init_attrs(handle_, &key_, &spec_, &status_);
}

void
Expand Down
4 changes: 2 additions & 2 deletions sw/nic/gpuagent/api/gpu.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,8 @@ class gpu_entry : public api_base {
num_gpu_watch_--;
}

/// \brief initialize immutable attributes in GPU spec and status
void init_immutable_attrs(void);
/// \brief initialize attributes in GPU spec and status
void init_attrs(void);

/// \brief return parent GPU uuid
/// \return parent gpu uuid
Expand Down
97 changes: 56 additions & 41 deletions sw/nic/gpuagent/api/smi/amdsmi/smi_api.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,8 @@ smi_gpu_fill_spec (aga_gpu_handle_t gpu_handle,
const aga_obj_key_t *gpu_key,
aga_gpu_spec_t *spec)
{
uint32_t value_32;
amdsmi_status_t amdsmi_ret;
amdsmi_dev_perf_level_t perf_level = {};
amdsmi_gpu_metrics_t metrics_info = { 0 };
amdsmi_power_cap_info_t power_cap_info = {};
uint32_t sensor_idx[AGA_GPU_MAX_POWER_CAP_SENSOR];
amdsmi_power_cap_type_t sensor_types[AGA_GPU_MAX_POWER_CAP_SENSOR];

// re-fetch gpu metrics for this GPU handle
amdsmi_ret = amdsmi_get_gpu_metrics_info(gpu_handle, &metrics_info);
Expand All @@ -159,6 +154,30 @@ smi_gpu_fill_spec (aga_gpu_handle_t gpu_handle,
g_gpu_metrics[gpu_handle] = metrics_info;
}
}
// TODO: get admin_state
// TODO: get RAS spec
return SDK_RET_OK;
}

/// \brief read the config attributes (overdrive/perf/power-cap and voltage
/// curve) once at create time and cache them in the GPU spec/status;
/// the cached copy is refreshed by the GPUUpdate handler whenever a
/// value is updated; keeps the per-GpuGet path off the render node
/// \param[in] gpu_handle GPU handle
/// \param[out] spec GPU spec (cached config values)
/// \param[out] status GPU status (cached voltage curve)
static void
smi_gpu_init_config_ (aga_gpu_handle_t gpu_handle, aga_gpu_spec_t *spec,
aga_gpu_status_t *status)
{
uint32_t value_32;
amdsmi_status_t amdsmi_ret;
amdsmi_dev_perf_level_t perf_level = {};
amdsmi_od_volt_freq_data_t vc_data = {};
amdsmi_power_cap_info_t power_cap_info = {};
uint32_t sensor_idx[AGA_GPU_MAX_POWER_CAP_SENSOR];
amdsmi_power_cap_type_t sensor_types[AGA_GPU_MAX_POWER_CAP_SENSOR];

// fill the overdrive level
amdsmi_ret = amdsmi_get_gpu_overdrive_level(gpu_handle, &value_32);
if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) {
Expand Down Expand Up @@ -205,9 +224,22 @@ smi_gpu_fill_spec (aga_gpu_handle_t gpu_handle,
}
spec->num_gpu_power_cap = value_32;
}
// TODO: get admin_state
// TODO: get RAS spec
return SDK_RET_OK;
// fill the voltage curve points
amdsmi_ret = amdsmi_get_gpu_od_volt_info(gpu_handle, &vc_data);
if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) {
AGA_TRACE_ERR("Failed to get voltage curve points for GPU {}, err {}",
gpu_handle, amdsmi_ret);
} else {
for (uint32_t i = 0;
(i < AGA_GPU_MAX_VOLTAGE_CURVE_POINT) &&
(i < AMDSMI_NUM_VOLTAGE_CURVE_POINTS); i++) {
status->voltage_curve_point[i].point = i;
status->voltage_curve_point[i].frequency =
vc_data.curve.vc_points[i].frequency/1000000;
status->voltage_curve_point[i].voltage =
vc_data.curve.vc_points[i].voltage;
}
}
}

/// \brief function to get name for amdsmi firmware block enum
Expand Down Expand Up @@ -799,7 +831,6 @@ smi_gpu_fill_status (aga_gpu_handle_t gpu_handle,
{
amdsmi_status_t amdsmi_ret;
amdsmi_xgmi_status_t xgmi_st;
amdsmi_od_volt_freq_data_t vc_data;
amdsmi_gpu_metrics_t metrics_info = { 0 };

{
Expand Down Expand Up @@ -836,24 +867,8 @@ smi_gpu_fill_status (aga_gpu_handle_t gpu_handle,
} else {
status->xgmi_status.error_status = smi_to_aga_gpu_xgmi_error(xgmi_st);
}
// fill the voltage curve points
amdsmi_ret = amdsmi_get_gpu_od_volt_info(gpu_handle, &vc_data);
if (unlikely(amdsmi_ret != AMDSMI_STATUS_SUCCESS)) {
AGA_TRACE_ERR("Failed to get voltage curve points for GPU {}, err {}",
gpu_handle, amdsmi_ret);
} else {
for (uint32_t i = 0;
(i < AGA_GPU_MAX_VOLTAGE_CURVE_POINT) &&
(i < AMDSMI_NUM_VOLTAGE_CURVE_POINTS); i++) {
status->voltage_curve_point[i].point = i;
status->voltage_curve_point[i].frequency =
vc_data.curve.vc_points[i].frequency/1000000;
status->voltage_curve_point[i].voltage =
vc_data.curve.vc_points[i].voltage;
}
}
// fill list of pids using the GPU
smi_fill_gpu_kfd_pid_status_(gpu_handle, gpu_id, status);
smi_fill_gpu_enumeration_id_status_(gpu_handle, status);
// TODO: oper status
// TODO: RAS status
return SDK_RET_OK;
Expand Down Expand Up @@ -1270,9 +1285,6 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle,
aga_gpu_stats_t *stats)
{
amdsmi_status_t amdsmi_ret;
amdsmi_npm_info_t npm_info = {};
amdsmi_power_info_t power_info = {};
amdsmi_node_handle node_handle = {};
uint64_t sent, received, max_pkt_size;
amdsmi_gpu_metrics_t metrics_info = {};
amdsmi_gpu_metrics_t cached_metrics = {};
Expand All @@ -1286,18 +1298,6 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle,

// fill VRAM usage
smi_fill_vram_usage_(gpu_handle, &stats->vram_usage);
// UBB node power + cap (MI350X+); decodes to 0 where unsupported
amdsmi_ret = amdsmi_get_power_info(gpu_handle, &power_info);
if (likely(amdsmi_ret == AMDSMI_STATUS_SUCCESS)) {
stats->ubb_power = power_info.ubb_power;
}
amdsmi_ret = amdsmi_get_node_handle(gpu_handle, &node_handle);
if (likely(amdsmi_ret == AMDSMI_STATUS_SUCCESS)) {
amdsmi_ret = amdsmi_get_npm_info(node_handle, &npm_info);
if (likely(amdsmi_ret == AMDSMI_STATUS_SUCCESS)) {
stats->ubb_power_cap = npm_info.ubb_power_threshold;
}
}
// fill additional statistics from gpu metrics
{
std::lock_guard<std::mutex> lock(g_gpu_metrics_mutex);
Expand Down Expand Up @@ -2239,6 +2239,8 @@ smi_gpu_init_immutable_attrs (aga_gpu_handle_t gpu_handle,
((uint32_t)((value_64 >> 3) & 0x1f)),
((uint32_t)(value_64 & 0x7)));
}
// enumeration/kfd identity ids
smi_fill_gpu_enumeration_id_status_(gpu_handle, status);
// get energy counter resolution if not already set
if (g_energy_counter_resolution == 0.0) {
amdsmi_ret = amdsmi_get_energy_count(gpu_handle, &value_64,
Expand All @@ -2253,6 +2255,19 @@ smi_gpu_init_immutable_attrs (aga_gpu_handle_t gpu_handle,
return SDK_RET_OK;
}

sdk_ret_t
smi_gpu_init_attrs (aga_gpu_handle_t gpu_handle, const aga_obj_key_t *gpu_key,
aga_gpu_spec_t *spec, aga_gpu_status_t *status)
{
// initialize the immutable attributes
smi_gpu_init_immutable_attrs(gpu_handle, gpu_key, spec, status);
// read the attributes that are also read on every get
smi_gpu_fill_spec(gpu_handle, gpu_key, spec);
// read the config attributes
smi_gpu_init_config_(gpu_handle, spec, status);
return SDK_RET_OK;
}

static inline std::string
timestamp_string_from_cper_timestamp (amdsmi_cper_timestamp_t *ts)
{
Expand Down
23 changes: 11 additions & 12 deletions sw/nic/gpuagent/api/smi/gimamdsmi/smi_api.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1088,10 +1088,8 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle_in,
sdk_ret_t ret;
int64_t temperature;
amdsmi_status_t amdsmi_ret;
amdsmi_npm_info_t npm_info = {};
amdsmi_pcie_info_t pcie_info = {};
amdsmi_power_info_t power_info = {};
amdsmi_node_handle node_handle = {};
amdsmi_engine_usage_t usage_info = {};

AGA_SMI_SESSION_GUARD(gpu_key, gpu_handle_in);
Expand All @@ -1106,16 +1104,6 @@ smi_gpu_fill_stats (aga_gpu_handle_t gpu_handle_in,
stats->voltage.voltage = power_info.soc_voltage;
stats->voltage.gfx_voltage = power_info.gfx_voltage;
stats->voltage.memory_voltage = power_info.mem_voltage;
// UBB node power (MI350X+, GIM 9.1.0.K+); 0 where unsupported
stats->ubb_power = power_info.ubb_power;
}
// UBB node power cap = node-level power threshold (npm); 0 where unsupported
amdsmi_ret = amdsmi_get_node_handle(gpu_handle, &node_handle);
if (likely(amdsmi_ret == AMDSMI_STATUS_SUCCESS)) {
amdsmi_ret = amdsmi_get_npm_info(node_handle, &npm_info);
if (likely(amdsmi_ret == AMDSMI_STATUS_SUCCESS)) {
stats->ubb_power_cap = npm_info.ubb_power_threshold;
}
}
// fill the GPU usage
amdsmi_ret = amdsmi_get_gpu_activity(gpu_handle, &usage_info);
Expand Down Expand Up @@ -1674,6 +1662,17 @@ smi_gpu_init_immutable_attrs (aga_gpu_handle_t gpu_handle_in,
return SDK_RET_OK;
}

sdk_ret_t
smi_gpu_init_attrs (aga_gpu_handle_t gpu_handle, const aga_obj_key_t *gpu_key,
aga_gpu_spec_t *spec, aga_gpu_status_t *status)
{
// initialize the immutable attributes
smi_gpu_init_immutable_attrs(gpu_handle, gpu_key, spec, status);
// read the attributes that are also read on every get
smi_gpu_fill_spec(gpu_handle, gpu_key, spec);
return SDK_RET_OK;
}

static inline std::string
timestamp_string_from_cper_timestamp (amdsmi_cper_timestamp_t *ts)
{
Expand Down
12 changes: 12 additions & 0 deletions sw/nic/gpuagent/api/smi/smi_api.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,18 @@ sdk_ret_t smi_gpu_init_immutable_attrs(aga_gpu_handle_t gpu_handle,
aga_gpu_spec_t *spec,
aga_gpu_status_t *status);

/// \brief initialize the immutable attributes, the attributes also read on
/// every get, and the config attributes
/// \param[in] gpu_handle GPU handle
/// \param[in] gpu_key stable GPU UUID
/// \param[out] spec GPU spec
/// \param[out] status GPU status
/// \return SDK_RET_OK or error code in case of failure
sdk_ret_t smi_gpu_init_attrs(aga_gpu_handle_t gpu_handle,
const aga_obj_key_t *gpu_key,
aga_gpu_spec_t *spec,
aga_gpu_status_t *status);

/// \brief function to get GPU CPER entries
/// \param[in] gpu_handle handle of GPU device (uuid refreshes inside)
/// \param[in] uuid stable GPU UUID
Expand Down
9 changes: 9 additions & 0 deletions sw/nic/gpuagent/api/smi/smi_api_mock.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,15 @@ smi_gpu_init_immutable_attrs (aga_gpu_handle_t gpu_handle,
return SDK_RET_OK;
}

sdk_ret_t
smi_gpu_init_attrs (aga_gpu_handle_t gpu_handle, const aga_obj_key_t *gpu_key,
aga_gpu_spec_t *spec, aga_gpu_status_t *status)
{
smi_gpu_init_immutable_attrs(gpu_handle, gpu_key, spec, status);
smi_gpu_fill_spec(gpu_handle, gpu_key, spec);
return SDK_RET_OK;
}

sdk_ret_t
smi_gpu_fill_spec (aga_gpu_handle_t gpu_handle,
const aga_obj_key_t *gpu_key,
Expand Down
4 changes: 2 additions & 2 deletions sw/nic/gpuagent/init.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ create_gpus (void)
}
// set partition id
entry->set_partition_id(gpu[i].partition_id);
// initialize immutable attributes in GPU spec and status
entry->init_immutable_attrs();
// initialize attributes in GPU spec and status at create time
entry->init_attrs();
// insert in handle db
gpu_db()->insert_in_handle_db(entry);
// if GPU is a child GPU, add to the parent GPU
Expand Down
Loading