blob: 6e82858222f0840fdd6ba83f6153ad0e7588982d [file] [log] [blame]
George Keishing0fff68a2017-04-27 22:25:49 -05001*** Settings ***
Steven Sombar130a04f2017-07-16 10:02:37 -05002Documentation Keywords for system test.
George Keishing0fff68a2017-04-27 22:25:49 -05003
George Keishing91c852f2017-05-03 02:48:21 -05004Library ../lib/gen_robot_keyword.py
Steven Sombar021cbc42019-03-28 11:49:21 -05005Library ../lib/gen_print.py
George Keishing36efbc02018-12-12 10:18:23 -06006Library ../lib/gen_robot_print.py
Steven Sombar07be3742018-01-11 13:02:40 -06007Resource ../lib/boot_utils.robot
George Keishing91c852f2017-05-03 02:48:21 -05008Resource ../extended/obmc_boot_test_resource.robot
George Keishing0fff68a2017-04-27 22:25:49 -05009Resource ../lib/utils.robot
George Keishing91c852f2017-05-03 02:48:21 -050010Resource ../lib/state_manager.robot
George Keishingcff7aa02017-05-16 08:05:10 -050011Resource ../lib/rest_client.robot
George Keishing3fc63132023-12-14 16:19:07 +053012Resource ../lib/htx_resource.robot
George Keishing4ef29082017-05-13 05:01:23 -050013Library OperatingSystem
14Library DateTime
15
George Keishing0fff68a2017-04-27 22:25:49 -050016*** Variables ***
17
Steven Sombar130a04f2017-07-16 10:02:37 -050018${htx_log_dir_path} ${EXECDIR}${/}logs${/}
George Keishing4ef29082017-05-13 05:01:23 -050019
George Keishing4bbf5202017-05-18 06:55:53 -050020# Error strings to check from dmesg.
Steven Sombar130a04f2017-07-16 10:02:37 -050021${ERROR_REGEX} error|GPU|NVRM|nvidia
George Keishing4bbf5202017-05-18 06:55:53 -050022
23# GPU specific error message from dmesg.
Steven Sombar130a04f2017-07-16 10:02:37 -050024${ERROR_DBE_MSG} (DBE) has been detected on GPU
25
26# Inventory - List of I/O devices to collect for Inventory
27@{I/O} communication disk display generic input multimedia
28... network printer tape
29
30# Inventory Paths of the JSON and YAML files
Steven Sombara2f166c2017-08-02 14:22:45 -050031${json_tmp_file_path} ${EXECDIR}/inventory_temp_file.json
32${yaml_file_path} ${EXECDIR}/inventory_temp_file.yaml
Steven Sombar130a04f2017-07-16 10:02:37 -050033
34
George Keishing4ef29082017-05-13 05:01:23 -050035
George Keishing0fff68a2017-04-27 22:25:49 -050036*** Keywords ***
37
George Keishing0fff68a2017-04-27 22:25:49 -050038Login To OS
39 [Documentation] Login to OS Host.
40 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
41 ... ${os_password}=${OS_PASSWORD}
George Keishingcff7aa02017-05-16 08:05:10 -050042 ... ${alias_name}=os_connection
George Keishing91c852f2017-05-03 02:48:21 -050043 # Description of argument(s):
44 # os_host IP address of the OS Host.
45 # os_username OS Host Login user name.
46 # os_password OS Host Login passwrd.
George Keishingcff7aa02017-05-16 08:05:10 -050047 # alias_name Default OS SSH session connection alias name.
George Keishing0fff68a2017-04-27 22:25:49 -050048
George Keishing91c852f2017-05-03 02:48:21 -050049 Ping Host ${os_host}
George Keishing664a0162017-06-05 12:24:24 -050050 SSHLibrary.Open Connection ${os_host} alias=${alias_name}
Steven Sombar06d59682019-02-18 11:03:42 -060051 SSHLibrary.Login ${os_username} ${os_password}
George Keishing91c852f2017-05-03 02:48:21 -050052
53
George Keishing88366b32017-05-12 13:50:48 -050054Tool Exist
55 [Documentation] Check whether given tool is installed on OS.
56 [Arguments] ${tool_name}
57 # Description of argument(s):
58 # tool_name Tool name whose existence is to be checked.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -050059
60 ${output} ${stderr} ${rc}= OS Execute Command which ${tool_name}
George Keishing88366b32017-05-12 13:50:48 -050061 Should Contain ${output} ${tool_name}
62 ... msg=Please install ${tool_name} tool.
George Keishing91c852f2017-05-03 02:48:21 -050063
64
65Boot To OS
66 [Documentation] Boot host OS.
67 Run Key OBMC Boot Test \ REST Power On
68
69
70Power Off Host
71 [Documentation] Power off host.
72 Run Key OBMC Boot Test \ REST Power Off
George Keishing0fff68a2017-04-27 22:25:49 -050073
George Keishing6fec3ab2017-05-05 10:45:08 -050074
75File Exist On OS
76 [Documentation] Check if the given file path exist on OS.
77 [Arguments] ${file_path}
78 # Description of argument(s):
79 # file_path Absolute file path.
80
81 Login To OS
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -050082 ${out} ${stderr} ${rc}= OS Execute Command ls ${file_path}
George Keishing6fec3ab2017-05-05 10:45:08 -050083 Log To Console \n File Exist: ${out}
84
George Keishing8740a0c2017-05-13 07:19:16 -050085
86Is HTX Running
Steven Sombar3ecb6892018-03-23 11:41:08 -050087 [Documentation] Return "True" if the HTX is running, "False"
88 ... otherwise.
89
90 # Example usage:
91 # ${status}= Is HTX Running
92 # Run Keyword If '${status}' == 'True' Shutdown HTX Exerciser
93
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -050094 ${status} ${stderr} ${rc}= OS Execute Command
Joy Onyerikwu4dbd4992018-05-03 16:03:17 -050095 ... htxcmdline -getstats ignore_err=1
96 # Get HTX state
97 # (idle, currently running, selected_mdt but not running).
98 ${running}= Set Variable If
99 ... "Currently running" in """${status}""" ${True} ${False}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500100
George Keishing409df052024-01-17 22:36:14 +0530101 RETURN ${running}
George Keishing4ef29082017-05-13 05:01:23 -0500102
103
104Write Log Data To File
105 [Documentation] Write log data to the logs directory.
106 [Arguments] ${data}= ${log_file_path}=
107 # Description of argument(s):
108 # data String buffer.
109 # log_file_path The log file path.
110
111 Create File ${log_file_path} ${data}
112
113
114Collect HTX Log Files
115 [Documentation] Collect status and error log files.
116 # Collects the following files:
117 # HTX error log file /tmp/htxerr
118 # HTX status log file /tmp/htxstats
119
120 # Create logs directory and get current datetime.
121 Create Directory ${htx_log_dir_path}
122 ${cur_datetime}= Get Current Date result_format=%Y%m%d%H%M%S%f
123
124 File Exist On OS /tmp/htxerr
Joy Onyerikwub9922612018-05-14 12:36:57 -0500125 ${htx_err} ${std_err} ${rc}= OS Execute Command cat /tmp/htxerr
George Keishing4ef29082017-05-13 05:01:23 -0500126 Write Log Data To File
127 ... ${htx_err} ${htx_log_dir_path}/${OS_HOST}${cur_datetime}.htxerr
128
129 File Exist On OS /tmp/htxstats
Joy Onyerikwub9922612018-05-14 12:36:57 -0500130 ${htx_stats} ${std_err} ${rc}= OS Execute Command
131 ... cat /tmp/htxstats
George Keishing4ef29082017-05-13 05:01:23 -0500132 Write Log Data To File
133 ... ${htx_stats} ${htx_log_dir_path}/${OS_HOST}_${cur_datetime}.htxstats
134
George Keishingcff7aa02017-05-16 08:05:10 -0500135
136REST Upload File To BMC
137 [Documentation] Upload a file via REST to BMC.
138
139 # Generate 32 MB file size
140 Run dd if=/dev/zero of=dummyfile bs=1 count=0 seek=32MB
141 OperatingSystem.File Should Exist dummyfile
142
143 # Get the content of the file and upload to BMC
144 ${image_data}= OperatingSystem.Get Binary File dummyfile
145
146 # Get REST session to BMC
147 Initialize OpenBMC
148
149 # Create the REST payload headers and data
150 ${data}= Create Dictionary data ${image_data}
151 ${headers}= Create Dictionary Content-Type=application/octet-stream
152 ... Accept=application/octet-stream
153 Set To Dictionary ${data} headers ${headers}
154
155 ${resp}= Post Request openbmc /upload/image &{data}
George Keishing1c1bf1a2018-11-16 05:20:48 -0600156 Should Be Equal As Strings ${resp.status_code} ${HTTP_BAD_REQUEST}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500157 ... msg=Openbmc /upload/image failed.
George Keishingcff7aa02017-05-16 08:05:10 -0500158
George Keishingcff7aa02017-05-16 08:05:10 -0500159 # Take SSH connection to BMC and switch to BMC connection to perform
160 # the task.
161 &{bmc_connection_args}= Create Dictionary alias=bmc_connection
162 Open Connection And Log In &{bmc_connection_args}
163
164 # Currently OS SSH session is active, switch to BMC connection.
165 Switch Connection bmc_connection
George Keishingcff7aa02017-05-16 08:05:10 -0500166
167 # Switch back to OS SSH connection.
168 Switch Connection os_connection
169
George Keishing4bbf5202017-05-18 06:55:53 -0500170
Steven Sombar1dc6fa72018-01-12 10:58:20 -0600171Get CPU Min Frequency Limit
172 [Documentation] Get CPU minimum assignable frequency.
173
174 # lscpu | grep min returns
175 # CPU min MHz: 1983.0000
176
177 ${cmd}= Catenate lscpu | grep min | tr -dc '0-9.\n'
178 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530179 RETURN ${cpu_freq}
Steven Sombar1dc6fa72018-01-12 10:58:20 -0600180
181
182Get CPU Min Frequency
183 [Documentation] Get CPU assigned minimum frequency.
184
185 # ppc64_cpu --frequency -t 10 returns
186 # min: 3.295 GHz (cpu 143)
187 # max: 3.295 GHz (cpu 0)
188 # avg: 3.295 GHz
189
190 ${cmd}= Catenate ppc64_cpu --frequency -t 10 | grep min
191 ... | cut -f 2 | cut -d ' ' -f 1 | tr -dc '0-9\n'
192 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530193 RETURN ${cpu_freq}
Steven Sombar1dc6fa72018-01-12 10:58:20 -0600194
195
196Get CPU Max Frequency Limit
197 [Documentation] Get CPU maximum assignable frequency.
198
199 # lscpu | grep max returns
200 # CPU max MHz: 3300.0000
201
202 ${cmd}= Catenate lscpu | grep max | tr -dc '0-9.\n'
203 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530204 RETURN ${cpu_freq}
Steven Sombar1dc6fa72018-01-12 10:58:20 -0600205
206
207Get CPU Max Frequency
208 [Documentation] Get CPU assigned maximum frequency.
209
210 # ppc64_cpu --frequency -t 10 returns
211 # min: 3.295 GHz (cpu 143)
212 # max: 3.295 GHz (cpu 0)
213 # avg: 3.295 GHz
214
215 ${cmd}= Catenate ppc64_cpu --frequency -t 10 | grep max
216 ... | cut -f 2 | cut -d ' ' -f 1 | tr -dc '0-9\n'
217 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530218 RETURN ${cpu_freq}
Steven Sombar1dc6fa72018-01-12 10:58:20 -0600219
220
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500221Get CPU Max Temperature
222 [Documentation] Get the highest CPU Temperature.
223
224 ${temperature_objs}= Read Properties
225 ... ${SENSORS_URI}temperature/enumerate
226 # Filter the dictionary to get just the CPU temperature info.
George Keishing36efbc02018-12-12 10:18:23 -0600227 ${cmd}= Catenate {k:v for k,v in $temperature_objs.items()
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500228 ... if re.match('${SENSORS_URI}temperature/p.*core.*temp', k)}
229 ${cpu_temperatuture_objs} Evaluate ${cmd} modules=re
230 # Create a list of the CPU temperature values (current).
231 ${cpu_temperatures}= Evaluate
232 ... [ x['Value'] for x in $cpu_temperatuture_objs.values() ]
233
George Keishing36efbc02018-12-12 10:18:23 -0600234 ${cpu_max_temp} Evaluate int(max(map(int, $cpu_temperatures))/1000)
George Keishing409df052024-01-17 22:36:14 +0530235 RETURN ${cpu_max_temp}
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500236
237
238Get CPU Min Temperature
239 [Documentation] Get the CPU Temperature.
240
241 ${temperature_objs}= Read Properties
242 ... ${SENSORS_URI}temperature/enumerate
243 # Filter the dictionary to get just the CPU temperature info.
George Keishing36efbc02018-12-12 10:18:23 -0600244 ${cmd}= Catenate {k:v for k,v in $temperature_objs.items()
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500245 ... if re.match('${SENSORS_URI}temperature/p.*core.*temp', k)}
246 ${cpu_temperatuture_objs}= Evaluate ${cmd} modules=re
247 # Create a list of the CPU temperature values (current).
248 ${cpu_temperatures}= Evaluate
249 ... [ x['Value'] for x in $cpu_temperatuture_objs.values() ]
250
George Keishing36efbc02018-12-12 10:18:23 -0600251 ${cpu_min_temp} Evaluate int(min(map(int, $cpu_temperatures))/1000)
George Keishing409df052024-01-17 22:36:14 +0530252 RETURN ${cpu_min_temp}
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500253
254
George Keishing4bbf5202017-05-18 06:55:53 -0500255Check For Errors On OS Dmesg Log
256 [Documentation] Check if dmesg has nvidia errors logged.
257
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500258 ${dmesg_log} ${stderr} ${rc}= OS Execute Command
259 ... dmesg | egrep '${ERROR_REGEX}'
George Keishing4bbf5202017-05-18 06:55:53 -0500260 # To enable multiple string check.
Steven Sombar0278b132018-01-09 14:41:32 -0600261 Should Not Contain ${dmesg_log} ${ERROR_DBE_MSG}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500262 ... msg=OS dmesg shows ${ERROR_DBE_MSG}.
George Keishing4bbf5202017-05-18 06:55:53 -0500263
264
265Collect NVIDIA Log File
266 [Documentation] Collect ndivia-smi command output.
George Keishing664a0162017-06-05 12:24:24 -0500267 [Arguments] ${suffix}
268 # Description of argument(s):
269 # suffix String name to append.
George Keishing4bbf5202017-05-18 06:55:53 -0500270
271 # Collects the output of ndivia-smi cmd output.
Steven Sombare27cca92018-02-28 10:56:38 -0600272 # +--------------------------------------------------------------------+
273 # | NVIDIA-SMI 361.89 Driver Version: 361.89 |
274 # |-------------------------------+----------------------+-------------+
275 # | GPU Name Persistence-M| Bus-Id Disp.A | GPU ECC |
276 # | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | Utiliz err |
277 # |===============================+======================+=============|
278 # | 0 Tesla P100-SXM2... On | 0002:01:00.0 Off | 0 |
279 # | N/A 25C P0 35W / 300W | 931MiB / 16280MiB | 0% Default |
280 # +-------------------------------+----------------------+-------------+
281 # | 1 Tesla P100-SXM2... On | 0003:01:00.0 Off | 0 |
282 # | N/A 26C P0 40W / 300W | 1477MiB / 16280MiB | 0% Default |
283 # +-------------------------------+----------------------+-------------+
284 # | 2 Tesla P100-SXM2... On | 0006:01:00.0 Off | 0 |
285 # | N/A 25C P0 35W / 300W | 931MiB / 16280MiB | 0% Default |
286 # +-------------------------------+----------------------+-------------+
287 # | 3 Tesla P100-SXM2... On | 0007:01:00.0 Off | 0 |
288 # | N/A 44C P0 290W / 300W | 965MiB / 16280MiB | 0% Default |
289 # +-------------------------------+----------------------+-------------+
290 # +--------------------------------------------------------------------+
291 # | Processes: GPU Memory |
292 # | GPU PID Type Process name Usage |
293 # |====================================================================|
294 # | 0 28459 C hxenvidia 929MiB |
295 # | 1 28460 C hxenvidia 1475MiB |
296 # | 2 28461 C hxenvidia 929MiB |
297 # | 3 28462 C hxenvidia 963MiB |
298 # +--------------------------------------------------------------------+
George Keishing4bbf5202017-05-18 06:55:53 -0500299
300 # Create logs directory and get current datetime.
301 Create Directory ${htx_log_dir_path}
302 ${cur_datetime}= Get Current Date result_format=%Y%m%d%H%M%S%f
303
Steven Sombar0278b132018-01-09 14:41:32 -0600304 ${nvidia_out} ${stderr} ${rc}= OS Execute Command nvidia-smi
George Keishing4bbf5202017-05-18 06:55:53 -0500305 Write Log Data To File
George Keishing664a0162017-06-05 12:24:24 -0500306 ... ${nvidia_out}
307 ... ${htx_log_dir_path}/${OS_HOST}_${cur_datetime}.nvidia_${suffix}
George Keishingbbfec9e2017-05-17 10:47:21 -0500308
309
Steven Sombar0278b132018-01-09 14:41:32 -0600310Get GPU Power Limit
311 [Documentation] Get NVIDIA GPU maximum permitted power draw.
George Keishingbbfec9e2017-05-17 10:47:21 -0500312
Steven Sombar0278b132018-01-09 14:41:32 -0600313 # nvidia-smi --query-gpu=power.limit --format=csv returns
314 # power.limit [W]
315 # 300.00 W
316 # 300.00 W
317 # 300.00 W
318 # 300.00 W
George Keishingbbfec9e2017-05-17 10:47:21 -0500319
Steven Sombar0278b132018-01-09 14:41:32 -0600320 ${cmd}= Catenate nvidia-smi --query-gpu=power.limit
321 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
322 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
323 # Allow for sensor overshoot. That is, max power reported for
324 # a GPU could be a few watts above the limit.
325 ${power_max}= Evaluate ${nvidia_out}+${7.00}
George Keishing409df052024-01-17 22:36:14 +0530326 RETURN ${power_max}
Steven Sombar0278b132018-01-09 14:41:32 -0600327
328
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500329Get GPU Max Power
330 [Documentation] Get the maximum GPU power dissipation.
Steven Sombar0278b132018-01-09 14:41:32 -0600331
332 # nvidia-smi --query-gpu=power.draw --format=csv returns
333 # power.draw [W]
334 # 34.12 W
335 # 34.40 W
336 # 36.55 W
337 # 36.05 W
338
339 ${cmd}= Catenate nvidia-smi --query-gpu=power.draw
340 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
341 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530342 RETURN ${nvidia_out}
Steven Sombar0278b132018-01-09 14:41:32 -0600343
344
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500345Get GPU Min Power
346 [Documentation] Return the minimum GPU power value as record by
347 ... nvidia-smi.
348
349 ${cmd}= Catenate nvidia-smi --query-gpu=power.draw --format=csv |
350 ... grep -v 'power.draw' | cut -f 1 -d ' ' | sort -n -u | head -1
351 ${gpu_min_power} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530352 RETURN ${gpu_min_power}
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500353
354
Steven Sombar0278b132018-01-09 14:41:32 -0600355Get GPU Temperature Limit
356 [Documentation] Get NVIDIA GPU maximum permitted temperature.
357
358 # nvidia-smi -q -d TEMPERATURE | grep "GPU Max" returns
359 # GPU Max Operating Temp : 83 C
360 # GPU Max Operating Temp : 83 C
361 # GPU Max Operating Temp : 83 C
362 # GPU Max Operating Temp : 83 C
363
364 ${cmd}= Catenate nvidia-smi -q -d TEMPERATURE | grep "GPU Max"
365 ... | cut -f 2 -d ":" | tr -dc '0-9\n' | sort -n -u | tail -n 1
366 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530367 RETURN ${nvidia_out}
Steven Sombar0278b132018-01-09 14:41:32 -0600368
369
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500370Get GPU Min Temperature
371 [Documentation] Get the minimum GPU temperature.
372
373 ${cmd}= Catenate nvidia-smi --query-gpu=temperature.gpu
374 ... --format=csv | grep -v 'temperature.gpu' | sort -n -u | head -1
375 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530376 RETURN ${nvidia_out}
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500377
378
379Get GPU Max Temperature
380 [Documentation] Get the maximum GPU temperature.
Steven Sombar0278b132018-01-09 14:41:32 -0600381
382 # nvidia-smi --query-gpu=temperature.gpu --format=csv returns
383 # 38
384 # 41
385 # 38
386 # 40
387
388 ${cmd}= Catenate nvidia-smi --query-gpu=temperature.gpu
389 ... --format=csv | sort -n -u | tail -n 1
390 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530391 RETURN ${nvidia_out}
Steven Sombar0278b132018-01-09 14:41:32 -0600392
393
Steven Sombarc02dde82018-05-01 09:49:14 -0500394Get GPU Temperature Via REST
395 [Documentation] Return the temperature in degrees C of the warmest GPU
396 ... as reportd by REST.
397
398 # NOTE: This endpoint path is not defined until system has been powered-on.
399 ${temperature_objs}= Read Properties ${SENSORS_URI}temperature/enumerate
400 ... timeout=30 quiet=1
401
George Keishing36efbc02018-12-12 10:18:23 -0600402 ${core_temperatures_list}= Catenate {k:v for k,v in $temperature_objs.items()
Steven Sombarc02dde82018-05-01 09:49:14 -0500403 ... if re.match('${SENSORS_URI}temperature/.*_core_temp', k)}
404 ${gpu_temperature_objs_dict}= Evaluate ${core_temperatures_list} modules=re
405
406 # Create a list containing all of the GPU temperatures.
407 ${gpu_temperatures}= Evaluate
408 ... [ x['Value'] for x in $gpu_temperature_objs_dict.values() ]
409
410 # Find the max temperature value and divide by 1000 to get just the integer
411 # portion.
George Keishing36efbc02018-12-12 10:18:23 -0600412 ${max_gpu_temperature}= Evaluate
413 ... int(max(map(int, $gpu_temperatures))/1000)
Steven Sombarc02dde82018-05-01 09:49:14 -0500414
George Keishing409df052024-01-17 22:36:14 +0530415 RETURN ${max_gpu_temperature}
Steven Sombarc02dde82018-05-01 09:49:14 -0500416
417
Steven Sombar0278b132018-01-09 14:41:32 -0600418Get GPU Clock Limit
419 [Documentation] Get NVIDIA GPU maximum permitted graphics clock.
420
421 # nvidia-smi --query-gpu=clocks.max.gr --format=csv returns
422 # 1530 MHz
423 # 1530 MHz
424 # 1530 MHz
425 # 1530 MHz
426
427 ${cmd}= Catenate nvidia-smi --query-gpu=clocks.max.gr
428 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
429 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530430 RETURN ${nvidia_out}
Steven Sombar0278b132018-01-09 14:41:32 -0600431
432
433Get GPU Clock
434 [Documentation] Get the highest assigned value of the GPU graphics clock.
435
436 # nvidia-smi --query-gpu=clocks.gr --format=csv returns
437 # 1230 MHz
438 # 1230 MHz
439 # 135 MHz
440 # 150 MHz
441
442 ${cmd}= Catenate nvidia-smi --query-gpu=clocks.gr
443 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
444 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
George Keishing409df052024-01-17 22:36:14 +0530445 RETURN ${nvidia_out}
Steven Sombar0278b132018-01-09 14:41:32 -0600446
447
448Count GPUs From BMC
449 [Documentation] Determine number of GPUs from the BMC. Hostboot
450 ... needs to have been run previously because the BMC gets GPU data
451 ... from Hostboot.
452
453 # Example of gv* endpoint data:
454 # "/xyz/openbmc_project/inventory/system/chassis/motherboard/gv100card0": {
455 # "Functional": 1,
456 # "Present": 1,
457 # "PrettyName": ""
458 # },
459
460 ${num_bmc_gpus}= Set Variable ${0}
461
462 ${gpu_list}= Get Endpoint Paths
463 ... ${HOST_INVENTORY_URI}system/chassis/motherboard gv*
464
Sushil Singh552a3c62019-06-13 05:34:18 -0500465 FOR ${gpu_uri} IN @{gpu_list}
466 ${present}= Read Attribute ${gpu_uri} Present
467 ${state}= Read Attribute ${gpu_uri} Functional
468 Rpvars gpu_uri present state
469 ${num_bmc_gpus}= Run Keyword If ${present} and ${state}
470 ... Evaluate ${num_bmc_gpus}+${1}
471 END
George Keishing409df052024-01-17 22:36:14 +0530472 RETURN ${num_bmc_gpus}
George Keishingbbfec9e2017-05-17 10:47:21 -0500473
474
475Create Default MDT Profile
476 [Documentation] Create default mdt.bu profile and run.
477
Steven Sombar021cbc42019-03-28 11:49:21 -0500478 Print Timen Create HTX mdt profile.
George Keishingbbfec9e2017-05-17 10:47:21 -0500479
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500480 ${profile} ${stderr} ${rc}= OS Execute Command
481 ... htxcmdline -createmdt
Steven Sombar021cbc42019-03-28 11:49:21 -0500482 Printn ${profile}
George Keishingbbfec9e2017-05-17 10:47:21 -0500483 Should Contain ${profile} mdts are created successfully
Steven Sombar3ecb6892018-03-23 11:41:08 -0500484 ... msg=Create MDT profile failed. response=${profile}
George Keishingbbfec9e2017-05-17 10:47:21 -0500485
486
487Run MDT Profile
488 [Documentation] Load user pre-defined MDT profile.
Daniel Gonzalezbfd8aff2018-03-27 10:20:37 -0600489 [Arguments] ${HTX_MDT_PROFILE}=${HTX_MDT_PROFILE}
490 # Description of argument(s):
491 # HTX_MDT_PROFILE MDT profile to be executed (e.g. "mdt.bu").
George Keishingbbfec9e2017-05-17 10:47:21 -0500492
Steven Sombar021cbc42019-03-28 11:49:21 -0500493 Print Timen Start HTX mdt profile execution.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500494 ${htx_run} ${stderr} ${rc}= OS Execute Command
George Keishingbbfec9e2017-05-17 10:47:21 -0500495 ... htxcmdline -run -mdt ${HTX_MDT_PROFILE}
Steven Sombar021cbc42019-03-28 11:49:21 -0500496 Printn ${htx_run}
George Keishingbbfec9e2017-05-17 10:47:21 -0500497 Should Contain ${htx_run} Activated
Steven Sombar3ecb6892018-03-23 11:41:08 -0500498 ... msg=htxcmdline run mdt did not return "Activated" status.
George Keishingbbfec9e2017-05-17 10:47:21 -0500499
500
501Check HTX Run Status
502 [Documentation] Get HTX exerciser status and check for error.
Steven Sombar9d5fc2d2018-04-30 09:09:13 -0500503 [Arguments] ${sleep_time}=${0}
Joy Onyerikwua8074172018-04-20 07:08:19 -0500504
505 # Description of argument(s):
Steven Sombar56845272018-07-18 14:32:01 -0500506 # sleep_time The amount of time to sleep after checking status,
507 # for example "3s" or "2m".
George Keishingbbfec9e2017-05-17 10:47:21 -0500508
Steven Sombar021cbc42019-03-28 11:49:21 -0500509 Print Timen Check HTX mdt Status and error.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500510 ${htx_status} ${stderr} ${rc}= OS Execute Command
George Keishingbbfec9e2017-05-17 10:47:21 -0500511 ... htxcmdline -status -mdt ${HTX_MDT_PROFILE}
Steven Sombar021cbc42019-03-28 11:49:21 -0500512 Printn ${htx_status}
George Keishingbbfec9e2017-05-17 10:47:21 -0500513
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500514 ${htx_errlog} ${stderr} ${rc}= OS Execute Command
515 ... htxcmdline -geterrlog
Steven Sombar021cbc42019-03-28 11:49:21 -0500516 Printn ${htx_errlog}
George Keishingbbfec9e2017-05-17 10:47:21 -0500517
George Keishing2d2df512021-04-12 10:11:15 -0500518 Should Contain ${htx_errlog} file </tmp/htx/htxerr> is empty
Steven Sombar3ecb6892018-03-23 11:41:08 -0500519 ... msg=HTX geterrorlog was not empty.
George Keishingbbfec9e2017-05-17 10:47:21 -0500520
Steven Sombar56845272018-07-18 14:32:01 -0500521 Return From Keyword If "${sleep_time}" == "${0}"
Joy Onyerikwua8074172018-04-20 07:08:19 -0500522
Steven Sombar56845272018-07-18 14:32:01 -0500523 Run Key U Sleep \ ${sleep_time}
Joy Onyerikwua8074172018-04-20 07:08:19 -0500524
George Keishingbbfec9e2017-05-17 10:47:21 -0500525
526Shutdown HTX Exerciser
527 [Documentation] Shut down HTX exerciser run.
528
Steven Sombar021cbc42019-03-28 11:49:21 -0500529 Print Timen Shutdown HTX Run
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500530 ${shutdown} ${stderr} ${rc}= OS Execute Command
George Keishingbbfec9e2017-05-17 10:47:21 -0500531 ... htxcmdline -shutdown -mdt ${HTX_MDT_PROFILE}
Steven Sombare27cca92018-02-28 10:56:38 -0600532
Steven Sombar021cbc42019-03-28 11:49:21 -0500533 Printn ${shutdown}
534
535 ${down1}= Evaluate 'shutdown successfully' in $shutdown
536 Return From Keyword If '${down1}' == 'True'
537 ${down2}= Evaluate 'No MDT is currently running' in $shutdown
538 Return From Keyword If '${down2}' == 'True'
539 Fail msg=Shutdown returned unexpected message.
Steven Sombare27cca92018-02-28 10:56:38 -0600540
George Keishingbbfec9e2017-05-17 10:47:21 -0500541
Steven Sombar130a04f2017-07-16 10:02:37 -0500542Create JSON Inventory File
543 [Documentation] Create a JSON inventory file, and make a YAML copy.
544 [Arguments] ${json_file_path}
545 # Description of argument:
George Keishing9bfdf8a2024-04-19 11:09:07 +0530546 # json_file_path Where the inventory file is written to.
Steven Sombar130a04f2017-07-16 10:02:37 -0500547
548 Login To OS
549 Compile Inventory JSON
550 Run json2yaml ${json_tmp_file_path} ${yaml_file_path}
551 # Format to JSON pretty print to file.
552 Run python -m json.tool ${json_tmp_file_path} > ${json_file_path}
553 OperatingSystem.File Should Exist ${json_file_path}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500554 ... msg=File ${json_file_path} does not exist.
Steven Sombar130a04f2017-07-16 10:02:37 -0500555
556
557Compile Inventory JSON
558 [Documentation] Compile the Inventory into a JSON file.
559 Create File ${json_tmp_file_path}
560 Write New JSON List ${json_tmp_file_path} Inventory
561 Retrieve HW Info And Write processor ${json_tmp_file_path}
562 Retrieve HW Info And Write memory ${json_tmp_file_path}
563 Retrieve HW Info And Write List ${I/O} ${json_tmp_file_path} I/O last
564 Close New JSON List ${json_tmp_file_path}
565
Steven Sombar3ecb6892018-03-23 11:41:08 -0500566
Steven Sombar130a04f2017-07-16 10:02:37 -0500567Write New JSON List
568 [Documentation] Start a new JSON list element in file.
569 [Arguments] ${json_tmp_file_path} ${json_field_name}
570 # Description of argument(s):
571 # json_tmp_file_path Name of file to write to.
572 # json_field_name Name to give json list element.
573 Append to File ${json_tmp_file_path} { "${json_field_name}" : [
574
Steven Sombar3ecb6892018-03-23 11:41:08 -0500575
Steven Sombar130a04f2017-07-16 10:02:37 -0500576Close New JSON List
577 [Documentation] Close JSON list element in file.
578 [Arguments] ${json_tmp_file_path}
579 # Description of argument(s):
580 # json_tmp_file_path Path of file to write to.
581 Append to File ${json_tmp_file_path} ]}
582
Steven Sombar3ecb6892018-03-23 11:41:08 -0500583
Steven Sombar130a04f2017-07-16 10:02:37 -0500584Retrieve HW Info And Write
585 [Documentation] Retrieve and write info, add a comma if not last item.
586 [Arguments] ${class} ${json_tmp_file_path} ${last}=false
587 # Description of argument(s):
588 # class Device class to retrieve with lshw.
589 # json_tmp_file_path Path of file to write to.
590 # last Is this the last element in the parent JSON?
591 Write New JSON List ${json_tmp_file_path} ${class}
George Keishingfb218a12018-11-28 11:20:42 -0600592 ${output}= Retrieve Hardware Info ${class}
593 ${output}= Clean Up String ${output}
Steven Sombar130a04f2017-07-16 10:02:37 -0500594 Run Keyword if ${output.__class__ is not type(None)}
595 ... Append To File ${json_tmp_file_path} ${output}
596 Close New JSON List ${json_tmp_file_path}
597 Run Keyword if '${last}' == 'false'
598 ... Append to File ${json_tmp_file_path} ,
599
Steven Sombar3ecb6892018-03-23 11:41:08 -0500600
Steven Sombar130a04f2017-07-16 10:02:37 -0500601Retrieve HW Info And Write List
602 [Documentation] Does a Retrieve/Write with a list of classes and
603 ... encapsulates them into one large JSON element.
604 [Arguments] ${list} ${json_tmp_file_path} ${json_field_name}
605 ... ${last}=false
606 # Description of argument(s):
607 # list The list of devices classes to retrieve with lshw.
608 # json_tmp_file_path Path of file to write to.
609 # json_field_name Name of the JSON element to encapsulate this list.
610 # last Is this the last element in the parent JSON?
611 Write New JSON List ${json_tmp_file_path} ${json_field_name}
Sushil Singh552a3c62019-06-13 05:34:18 -0500612 FOR ${class} IN @{list}
613 ${tail} Get From List ${list} -1
614 Run Keyword if '${tail}' == '${class}'
615 ... Retrieve HW Info And Write ${class} ${json_tmp_file_path} true
616 ... ELSE Retrieve HW Info And Write ${class} ${json_tmp_file_path}
617 END
Steven Sombar130a04f2017-07-16 10:02:37 -0500618 Close New JSON List ${json_tmp_file_path}
619 Run Keyword if '${last}' == 'false'
620 ... Append to File ${json_tmp_file_path} ,
621
Steven Sombar3ecb6892018-03-23 11:41:08 -0500622
Steven Sombar130a04f2017-07-16 10:02:37 -0500623Retrieve Hardware Info
624 [Documentation] Retrieves the lshw output of the device class as JSON.
625 [Arguments] ${class}
626 # Description of argument(s):
627 # class Device class to retrieve with lshw.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500628 ${output} ${stderr} ${rc}= OS Execute Command lshw -c ${class} -json
George Keishingfb218a12018-11-28 11:20:42 -0600629 ${output}= Verify JSON string ${output}
George Keishing409df052024-01-17 22:36:14 +0530630 RETURN ${output}
Steven Sombar130a04f2017-07-16 10:02:37 -0500631
Steven Sombar3ecb6892018-03-23 11:41:08 -0500632
Steven Sombar130a04f2017-07-16 10:02:37 -0500633Verify JSON String
Gunnar Mills28e403b2017-10-25 16:16:38 -0500634 [Documentation] Ensure the JSON string content is separated by commas.
Steven Sombar130a04f2017-07-16 10:02:37 -0500635 [Arguments] ${unver_string}
636 # Description of argument(s):
637 # unver_string JSON String we will be checking for lshw comma errors.
George Keishingfb218a12018-11-28 11:20:42 -0600638 ${unver_string}= Convert to String ${unver_string}
639 ${ver_string}= Replace String Using Regexp ${unver_string} }\\s*{ },{
George Keishing409df052024-01-17 22:36:14 +0530640 RETURN ${ver_string}
Steven Sombar130a04f2017-07-16 10:02:37 -0500641
Steven Sombar3ecb6892018-03-23 11:41:08 -0500642
Steven Sombar130a04f2017-07-16 10:02:37 -0500643Clean Up String
644 [Documentation] Remove extra whitespace and trailing commas.
645 [Arguments] ${dirty_string}
646 # Description of argument(s):
647 # dirty_string String that will be space stripped and have comma removed.
George Keishingfb218a12018-11-28 11:20:42 -0600648 ${clean_string}= Strip String ${dirty_string}
649 ${last_char}= Get Substring ${clean_string} -1
650 ${trimmed_string}= Get Substring ${clean_string} 0 -1
651 ${clean_string}= Set Variable If '${last_char}' == ','
Steven Sombar130a04f2017-07-16 10:02:37 -0500652 ... ${trimmed_string} ${clean_string}
George Keishing409df052024-01-17 22:36:14 +0530653 RETURN ${clean_string}
Joy Onyerikwu4cb8c0a2018-04-11 18:05:13 -0500654
655
656Get OS Network Interface Names
657 [Documentation] Return a list of interface names on the OS.
658
659 ${stdout} ${stderr} ${rc}= OS Execute Command ls /sys/class/net
660 @{interface_names}= Split String ${stdout}
George Keishing409df052024-01-17 22:36:14 +0530661 RETURN @{interface_names}
Daniel Gonzalezbfd8aff2018-03-27 10:20:37 -0600662
663
664Run Soft Bootme
665 [Documentation] Run a soft bootme for a period of an hour.
666 [Arguments] ${bootme_period}=3
667 # Description of argument(s):
668 # bootme_time Bootme period to be rebooting the system.
669
670 ${output} ${stderr} ${rc}= OS Execute Command
671 ... htxcmdline -bootme on mode:soft period:${bootme_period}
672 Should Contain ${output} bootme on is completed successfully
673
674
675Shutdown Bootme
676 [Documentation] Stop the bootme process.
677
678 ${output} ${stderr} ${rc}= OS Execute Command htxcmdline -bootme off
Steven Sombar56845272018-07-18 14:32:01 -0500679 Should Contain ${output} bootme off is completed successfully