blob: 02326abc1a4cc1f81b36c8417c3bb28533d54f85 [file] [log] [blame]
George Keishing0fff68a2017-04-27 22:25:49 -05001*** Settings ***
Steven Sombar130a04f2017-07-16 10:02:37 -05002Documentation Keywords for system test.
George Keishing0fff68a2017-04-27 22:25:49 -05003
George Keishing91c852f2017-05-03 02:48:21 -05004Library ../lib/gen_robot_keyword.py
Steven Sombar07be3742018-01-11 13:02:40 -06005Resource ../lib/boot_utils.robot
George Keishing91c852f2017-05-03 02:48:21 -05006Resource ../extended/obmc_boot_test_resource.robot
George Keishing0fff68a2017-04-27 22:25:49 -05007Resource ../lib/utils.robot
George Keishing91c852f2017-05-03 02:48:21 -05008Resource ../lib/state_manager.robot
George Keishingcff7aa02017-05-16 08:05:10 -05009Resource ../lib/rest_client.robot
George Keishingbbfec9e2017-05-17 10:47:21 -050010Resource resource.txt
George Keishing4ef29082017-05-13 05:01:23 -050011Library OperatingSystem
12Library DateTime
13
George Keishing0fff68a2017-04-27 22:25:49 -050014*** Variables ***
15
Steven Sombar130a04f2017-07-16 10:02:37 -050016${htx_log_dir_path} ${EXECDIR}${/}logs${/}
George Keishing4ef29082017-05-13 05:01:23 -050017
George Keishing4bbf5202017-05-18 06:55:53 -050018# Error strings to check from dmesg.
Steven Sombar130a04f2017-07-16 10:02:37 -050019${ERROR_REGEX} error|GPU|NVRM|nvidia
George Keishing4bbf5202017-05-18 06:55:53 -050020
21# GPU specific error message from dmesg.
Steven Sombar130a04f2017-07-16 10:02:37 -050022${ERROR_DBE_MSG} (DBE) has been detected on GPU
23
24# Inventory - List of I/O devices to collect for Inventory
25@{I/O} communication disk display generic input multimedia
26... network printer tape
27
28# Inventory Paths of the JSON and YAML files
Steven Sombara2f166c2017-08-02 14:22:45 -050029${json_tmp_file_path} ${EXECDIR}/inventory_temp_file.json
30${yaml_file_path} ${EXECDIR}/inventory_temp_file.yaml
Steven Sombar130a04f2017-07-16 10:02:37 -050031
32
George Keishing4ef29082017-05-13 05:01:23 -050033
George Keishing0fff68a2017-04-27 22:25:49 -050034*** Keywords ***
35
George Keishing0fff68a2017-04-27 22:25:49 -050036Login To OS
37 [Documentation] Login to OS Host.
38 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
39 ... ${os_password}=${OS_PASSWORD}
George Keishingcff7aa02017-05-16 08:05:10 -050040 ... ${alias_name}=os_connection
George Keishing91c852f2017-05-03 02:48:21 -050041 # Description of argument(s):
42 # os_host IP address of the OS Host.
43 # os_username OS Host Login user name.
44 # os_password OS Host Login passwrd.
George Keishingcff7aa02017-05-16 08:05:10 -050045 # alias_name Default OS SSH session connection alias name.
George Keishing0fff68a2017-04-27 22:25:49 -050046
George Keishing91c852f2017-05-03 02:48:21 -050047 Ping Host ${os_host}
George Keishing664a0162017-06-05 12:24:24 -050048 SSHLibrary.Open Connection ${os_host} alias=${alias_name}
George Keishing91c852f2017-05-03 02:48:21 -050049 Login ${os_username} ${os_password}
50
51
George Keishing88366b32017-05-12 13:50:48 -050052Tool Exist
53 [Documentation] Check whether given tool is installed on OS.
54 [Arguments] ${tool_name}
55 # Description of argument(s):
56 # tool_name Tool name whose existence is to be checked.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -050057
58 ${output} ${stderr} ${rc}= OS Execute Command which ${tool_name}
George Keishing88366b32017-05-12 13:50:48 -050059 Should Contain ${output} ${tool_name}
60 ... msg=Please install ${tool_name} tool.
George Keishing91c852f2017-05-03 02:48:21 -050061
62
63Boot To OS
64 [Documentation] Boot host OS.
65 Run Key OBMC Boot Test \ REST Power On
66
67
68Power Off Host
69 [Documentation] Power off host.
70 Run Key OBMC Boot Test \ REST Power Off
George Keishing0fff68a2017-04-27 22:25:49 -050071
George Keishing6fec3ab2017-05-05 10:45:08 -050072
73File Exist On OS
74 [Documentation] Check if the given file path exist on OS.
75 [Arguments] ${file_path}
76 # Description of argument(s):
77 # file_path Absolute file path.
78
79 Login To OS
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -050080 ${out} ${stderr} ${rc}= OS Execute Command ls ${file_path}
George Keishing6fec3ab2017-05-05 10:45:08 -050081 Log To Console \n File Exist: ${out}
82
George Keishing8740a0c2017-05-13 07:19:16 -050083
84Is HTX Running
Steven Sombar3ecb6892018-03-23 11:41:08 -050085 [Documentation] Return "True" if the HTX is running, "False"
86 ... otherwise.
87
88 # Example usage:
89 # ${status}= Is HTX Running
90 # Run Keyword If '${status}' == 'True' Shutdown HTX Exerciser
91
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -050092 ${status} ${stderr} ${rc}= OS Execute Command
Joy Onyerikwu4dbd4992018-05-03 16:03:17 -050093 ... htxcmdline -getstats ignore_err=1
94 # Get HTX state
95 # (idle, currently running, selected_mdt but not running).
96 ${running}= Set Variable If
97 ... "Currently running" in """${status}""" ${True} ${False}
Steven Sombar3ecb6892018-03-23 11:41:08 -050098
Steven Sombar3ecb6892018-03-23 11:41:08 -050099 [Return] ${running}
George Keishing4ef29082017-05-13 05:01:23 -0500100
101
102Write Log Data To File
103 [Documentation] Write log data to the logs directory.
104 [Arguments] ${data}= ${log_file_path}=
105 # Description of argument(s):
106 # data String buffer.
107 # log_file_path The log file path.
108
109 Create File ${log_file_path} ${data}
110
111
112Collect HTX Log Files
113 [Documentation] Collect status and error log files.
114 # Collects the following files:
115 # HTX error log file /tmp/htxerr
116 # HTX status log file /tmp/htxstats
117
118 # Create logs directory and get current datetime.
119 Create Directory ${htx_log_dir_path}
120 ${cur_datetime}= Get Current Date result_format=%Y%m%d%H%M%S%f
121
122 File Exist On OS /tmp/htxerr
Joy Onyerikwub9922612018-05-14 12:36:57 -0500123 ${htx_err} ${std_err} ${rc}= OS Execute Command cat /tmp/htxerr
George Keishing4ef29082017-05-13 05:01:23 -0500124 Write Log Data To File
125 ... ${htx_err} ${htx_log_dir_path}/${OS_HOST}${cur_datetime}.htxerr
126
127 File Exist On OS /tmp/htxstats
Joy Onyerikwub9922612018-05-14 12:36:57 -0500128 ${htx_stats} ${std_err} ${rc}= OS Execute Command
129 ... cat /tmp/htxstats
George Keishing4ef29082017-05-13 05:01:23 -0500130 Write Log Data To File
131 ... ${htx_stats} ${htx_log_dir_path}/${OS_HOST}_${cur_datetime}.htxstats
132
George Keishingcff7aa02017-05-16 08:05:10 -0500133
134REST Upload File To BMC
135 [Documentation] Upload a file via REST to BMC.
136
137 # Generate 32 MB file size
138 Run dd if=/dev/zero of=dummyfile bs=1 count=0 seek=32MB
139 OperatingSystem.File Should Exist dummyfile
140
141 # Get the content of the file and upload to BMC
142 ${image_data}= OperatingSystem.Get Binary File dummyfile
143
144 # Get REST session to BMC
145 Initialize OpenBMC
146
147 # Create the REST payload headers and data
148 ${data}= Create Dictionary data ${image_data}
149 ${headers}= Create Dictionary Content-Type=application/octet-stream
150 ... Accept=application/octet-stream
151 Set To Dictionary ${data} headers ${headers}
152
153 ${resp}= Post Request openbmc /upload/image &{data}
George Keishing1c1bf1a2018-11-16 05:20:48 -0600154 Should Be Equal As Strings ${resp.status_code} ${HTTP_BAD_REQUEST}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500155 ... msg=Openbmc /upload/image failed.
George Keishingcff7aa02017-05-16 08:05:10 -0500156
George Keishingcff7aa02017-05-16 08:05:10 -0500157 # Take SSH connection to BMC and switch to BMC connection to perform
158 # the task.
159 &{bmc_connection_args}= Create Dictionary alias=bmc_connection
160 Open Connection And Log In &{bmc_connection_args}
161
162 # Currently OS SSH session is active, switch to BMC connection.
163 Switch Connection bmc_connection
George Keishingcff7aa02017-05-16 08:05:10 -0500164
165 # Switch back to OS SSH connection.
166 Switch Connection os_connection
167
George Keishing4bbf5202017-05-18 06:55:53 -0500168
Steven Sombar1dc6fa72018-01-12 10:58:20 -0600169Get CPU Min Frequency Limit
170 [Documentation] Get CPU minimum assignable frequency.
171
172 # lscpu | grep min returns
173 # CPU min MHz: 1983.0000
174
175 ${cmd}= Catenate lscpu | grep min | tr -dc '0-9.\n'
176 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
177 [Return] ${cpu_freq}
178
179
180Get CPU Min Frequency
181 [Documentation] Get CPU assigned minimum frequency.
182
183 # ppc64_cpu --frequency -t 10 returns
184 # min: 3.295 GHz (cpu 143)
185 # max: 3.295 GHz (cpu 0)
186 # avg: 3.295 GHz
187
188 ${cmd}= Catenate ppc64_cpu --frequency -t 10 | grep min
189 ... | cut -f 2 | cut -d ' ' -f 1 | tr -dc '0-9\n'
190 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
191 [Return] ${cpu_freq}
192
193
194Get CPU Max Frequency Limit
195 [Documentation] Get CPU maximum assignable frequency.
196
197 # lscpu | grep max returns
198 # CPU max MHz: 3300.0000
199
200 ${cmd}= Catenate lscpu | grep max | tr -dc '0-9.\n'
201 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
202 [Return] ${cpu_freq}
203
204
205Get CPU Max Frequency
206 [Documentation] Get CPU assigned maximum frequency.
207
208 # ppc64_cpu --frequency -t 10 returns
209 # min: 3.295 GHz (cpu 143)
210 # max: 3.295 GHz (cpu 0)
211 # avg: 3.295 GHz
212
213 ${cmd}= Catenate ppc64_cpu --frequency -t 10 | grep max
214 ... | cut -f 2 | cut -d ' ' -f 1 | tr -dc '0-9\n'
215 ${cpu_freq} ${stderr} ${rc}= OS Execute Command ${cmd}
216 [Return] ${cpu_freq}
217
218
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500219Get CPU Max Temperature
220 [Documentation] Get the highest CPU Temperature.
221
222 ${temperature_objs}= Read Properties
223 ... ${SENSORS_URI}temperature/enumerate
224 # Filter the dictionary to get just the CPU temperature info.
225 ${cmd}= Catenate {k:v for k,v in $temperature_objs.iteritems()
226 ... if re.match('${SENSORS_URI}temperature/p.*core.*temp', k)}
227 ${cpu_temperatuture_objs} Evaluate ${cmd} modules=re
228 # Create a list of the CPU temperature values (current).
229 ${cpu_temperatures}= Evaluate
230 ... [ x['Value'] for x in $cpu_temperatuture_objs.values() ]
231
232 ${cpu_max_temp} Evaluate max(map(int, $cpu_temperatures))/1000
233 [Return] ${cpu_max_temp}
234
235
236Get CPU Min Temperature
237 [Documentation] Get the CPU Temperature.
238
239 ${temperature_objs}= Read Properties
240 ... ${SENSORS_URI}temperature/enumerate
241 # Filter the dictionary to get just the CPU temperature info.
242 ${cmd}= Catenate {k:v for k,v in $temperature_objs.iteritems()
243 ... if re.match('${SENSORS_URI}temperature/p.*core.*temp', k)}
244 ${cpu_temperatuture_objs}= Evaluate ${cmd} modules=re
245 # Create a list of the CPU temperature values (current).
246 ${cpu_temperatures}= Evaluate
247 ... [ x['Value'] for x in $cpu_temperatuture_objs.values() ]
248
249 ${cpu_min_temp} Evaluate min(map(int, $cpu_temperatures))/1000
250 [Return] ${cpu_min_temp}
251
252
George Keishing4bbf5202017-05-18 06:55:53 -0500253Check For Errors On OS Dmesg Log
254 [Documentation] Check if dmesg has nvidia errors logged.
255
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500256 ${dmesg_log} ${stderr} ${rc}= OS Execute Command
257 ... dmesg | egrep '${ERROR_REGEX}'
George Keishing4bbf5202017-05-18 06:55:53 -0500258 # To enable multiple string check.
Steven Sombar0278b132018-01-09 14:41:32 -0600259 Should Not Contain ${dmesg_log} ${ERROR_DBE_MSG}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500260 ... msg=OS dmesg shows ${ERROR_DBE_MSG}.
George Keishing4bbf5202017-05-18 06:55:53 -0500261
262
263Collect NVIDIA Log File
264 [Documentation] Collect ndivia-smi command output.
George Keishing664a0162017-06-05 12:24:24 -0500265 [Arguments] ${suffix}
266 # Description of argument(s):
267 # suffix String name to append.
George Keishing4bbf5202017-05-18 06:55:53 -0500268
269 # Collects the output of ndivia-smi cmd output.
Steven Sombare27cca92018-02-28 10:56:38 -0600270 # +--------------------------------------------------------------------+
271 # | NVIDIA-SMI 361.89 Driver Version: 361.89 |
272 # |-------------------------------+----------------------+-------------+
273 # | GPU Name Persistence-M| Bus-Id Disp.A | GPU ECC |
274 # | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | Utiliz err |
275 # |===============================+======================+=============|
276 # | 0 Tesla P100-SXM2... On | 0002:01:00.0 Off | 0 |
277 # | N/A 25C P0 35W / 300W | 931MiB / 16280MiB | 0% Default |
278 # +-------------------------------+----------------------+-------------+
279 # | 1 Tesla P100-SXM2... On | 0003:01:00.0 Off | 0 |
280 # | N/A 26C P0 40W / 300W | 1477MiB / 16280MiB | 0% Default |
281 # +-------------------------------+----------------------+-------------+
282 # | 2 Tesla P100-SXM2... On | 0006:01:00.0 Off | 0 |
283 # | N/A 25C P0 35W / 300W | 931MiB / 16280MiB | 0% Default |
284 # +-------------------------------+----------------------+-------------+
285 # | 3 Tesla P100-SXM2... On | 0007:01:00.0 Off | 0 |
286 # | N/A 44C P0 290W / 300W | 965MiB / 16280MiB | 0% Default |
287 # +-------------------------------+----------------------+-------------+
288 # +--------------------------------------------------------------------+
289 # | Processes: GPU Memory |
290 # | GPU PID Type Process name Usage |
291 # |====================================================================|
292 # | 0 28459 C hxenvidia 929MiB |
293 # | 1 28460 C hxenvidia 1475MiB |
294 # | 2 28461 C hxenvidia 929MiB |
295 # | 3 28462 C hxenvidia 963MiB |
296 # +--------------------------------------------------------------------+
George Keishing4bbf5202017-05-18 06:55:53 -0500297
298 # Create logs directory and get current datetime.
299 Create Directory ${htx_log_dir_path}
300 ${cur_datetime}= Get Current Date result_format=%Y%m%d%H%M%S%f
301
Steven Sombar0278b132018-01-09 14:41:32 -0600302 ${nvidia_out} ${stderr} ${rc}= OS Execute Command nvidia-smi
George Keishing4bbf5202017-05-18 06:55:53 -0500303 Write Log Data To File
George Keishing664a0162017-06-05 12:24:24 -0500304 ... ${nvidia_out}
305 ... ${htx_log_dir_path}/${OS_HOST}_${cur_datetime}.nvidia_${suffix}
George Keishingbbfec9e2017-05-17 10:47:21 -0500306
307
Steven Sombar0278b132018-01-09 14:41:32 -0600308Get GPU Power Limit
309 [Documentation] Get NVIDIA GPU maximum permitted power draw.
George Keishingbbfec9e2017-05-17 10:47:21 -0500310
Steven Sombar0278b132018-01-09 14:41:32 -0600311 # nvidia-smi --query-gpu=power.limit --format=csv returns
312 # power.limit [W]
313 # 300.00 W
314 # 300.00 W
315 # 300.00 W
316 # 300.00 W
George Keishingbbfec9e2017-05-17 10:47:21 -0500317
Steven Sombar0278b132018-01-09 14:41:32 -0600318 ${cmd}= Catenate nvidia-smi --query-gpu=power.limit
319 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
320 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
321 # Allow for sensor overshoot. That is, max power reported for
322 # a GPU could be a few watts above the limit.
323 ${power_max}= Evaluate ${nvidia_out}+${7.00}
324 [Return] ${power_max}
325
326
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500327Get GPU Max Power
328 [Documentation] Get the maximum GPU power dissipation.
Steven Sombar0278b132018-01-09 14:41:32 -0600329
330 # nvidia-smi --query-gpu=power.draw --format=csv returns
331 # power.draw [W]
332 # 34.12 W
333 # 34.40 W
334 # 36.55 W
335 # 36.05 W
336
337 ${cmd}= Catenate nvidia-smi --query-gpu=power.draw
338 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
339 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
340 [Return] ${nvidia_out}
341
342
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500343Get GPU Min Power
344 [Documentation] Return the minimum GPU power value as record by
345 ... nvidia-smi.
346
347 ${cmd}= Catenate nvidia-smi --query-gpu=power.draw --format=csv |
348 ... grep -v 'power.draw' | cut -f 1 -d ' ' | sort -n -u | head -1
349 ${gpu_min_power} ${stderr} ${rc}= OS Execute Command ${cmd}
350 [Return] ${gpu_min_power}
351
352
Steven Sombar0278b132018-01-09 14:41:32 -0600353Get GPU Temperature Limit
354 [Documentation] Get NVIDIA GPU maximum permitted temperature.
355
356 # nvidia-smi -q -d TEMPERATURE | grep "GPU Max" returns
357 # GPU Max Operating Temp : 83 C
358 # GPU Max Operating Temp : 83 C
359 # GPU Max Operating Temp : 83 C
360 # GPU Max Operating Temp : 83 C
361
362 ${cmd}= Catenate nvidia-smi -q -d TEMPERATURE | grep "GPU Max"
363 ... | cut -f 2 -d ":" | tr -dc '0-9\n' | sort -n -u | tail -n 1
364 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
365 [Return] ${nvidia_out}
366
367
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500368Get GPU Min Temperature
369 [Documentation] Get the minimum GPU temperature.
370
371 ${cmd}= Catenate nvidia-smi --query-gpu=temperature.gpu
372 ... --format=csv | grep -v 'temperature.gpu' | sort -n -u | head -1
373 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
374 [Return] ${nvidia_out}
375
376
377Get GPU Max Temperature
378 [Documentation] Get the maximum GPU temperature.
Steven Sombar0278b132018-01-09 14:41:32 -0600379
380 # nvidia-smi --query-gpu=temperature.gpu --format=csv returns
381 # 38
382 # 41
383 # 38
384 # 40
385
386 ${cmd}= Catenate nvidia-smi --query-gpu=temperature.gpu
387 ... --format=csv | sort -n -u | tail -n 1
388 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
389 [Return] ${nvidia_out}
390
391
Steven Sombarc02dde82018-05-01 09:49:14 -0500392Get GPU Temperature Via REST
393 [Documentation] Return the temperature in degrees C of the warmest GPU
394 ... as reportd by REST.
395
396 # NOTE: This endpoint path is not defined until system has been powered-on.
397 ${temperature_objs}= Read Properties ${SENSORS_URI}temperature/enumerate
398 ... timeout=30 quiet=1
399
400 ${core_temperatures_list}= Catenate {k:v for k,v in $temperature_objs.iteritems()
401 ... if re.match('${SENSORS_URI}temperature/.*_core_temp', k)}
402 ${gpu_temperature_objs_dict}= Evaluate ${core_temperatures_list} modules=re
403
404 # Create a list containing all of the GPU temperatures.
405 ${gpu_temperatures}= Evaluate
406 ... [ x['Value'] for x in $gpu_temperature_objs_dict.values() ]
407
408 # Find the max temperature value and divide by 1000 to get just the integer
409 # portion.
410 ${max_gpu_temperature}= Evaluate max(map(int, $gpu_temperatures))/1000
411
412 [Return] ${max_gpu_temperature}
413
414
Steven Sombar0278b132018-01-09 14:41:32 -0600415Get GPU Clock Limit
416 [Documentation] Get NVIDIA GPU maximum permitted graphics clock.
417
418 # nvidia-smi --query-gpu=clocks.max.gr --format=csv returns
419 # 1530 MHz
420 # 1530 MHz
421 # 1530 MHz
422 # 1530 MHz
423
424 ${cmd}= Catenate nvidia-smi --query-gpu=clocks.max.gr
425 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
426 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
427 [Return] ${nvidia_out}
428
429
430Get GPU Clock
431 [Documentation] Get the highest assigned value of the GPU graphics clock.
432
433 # nvidia-smi --query-gpu=clocks.gr --format=csv returns
434 # 1230 MHz
435 # 1230 MHz
436 # 135 MHz
437 # 150 MHz
438
439 ${cmd}= Catenate nvidia-smi --query-gpu=clocks.gr
440 ... --format=csv | cut -f 1 -d ' ' | sort -n -u | tail -n 1
441 ${nvidia_out} ${stderr} ${rc}= OS Execute Command ${cmd}
442 [Return] ${nvidia_out}
443
444
445Count GPUs From BMC
446 [Documentation] Determine number of GPUs from the BMC. Hostboot
447 ... needs to have been run previously because the BMC gets GPU data
448 ... from Hostboot.
449
450 # Example of gv* endpoint data:
451 # "/xyz/openbmc_project/inventory/system/chassis/motherboard/gv100card0": {
452 # "Functional": 1,
453 # "Present": 1,
454 # "PrettyName": ""
455 # },
456
457 ${num_bmc_gpus}= Set Variable ${0}
458
459 ${gpu_list}= Get Endpoint Paths
460 ... ${HOST_INVENTORY_URI}system/chassis/motherboard gv*
461
462 :FOR ${gpu_uri} IN @{gpu_list}
463 \ ${present}= Read Attribute ${gpu_uri} Present
464 \ ${state}= Read Attribute ${gpu_uri} Functional
465 \ Rpvars gpu_uri present state
466 \ ${num_bmc_gpus}= Run Keyword If ${present} and ${state}
467 ... Evaluate ${num_bmc_gpus}+${1}
468 [Return] ${num_bmc_gpus}
George Keishingbbfec9e2017-05-17 10:47:21 -0500469
470
471Create Default MDT Profile
472 [Documentation] Create default mdt.bu profile and run.
473
474 Rprint Timen Create HTX mdt profile.
475
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500476 ${profile} ${stderr} ${rc}= OS Execute Command
477 ... htxcmdline -createmdt
George Keishingbbfec9e2017-05-17 10:47:21 -0500478 Rprintn ${profile}
479 Should Contain ${profile} mdts are created successfully
Steven Sombar3ecb6892018-03-23 11:41:08 -0500480 ... msg=Create MDT profile failed. response=${profile}
George Keishingbbfec9e2017-05-17 10:47:21 -0500481
482
483Run MDT Profile
484 [Documentation] Load user pre-defined MDT profile.
Daniel Gonzalezbfd8aff2018-03-27 10:20:37 -0600485 [Arguments] ${HTX_MDT_PROFILE}=${HTX_MDT_PROFILE}
486 # Description of argument(s):
487 # HTX_MDT_PROFILE MDT profile to be executed (e.g. "mdt.bu").
George Keishingbbfec9e2017-05-17 10:47:21 -0500488
489 Rprint Timen Start HTX mdt profile execution.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500490 ${htx_run} ${stderr} ${rc}= OS Execute Command
George Keishingbbfec9e2017-05-17 10:47:21 -0500491 ... htxcmdline -run -mdt ${HTX_MDT_PROFILE}
492 Rprintn ${htx_run}
493 Should Contain ${htx_run} Activated
Steven Sombar3ecb6892018-03-23 11:41:08 -0500494 ... msg=htxcmdline run mdt did not return "Activated" status.
George Keishingbbfec9e2017-05-17 10:47:21 -0500495
496
497Check HTX Run Status
498 [Documentation] Get HTX exerciser status and check for error.
Steven Sombar9d5fc2d2018-04-30 09:09:13 -0500499 [Arguments] ${sleep_time}=${0}
Joy Onyerikwua8074172018-04-20 07:08:19 -0500500
501 # Description of argument(s):
Steven Sombar56845272018-07-18 14:32:01 -0500502 # sleep_time The amount of time to sleep after checking status,
503 # for example "3s" or "2m".
George Keishingbbfec9e2017-05-17 10:47:21 -0500504
505 Rprint Timen Check HTX mdt Status and error.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500506 ${htx_status} ${stderr} ${rc}= OS Execute Command
George Keishingbbfec9e2017-05-17 10:47:21 -0500507 ... htxcmdline -status -mdt ${HTX_MDT_PROFILE}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500508 Rprintn ${htx_status}
George Keishingbbfec9e2017-05-17 10:47:21 -0500509
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500510 ${htx_errlog} ${stderr} ${rc}= OS Execute Command
511 ... htxcmdline -geterrlog
Steven Sombar3ecb6892018-03-23 11:41:08 -0500512 Rprintn ${htx_errlog}
George Keishingbbfec9e2017-05-17 10:47:21 -0500513
Steven Sombar3ecb6892018-03-23 11:41:08 -0500514 Should Contain ${htx_errlog} file </tmp/htxerr> is empty
515 ... msg=HTX geterrorlog was not empty.
George Keishingbbfec9e2017-05-17 10:47:21 -0500516
Steven Sombar56845272018-07-18 14:32:01 -0500517 Return From Keyword If "${sleep_time}" == "${0}"
Joy Onyerikwua8074172018-04-20 07:08:19 -0500518
Steven Sombar56845272018-07-18 14:32:01 -0500519 Run Key U Sleep \ ${sleep_time}
Joy Onyerikwua8074172018-04-20 07:08:19 -0500520
George Keishingbbfec9e2017-05-17 10:47:21 -0500521
522Shutdown HTX Exerciser
523 [Documentation] Shut down HTX exerciser run.
524
525 Rprint Timen Shutdown HTX Run
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500526 ${shutdown} ${stderr} ${rc}= OS Execute Command
George Keishingbbfec9e2017-05-17 10:47:21 -0500527 ... htxcmdline -shutdown -mdt ${HTX_MDT_PROFILE}
528 Rprintn ${shutdown}
Steven Sombare27cca92018-02-28 10:56:38 -0600529
530 ${match_count_no_mdt}= Count Values In List ${shutdown}
531 ... No MDT is currently running
532 ${match_count_success}= Count Values In List ${shutdown}
533 ... shutdown successfully
534 Run Keyword If ${match_count_no_mdt} == 0 and ${match_count_success} == 0
535 ... Fail msg=Shutdown command returned unexpected message.
536
George Keishingbbfec9e2017-05-17 10:47:21 -0500537
Steven Sombar130a04f2017-07-16 10:02:37 -0500538Create JSON Inventory File
539 [Documentation] Create a JSON inventory file, and make a YAML copy.
540 [Arguments] ${json_file_path}
541 # Description of argument:
542 # json_file_path Where the inventory file is wrtten to.
543
544 Login To OS
545 Compile Inventory JSON
546 Run json2yaml ${json_tmp_file_path} ${yaml_file_path}
547 # Format to JSON pretty print to file.
548 Run python -m json.tool ${json_tmp_file_path} > ${json_file_path}
549 OperatingSystem.File Should Exist ${json_file_path}
Steven Sombar3ecb6892018-03-23 11:41:08 -0500550 ... msg=File ${json_file_path} does not exist.
Steven Sombar130a04f2017-07-16 10:02:37 -0500551
552
553Compile Inventory JSON
554 [Documentation] Compile the Inventory into a JSON file.
555 Create File ${json_tmp_file_path}
556 Write New JSON List ${json_tmp_file_path} Inventory
557 Retrieve HW Info And Write processor ${json_tmp_file_path}
558 Retrieve HW Info And Write memory ${json_tmp_file_path}
559 Retrieve HW Info And Write List ${I/O} ${json_tmp_file_path} I/O last
560 Close New JSON List ${json_tmp_file_path}
561
Steven Sombar3ecb6892018-03-23 11:41:08 -0500562
Steven Sombar130a04f2017-07-16 10:02:37 -0500563Write New JSON List
564 [Documentation] Start a new JSON list element in file.
565 [Arguments] ${json_tmp_file_path} ${json_field_name}
566 # Description of argument(s):
567 # json_tmp_file_path Name of file to write to.
568 # json_field_name Name to give json list element.
569 Append to File ${json_tmp_file_path} { "${json_field_name}" : [
570
Steven Sombar3ecb6892018-03-23 11:41:08 -0500571
Steven Sombar130a04f2017-07-16 10:02:37 -0500572Close New JSON List
573 [Documentation] Close JSON list element in file.
574 [Arguments] ${json_tmp_file_path}
575 # Description of argument(s):
576 # json_tmp_file_path Path of file to write to.
577 Append to File ${json_tmp_file_path} ]}
578
Steven Sombar3ecb6892018-03-23 11:41:08 -0500579
Steven Sombar130a04f2017-07-16 10:02:37 -0500580Retrieve HW Info And Write
581 [Documentation] Retrieve and write info, add a comma if not last item.
582 [Arguments] ${class} ${json_tmp_file_path} ${last}=false
583 # Description of argument(s):
584 # class Device class to retrieve with lshw.
585 # json_tmp_file_path Path of file to write to.
586 # last Is this the last element in the parent JSON?
587 Write New JSON List ${json_tmp_file_path} ${class}
588 ${output} = Retrieve Hardware Info ${class}
589 ${output} = Clean Up String ${output}
590 Run Keyword if ${output.__class__ is not type(None)}
591 ... Append To File ${json_tmp_file_path} ${output}
592 Close New JSON List ${json_tmp_file_path}
593 Run Keyword if '${last}' == 'false'
594 ... Append to File ${json_tmp_file_path} ,
595
Steven Sombar3ecb6892018-03-23 11:41:08 -0500596
Steven Sombar130a04f2017-07-16 10:02:37 -0500597Retrieve HW Info And Write List
598 [Documentation] Does a Retrieve/Write with a list of classes and
599 ... encapsulates them into one large JSON element.
600 [Arguments] ${list} ${json_tmp_file_path} ${json_field_name}
601 ... ${last}=false
602 # Description of argument(s):
603 # list The list of devices classes to retrieve with lshw.
604 # json_tmp_file_path Path of file to write to.
605 # json_field_name Name of the JSON element to encapsulate this list.
606 # last Is this the last element in the parent JSON?
607 Write New JSON List ${json_tmp_file_path} ${json_field_name}
608 : FOR ${class} IN @{list}
609 \ ${tail} Get From List ${list} -1
610 \ Run Keyword if '${tail}' == '${class}'
611 \ ... Retrieve HW Info And Write ${class} ${json_tmp_file_path} true
612 \ ... ELSE Retrieve HW Info And Write ${class} ${json_tmp_file_path}
613 Close New JSON List ${json_tmp_file_path}
614 Run Keyword if '${last}' == 'false'
615 ... Append to File ${json_tmp_file_path} ,
616
Steven Sombar3ecb6892018-03-23 11:41:08 -0500617
Steven Sombar130a04f2017-07-16 10:02:37 -0500618Retrieve Hardware Info
619 [Documentation] Retrieves the lshw output of the device class as JSON.
620 [Arguments] ${class}
621 # Description of argument(s):
622 # class Device class to retrieve with lshw.
Joy Onyerikwuf73a62a2018-05-02 10:50:20 -0500623 ${output} ${stderr} ${rc}= OS Execute Command lshw -c ${class} -json
Steven Sombar130a04f2017-07-16 10:02:37 -0500624 ${output} = Verify JSON string ${output}
625 [Return] ${output}
626
Steven Sombar3ecb6892018-03-23 11:41:08 -0500627
Steven Sombar130a04f2017-07-16 10:02:37 -0500628Verify JSON String
Gunnar Mills28e403b2017-10-25 16:16:38 -0500629 [Documentation] Ensure the JSON string content is separated by commas.
Steven Sombar130a04f2017-07-16 10:02:37 -0500630 [Arguments] ${unver_string}
631 # Description of argument(s):
632 # unver_string JSON String we will be checking for lshw comma errors.
633 ${unver_string} = Convert to String ${unver_string}
634 ${ver_string} = Replace String Using Regexp ${unver_string} }\\s*{ },{
635 [Return] ${ver_string}
636
Steven Sombar3ecb6892018-03-23 11:41:08 -0500637
Steven Sombar130a04f2017-07-16 10:02:37 -0500638Clean Up String
639 [Documentation] Remove extra whitespace and trailing commas.
640 [Arguments] ${dirty_string}
641 # Description of argument(s):
642 # dirty_string String that will be space stripped and have comma removed.
643 ${clean_string} = Strip String ${dirty_string}
644 ${last_char} = Get Substring ${clean_string} -1
645 ${trimmed_string} = Get Substring ${clean_string} 0 -1
646 ${clean_string} = Set Variable If '${last_char}' == ','
647 ... ${trimmed_string} ${clean_string}
648 [Return] ${clean_string}
Joy Onyerikwu4cb8c0a2018-04-11 18:05:13 -0500649
650
651Get OS Network Interface Names
652 [Documentation] Return a list of interface names on the OS.
653
654 ${stdout} ${stderr} ${rc}= OS Execute Command ls /sys/class/net
655 @{interface_names}= Split String ${stdout}
656 [Return] @{interface_names}
Daniel Gonzalezbfd8aff2018-03-27 10:20:37 -0600657
658
659Run Soft Bootme
660 [Documentation] Run a soft bootme for a period of an hour.
661 [Arguments] ${bootme_period}=3
662 # Description of argument(s):
663 # bootme_time Bootme period to be rebooting the system.
664
665 ${output} ${stderr} ${rc}= OS Execute Command
666 ... htxcmdline -bootme on mode:soft period:${bootme_period}
667 Should Contain ${output} bootme on is completed successfully
668
669
670Shutdown Bootme
671 [Documentation] Stop the bootme process.
672
673 ${output} ${stderr} ${rc}= OS Execute Command htxcmdline -bootme off
Steven Sombar56845272018-07-18 14:32:01 -0500674 Should Contain ${output} bootme off is completed successfully