blob: ab6d84f2868c4e8851cf56cf2784e92703911680 [file] [log] [blame]
George Keishing664a0162017-06-05 12:24:24 -05001*** Settings ***
Steven Sombar0278b132018-01-09 14:41:32 -06002Documentation Stress the system GPUs using the HTX exerciser.
3
4# Test Parameters:
5# OPENBMC_HOST The BMC host name or IP address.
6# OS_HOST The OS host name or IP Address.
7# OS_USERNAME The OS login userid (usually "root").
8# OS_PASSWORD The password for the OS login.
9# HTX_DURATION Duration of HTX run, for example, "2h", or "30m".
10# HTX_LOOP The number of times to loop HTX.
11# HTX_INTERVAL The time delay between consecutive checks of HTX
12# status, for example, "15m".
13# In summary: Run HTX for $HTX_DURATION, looping
14# $HTX_LOOP times checking for errors every
15# $HTX_INTERVAL. Then allow extra time for OS
16# Boot, HTX startup, shutdown.
17# HTX_KEEP_RUNNING If set to 1, this indicates that the HTX is to
18# continue running after an error was found.
19
George Keishing664a0162017-06-05 12:24:24 -050020
George Keishinga28061a2023-12-15 14:46:54 +053021Resource ../lib/os_utilities.robot
George Keishing664a0162017-06-05 12:24:24 -050022
Steven Sombar197e3802017-11-02 08:07:29 -050023Suite Setup Run Keyword Start SOL Console Logging
George Keishingdc1691d2017-12-07 12:17:46 -060024Test Setup Test Setup Execution
Steven Sombar197e3802017-11-02 08:07:29 -050025Test Teardown Test Teardown Execution
George Keishing664a0162017-06-05 12:24:24 -050026
George Keishing87dc4422023-10-20 12:56:30 +053027Force Tags GPU_Stress
28
George Keishing664a0162017-06-05 12:24:24 -050029*** Variables ****
30
Steven Sombar0278b132018-01-09 14:41:32 -060031${HTX_DURATION} 1h
32${HTX_LOOP} ${1}
33${HTX_INTERVAL} 30m
34${HTX_KEEP_RUNNING} ${0}
George Keishing664a0162017-06-05 12:24:24 -050035${stack_mode} skip
36
37*** Test Cases ***
38
39GPU Stress Test
40 [Documentation] Stress the GPU using HTX exerciser.
41 [Tags] GPU_Stress_Test
42
Steven Sombar0278b132018-01-09 14:41:32 -060043 # Get number of GPU reported by the BMC.
44 ${num_bmc_gpus}= Count GPUs From BMC
45 Rpvars num_bmc_gpus
46
47 # The BMC and OS should report the same number of GPUs.
48 ${failmsg01}= Catenate OS reports ${num_os_gpus} GPUs, but BMC
49 ... reports ${num_bmc_gpus} present and functional GPUs.
50 Run Keyword If '${num_os_gpus}' != '${num_bmc_gpus}'
51 ... Fail msg=${failmsg01}
52
53 # Show parameters for HTX stress test.
Michael Walshc108e422019-03-28 12:27:18 -050054 Printn
Steven Sombar0278b132018-01-09 14:41:32 -060055 Rpvars HTX_DURATION HTX_LOOP HTX_INTERVAL
56
57 # Set the iteration (loop) counter.
58 Set Suite Variable ${iteration} ${0} children=true
59
60
61 # Shutdown HTX if it is already running.
Steven Sombar3ecb6892018-03-23 11:41:08 -050062 ${status}= Is HTX Running
Steven Sombar0278b132018-01-09 14:41:32 -060063 Run Keyword If '${status}' == 'True'
64 ... Shutdown HTX Exerciser
George Keishing664a0162017-06-05 12:24:24 -050065
66 Repeat Keyword ${HTX_LOOP} times Execute GPU Test
67
68
69*** Keywords ***
70
71Execute GPU Test
72 [Documentation] Start HTX exerciser.
73 # Test Flow:
74 # - Power on
75 # - Establish SSH connection session
76 # - Collect GPU nvidia status output
77 # - Create HTX mdt profile
78 # - Run GPU specific HTX exerciser
Steven Sombar0278b132018-01-09 14:41:32 -060079 # - Check for errors
80
81 Set Suite Variable ${iteration} ${iteration + 1}
82 ${loop_count}= Catenate Starting iteration: ${iteration}
Michael Walshc108e422019-03-28 12:27:18 -050083 Printn
Steven Sombar0278b132018-01-09 14:41:32 -060084 Rpvars loop_count
85
86 REST Power On stack_mode=skip
Steven Sombar1ddc7c62018-06-01 11:55:34 -050087 Run Key U Sleep \ 15s
George Keishing664a0162017-06-05 12:24:24 -050088
89 # Collect data before the test starts.
90 Collect NVIDIA Log File start
91
Steven Sombar0278b132018-01-09 14:41:32 -060092 # Collect NVIDIA maximum limits.
93 ${power_max}= Get GPU Power Limit
94 ${temperature_max}= Get GPU Temperature Limit
95 ${clock_max}= Get GPU Clock Limit
96
George Keishing664a0162017-06-05 12:24:24 -050097 Run Keyword If '${HTX_MDT_PROFILE}' == 'mdt.bu'
98 ... Create Default MDT Profile
99
100 Run MDT Profile
101
102 Loop HTX Health Check
103
104 # Post test loop look out for dmesg error logged.
105 Check For Errors On OS Dmesg Log
106
Steven Sombar0278b132018-01-09 14:41:32 -0600107 # Check NVIDIA power, temperature, and clocks.
Joy Onyerikwu26975f02018-05-08 12:55:12 -0500108 ${power}= Get GPU Max Power
109 ${temperature}= Get GPU Max Temperature
Steven Sombarc02dde82018-05-01 09:49:14 -0500110 ${temperature_via_rest}= Get GPU Temperature Via REST
Steven Sombar0278b132018-01-09 14:41:32 -0600111 ${clock}= Get GPU Clock
Michael Walshc108e422019-03-28 12:27:18 -0500112 Printn
Steven Sombarc02dde82018-05-01 09:49:14 -0500113 Rpvars power power_max temperature temperature_via_rest
114 ... temperature_max clock clock_max
115
Steven Sombar0278b132018-01-09 14:41:32 -0600116 Run Keyword If ${power} > ${power_max} Fail
117 ... msg=GPU Power ${power} exceeds limit of ${power_max}.
Steven Sombarc02dde82018-05-01 09:49:14 -0500118
119 ${err_msg}= Catenate GPU temperature of ${temperature} exceeds limit
Steven Sombar0278b132018-01-09 14:41:32 -0600120 ... of ${temperature_max}.
Steven Sombarc02dde82018-05-01 09:49:14 -0500121 Run Keyword If ${temperature} > ${temperature_max} Fail msg=${err_msg}
122
Steven Sombar0278b132018-01-09 14:41:32 -0600123 Run Keyword If ${clock} > ${clock_max} Fail
124 ... msg=GPU clock of ${clock} exceeds limit of ${clock_max}.
125
Steven Sombarc02dde82018-05-01 09:49:14 -0500126 ${err_msg}= Catenate The GPU temperature reported by REST is not within
127 ... 5 degrees of the nvidia_smi reported temperature.
128 ${upper_limit}= Evaluate ${temperature_via_rest}+5
129 ${lower_limit}= Evaluate ${temperature_via_rest}-5
130 Run Keyword If
131 ... ${temperature} > ${upper_limit} or ${temperature} < ${lower_limit}
132 ... Fail msg=${err_msg}
133
George Keishing664a0162017-06-05 12:24:24 -0500134 Shutdown HTX Exerciser
135
Steven Sombar0278b132018-01-09 14:41:32 -0600136 Collect NVIDIA Log File end
137 Error Logs Should Not Exist
138 REST Power Off
139
140 Flush REST Sessions
141
Michael Walshc108e422019-03-28 12:27:18 -0500142 Print Timen HTX Test ran for: ${HTX_DURATION}
George Keishing664a0162017-06-05 12:24:24 -0500143
Steven Sombar0278b132018-01-09 14:41:32 -0600144 ${loop_count}= Catenate Ending iteration: ${iteration}
Michael Walshc108e422019-03-28 12:27:18 -0500145 Printn
Steven Sombar0278b132018-01-09 14:41:32 -0600146 Rpvars loop_count
147
George Keishing664a0162017-06-05 12:24:24 -0500148
149Loop HTX Health Check
150 [Documentation] Run until HTX exerciser fails.
151
152 Repeat Keyword ${HTX_DURATION}
153 ... Run Keywords Check HTX Run Status
154 ... AND Sleep ${HTX_INTERVAL}
155
156
Steven Sombar0278b132018-01-09 14:41:32 -0600157Test Setup Execution
158 [Documentation] Do the initial test setup.
159
160 REST Power On stack_mode=skip
Steven Sombar1ddc7c62018-06-01 11:55:34 -0500161 Run Key U Sleep \ 15s
Steven Sombar0278b132018-01-09 14:41:32 -0600162 Delete All Error Logs
163 Tool Exist lspci
164 Tool Exist htxcmdline
165 Tool Exist nvidia-smi
166
167 # Get number of GPUs reported by the OS.
168 ${cmd}= Catenate lspci | grep NVIDIA | wc -l
169 ${num_os_gpus} ${stderr} ${rc}= OS Execute Command ${cmd}
Michael Walshc108e422019-03-28 12:27:18 -0500170 Printn
Steven Sombar0278b132018-01-09 14:41:32 -0600171 Rpvars num_os_gpus
172
173 # If no GPUs detected, we cannot continue.
174 Run Keyword If '${num_os_gpus}' == '${0}' Fail
175 ... msg=No GPUs detected so cannot run test.
176
177 Set Suite Variable ${num_os_gpus} children=true
178
179
180
Steven Sombar197e3802017-11-02 08:07:29 -0500181Test Teardown Execution
George Keishing664a0162017-06-05 12:24:24 -0500182 [Documentation] Do the post test teardown.
George Keishing664a0162017-06-05 12:24:24 -0500183
184 # Keep HTX running if user set HTX_KEEP_RUNNING to 1.
185 Run Keyword If '${TEST_STATUS}' == 'FAIL' and ${HTX_KEEP_RUNNING} == ${0}
186 ... Shutdown HTX Exerciser
187
Steven Sombar197e3802017-11-02 08:07:29 -0500188 ${keyword_buf}= Catenate Stop SOL Console Logging
189 ... \ targ_file_path=${EXECDIR}${/}logs${/}SOL.log
190 Run Key ${keyword_buf}
191
George Keishing664a0162017-06-05 12:24:24 -0500192 FFDC On Test Case Fail
193 Close All Connections