blob: 74c8d99e57934823ab0a7b20318e643161358057 [file] [log] [blame]
Chris Austenb29d2e82016-06-07 12:25:35 -05001*** Settings ***
2Resource ../lib/resource.txt
3Resource ../lib/rest_client.robot
George Keishing5e870cd2016-08-24 10:05:47 -05004Resource ../lib/connection_client.robot
George Keishing30c12ff2016-09-02 10:25:29 -05005Library DateTime
6Library Process
Chris Austenb29d2e82016-06-07 12:25:35 -05007Library OperatingSystem
Michael Walsha6723f22016-11-22 11:12:01 -06008Library gen_print.py
9Library gen_robot_print.py
Chris Austenb29d2e82016-06-07 12:25:35 -050010
11*** Variables ***
12${SYSTEM_SHUTDOWN_TIME} ${5}
Michael Walsha6723f22016-11-22 11:12:01 -060013${dbuscmdBase}
George Keishing85ca05e2016-11-30 09:47:18 -060014... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host
Michael Walsha6723f22016-11-22 11:12:01 -060015${dbuscmdGet}
George Keishing9485af92017-01-04 10:21:43 -060016... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get
George Keishing85ca05e2016-11-30 09:47:18 -060017# Enable when ready with openbmc/openbmc-test-automation#203
18#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string:
Michael Walsha6723f22016-11-22 11:12:01 -060019${dbuscmdString}= string:"org.openbmc.settings.Host" string:
20
21# Assign default value to QUIET for programs which may not define it.
22${QUIET} ${0}
Sivas SRRea85d1f2016-11-13 22:44:28 -060023${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4
24${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2
25${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9
Sridevi Ramesh1699d372016-12-06 00:20:22 -060026${HOST_SETTING} ${SETTINGS_URI}host0
Sivas SRR352bc162017-02-13 22:12:29 -060027# /run/initramfs/ro associate filesystem should be 100% full always
Sivas SRRaca55712016-12-21 04:32:35 -060028${bmc_file_system_usage_cmd}=
Sivas SRR352bc162017-02-13 22:12:29 -060029... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l
Chris Austenb29d2e82016-06-07 12:25:35 -050030
George Keishingcce9df22017-01-24 06:19:33 -060031${BOOT_TIME} ${0}
32${BOOT_COUNT} ${0}
Sivas SRR8254db62017-02-07 09:39:46 -060033${devicetree_base} /sys/firmware/devicetree/base/model
George Keishingcce9df22017-01-24 06:19:33 -060034
Chris Austenb29d2e82016-06-07 12:25:35 -050035*** Keywords ***
Sridevi Ramesh1699d372016-12-06 00:20:22 -060036
Sivas SRR8254db62017-02-07 09:39:46 -060037Get BMC System Model
38 [Documentation] Get the BMC model from the device tree.
39
George Keishing90ab1772017-02-24 05:40:55 -060040 ${bmc_model} ${stderr}= Execute Command
41 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True
Sivas SRR8254db62017-02-07 09:39:46 -060042 Should Be Empty ${stderr}
43 Should Not Be Empty ${bmc_model}
44 [Return] ${bmc_model}
45
46Verify BMC System Model
47 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}.
48 [Arguments] ${bmc_model}
49
50 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} /
51 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} .
52 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model}
53 ... ${tmp_bmc_model} ignore_case=True
54 [Return] ${ret}
55
Chris Austenb29d2e82016-06-07 12:25:35 -050056Wait For Host To Ping
root442f0ef2016-08-04 20:23:05 +000057 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min
58 ... ${interval}=5 sec
59
60 # host The DNS name or IP of the host to ping.
61 # timeout The amount of time after which attempts to ping cease.
62 # interval The amount of time in between attempts to ping.
63
64 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host}
Chris Austenb29d2e82016-06-07 12:25:35 -050065
66Ping Host
67 [Arguments] ${host}
George Keishing8a84f952016-08-25 04:54:53 -050068 Should Not Be Empty ${host} msg=No host provided
Michael Walsha6723f22016-11-22 11:12:01 -060069 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host}
Chris Austenb29d2e82016-06-07 12:25:35 -050070 Log RC: ${RC}\nOutput:\n${output}
71 Should be equal ${RC} ${0}
72
73Get Boot Progress
Michael Walsha6723f22016-11-22 11:12:01 -060074 [Arguments] ${quiet}=${QUIET}
75
George Keishing85ca05e2016-11-30 09:47:18 -060076 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress
Michael Walsha6723f22016-11-22 11:12:01 -060077 ... value quiet=${quiet}
Gunnar Millsc9ea9362016-12-13 16:21:13 -060078 [Return] ${state}
Chris Austenb29d2e82016-06-07 12:25:35 -050079
80Is Power On
root442f0ef2016-08-04 20:23:05 +000081 ${state}= Get Power State
82 Should be equal ${state} ${1}
Chris Austenb29d2e82016-06-07 12:25:35 -050083
84Is Power Off
root442f0ef2016-08-04 20:23:05 +000085 ${state}= Get Power State
86 Should be equal ${state} ${0}
Chris Austenb29d2e82016-06-07 12:25:35 -050087
root442f0ef2016-08-04 20:23:05 +000088Initiate Power On
89 [Documentation] Initiates the power on and waits until the Is Power On
90 ... keyword returns that the power state has switched to on.
Michael Walsha6723f22016-11-22 11:12:01 -060091 [Arguments] ${wait}=${1}
92
Chris Austenb29d2e82016-06-07 12:25:35 -050093 @{arglist}= Create List
94 ${args}= Create Dictionary data=@{arglist}
George Keishing85ca05e2016-11-30 09:47:18 -060095 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn
Michael Walsha6723f22016-11-22 11:12:01 -060096 ... data=${args}
Chris Austenb29d2e82016-06-07 12:25:35 -050097 should be equal as strings ${resp.status_code} ${HTTP_OK}
Chris Austenb29d2e82016-06-07 12:25:35 -050098
Michael Walsha6723f22016-11-22 11:12:01 -060099 # Does caller want to wait for power on status?
100 Run Keyword If '${wait}' == '${0}' Return From Keyword
101 Wait Until Keyword Succeeds 3 min 10 sec Is Power On
Rahul Maheshwarif684ba72016-10-25 07:24:41 -0500102
root442f0ef2016-08-04 20:23:05 +0000103Initiate Power Off
104 [Documentation] Initiates the power off and waits until the Is Power Off
105 ... keyword returns that the power state has switched to off.
Chris Austenb29d2e82016-06-07 12:25:35 -0500106 @{arglist}= Create List
107 ${args}= Create Dictionary data=@{arglist}
George Keishing85ca05e2016-11-30 09:47:18 -0600108 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff
Michael Walsha6723f22016-11-22 11:12:01 -0600109 ... data=${args}
Chris Austenb29d2e82016-06-07 12:25:35 -0500110 should be equal as strings ${resp.status_code} ${HTTP_OK}
Michael Walsha6723f22016-11-22 11:12:01 -0600111 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off
Chris Austenb29d2e82016-06-07 12:25:35 -0500112
Michael Walsh81843772017-03-02 14:44:44 -0600113Initiate OS Host Power Off
114 [Documentation] Initiate an OS reboot.
115 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
116 ... ${os_password}=${OS_PASSWORD}
117
118 # Description of arguments:
119 # os_host The DNS name or IP of the OS.
120 # os_username The username to be used to sign in to the OS.
121 # os_password The password to be used to sign in to the OS.
122
Michael Walshac275512017-03-07 11:39:28 -0600123 SSHLibrary.Open connection ${os_host}
Michael Walsh81843772017-03-02 14:44:44 -0600124 Login ${os_username} ${os_password}
125 ${cmd_buf} Catenate shutdown
126 Start Command ${cmd_buf}
Michael Walshac275512017-03-07 11:39:28 -0600127 SSHLibrary.Close Connection
Michael Walsh81843772017-03-02 14:44:44 -0600128
129Initiate OS Host Reboot
130 [Documentation] Initiate an OS reboot.
131 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
132 ... ${os_password}=${OS_PASSWORD}
133
134 # Description of arguments:
135 # os_host The DNS name or IP of the OS.
136 # os_username The username to be used to sign in to the OS.
137 # os_password The password to be used to sign in to the OS.
138
Michael Walshac275512017-03-07 11:39:28 -0600139 SSHLibrary.Open connection ${os_host}
Michael Walsh81843772017-03-02 14:44:44 -0600140 Login ${os_username} ${os_password}
141 ${cmd_buf} Catenate reboot
142 Start Command ${cmd_buf}
Michael Walshac275512017-03-07 11:39:28 -0600143 SSHLibrary.Close Connection
Michael Walsh81843772017-03-02 14:44:44 -0600144
145Initiate Auto Reboot
146 [Documentation] Initiate an auto reboot.
147
148 # Set the auto reboot policy.
149 Set Auto Reboot yes
150
Michael Walshac275512017-03-07 11:39:28 -0600151 SSHLibrary.Open connection ${openbmc_host}
Michael Walsh81843772017-03-02 14:44:44 -0600152 Login ${openbmc_username} ${openbmc_password}
153
154 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds.
155 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0
156 ... org.openbmc.Watchdog set i 5000
157 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf}
158 ... return_stderr=True return_rc=True
159 Should Be Empty ${stderr}
160 Should be equal ${rc} ${0}
161
162 # Start the watchdog.
163 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0
164 ... org.openbmc.Watchdog start
165 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf}
166 ... return_stderr=True return_rc=True
167 Should Be Empty ${stderr}
168 Should be equal ${rc} ${0}
169
Chris Austenb29d2e82016-06-07 12:25:35 -0500170Trigger Warm Reset
171 log to console "Triggering warm reset"
Michael Walsha6723f22016-11-22 11:12:01 -0600172 ${data}= create dictionary data=@{EMPTY}
George Keishing85ca05e2016-11-30 09:47:18 -0600173 ${resp}= openbmc post request
174 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data}
Chris Austenb29d2e82016-06-07 12:25:35 -0500175 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK}
George Keishingb3700812016-08-31 03:03:30 -0500176 ${session_active}= Check If warmReset is Initiated
177 Run Keyword If '${session_active}' == '${True}'
178 ... Fail msg=warm reset didn't occur
179
Chris Austenb29d2e82016-06-07 12:25:35 -0500180 Sleep ${SYSTEM_SHUTDOWN_TIME}min
Rahul Maheshwari5f253c42017-01-30 05:19:51 -0600181 Check If BMC Is Up
Michael Walsh49ab0f42016-07-20 11:44:33 -0500182
183Check OS
root442f0ef2016-08-04 20:23:05 +0000184 [Documentation] Attempts to ping the host OS and then checks that the host
185 ... OS is up by running an SSH command.
Michael Walsh49ab0f42016-07-20 11:44:33 -0500186
187 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
Michael Walsha6723f22016-11-22 11:12:01 -0600188 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET}
189 ... ${print_string}=${EMPTY}
Michael Walshac275512017-03-07 11:39:28 -0600190 [Teardown] SSHLibrary.Close Connection
Michael Walsh49ab0f42016-07-20 11:44:33 -0500191
192 # os_host The DNS name/IP of the OS host associated with our BMC.
193 # os_username The username to be used to sign on to the OS host.
194 # os_password The password to be used to sign on to the OS host.
Michael Walsha6723f22016-11-22 11:12:01 -0600195 # quiet Indicates whether this keyword should write to console.
196 # print_string A string to be printed before checking the OS.
197
198 rprint ${print_string}
Michael Walsh49ab0f42016-07-20 11:44:33 -0500199
root442f0ef2016-08-04 20:23:05 +0000200 # Attempt to ping the OS. Store the return code to check later.
201 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host}
202
Michael Walshac275512017-03-07 11:39:28 -0600203 SSHLibrary.Open connection ${os_host}
Michael Walsh49ab0f42016-07-20 11:44:33 -0500204
Michael Walsha6723f22016-11-22 11:12:01 -0600205 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username}
206 ... ${os_password}
207 ${err_msg1}= Sprint Error ${msg}
208 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1}
209 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg}
Michael Walsh49ab0f42016-07-20 11:44:33 -0500210 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True
211 ... return_rc=True
212
Michael Walsha6723f22016-11-22 11:12:01 -0600213 ${temp_msg}= Catenate Could not execute a command on the operating
214 ... system.\n
215 ${err_msg1}= Sprint Error ${temp_msg}
216 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1}
217
218 # If the return code returned by "Execute Command" is non-zero, this
219 # keyword will fail.
220 Should Be Equal ${rc} ${0} msg=${err_msg}
Michael Walsh49ab0f42016-07-20 11:44:33 -0500221 # We will likewise fail if there is any stderr data.
222 Should Be Empty ${stderr}
223
Michael Walsha6723f22016-11-22 11:12:01 -0600224 ${temp_msg}= Set Variable Could not ping the operating system.\n
225 ${err_msg1}= Sprint Error ${temp_msg}
226 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1}
227 # We will likewise fail if the OS did not ping, as we could SSH but not
228 # ping
229 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg}
root442f0ef2016-08-04 20:23:05 +0000230
Michael Walsh49ab0f42016-07-20 11:44:33 -0500231Wait for OS
232 [Documentation] Waits for the host OS to come up via calls to "Check OS".
233 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
234 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT}
Michael Walsha6723f22016-11-22 11:12:01 -0600235 ... ${quiet}=${0}
236 [Teardown] rprintn
Michael Walsh49ab0f42016-07-20 11:44:33 -0500237
238 # os_host The DNS name or IP of the OS host associated with our
239 # BMC.
240 # os_username The username to be used to sign on to the OS host.
241 # os_password The password to be used to sign on to the OS host.
242 # timeout The timeout in seconds indicating how long you're
243 # willing to wait for the OS to respond.
Michael Walsha6723f22016-11-22 11:12:01 -0600244 # quiet Indicates whether this keyword should write to console.
Michael Walsh49ab0f42016-07-20 11:44:33 -0500245
246 # The interval to be used between calls to "Check OS".
247 ${interval}= Set Variable 5
248
Michael Walsha6723f22016-11-22 11:12:01 -0600249 ${message}= Catenate Checking every ${interval} seconds for up to
250 ... ${timeout} seconds for the operating system to communicate.
251 rqprint_timen ${message}
252
Michael Walsh49ab0f42016-07-20 11:44:33 -0500253 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS
254 ... ${os_host} ${os_username} ${os_password}
Michael Walsha6723f22016-11-22 11:12:01 -0600255 ... print_string=\#
256
257 rqprintn
258
259 rqprint_timen The operating system is now communicating.
root442f0ef2016-08-04 20:23:05 +0000260
Rahul Maheshwari2c725042017-01-29 22:55:28 -0600261Get BMC State Deprecated
root442f0ef2016-08-04 20:23:05 +0000262 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY)
Michael Walsha6723f22016-11-22 11:12:01 -0600263 [Arguments] ${quiet}=${QUIET}
264
root442f0ef2016-08-04 20:23:05 +0000265 @{arglist}= Create List
266 ${args}= Create Dictionary data=@{arglist}
George Keishing85ca05e2016-11-30 09:47:18 -0600267 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState
Michael Walsha6723f22016-11-22 11:12:01 -0600268 ... data=${args} quiet=${quiet}
root442f0ef2016-08-04 20:23:05 +0000269 Should be equal as strings ${resp.status_code} ${HTTP_OK}
270 ${content}= to json ${resp.content}
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600271 [Return] ${content["data"]}
root442f0ef2016-08-04 20:23:05 +0000272
273Get Power State
274 [Documentation] Returns the power state as an integer. Either 0 or 1.
Michael Walsha6723f22016-11-22 11:12:01 -0600275 [Arguments] ${quiet}=${QUIET}
276
root442f0ef2016-08-04 20:23:05 +0000277 @{arglist}= Create List
278 ${args}= Create Dictionary data=@{arglist}
Michael Walsha6723f22016-11-22 11:12:01 -0600279
George Keishing85ca05e2016-11-30 09:47:18 -0600280 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState
Michael Walsha6723f22016-11-22 11:12:01 -0600281 ... data=${args} quiet=${quiet}
root442f0ef2016-08-04 20:23:05 +0000282 Should be equal as strings ${resp.status_code} ${HTTP_OK}
283 ${content}= to json ${resp.content}
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600284 [Return] ${content["data"]}
Jay Azurine4c52eb2016-08-16 20:51:10 -0500285
286Clear BMC Record Log
287 [Documentation] Clears all the event logs on the BMC. This would be
288 ... equivalent to ipmitool sel clear.
289 @{arglist}= Create List
290 ${args}= Create Dictionary data=@{arglist}
George Keishing85ca05e2016-11-30 09:47:18 -0600291 ${resp}= Call Method
292 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args}
Jay Azurine4c52eb2016-08-16 20:51:10 -0500293 should be equal as strings ${resp.status_code} ${HTTP_OK}
294
295Copy PNOR to BMC
296 Import Library SCPLibrary WITH NAME scp
297 Open Connection for SCP
298 Log Copying ${PNOR_IMAGE_PATH} to /tmp
299 scp.Put File ${PNOR_IMAGE_PATH} /tmp
300
301Flash PNOR
302 [Documentation] Calls flash bios update method to flash PNOR image
Gunnar Mills38032802016-12-12 13:43:40 -0600303 [Arguments] ${pnor_image}
Jay Azurine4c52eb2016-08-16 20:51:10 -0500304 @{arglist}= Create List ${pnor_image}
305 ${args}= Create Dictionary data=@{arglist}
George Keishing85ca05e2016-11-30 09:47:18 -0600306 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update
Michael Walsha6723f22016-11-22 11:12:01 -0600307 ... data=${args}
Jay Azurine4c52eb2016-08-16 20:51:10 -0500308 should be equal as strings ${resp.status_code} ${HTTP_OK}
309 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing
310
311Get Flash BIOS Status
312 [Documentation] Returns the status of the flash BIOS API as a string. For
313 ... example 'Flashing', 'Flash Done', etc
George Keishing85ca05e2016-11-30 09:47:18 -0600314 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600315 [Return] ${data['status']}
Jay Azurine4c52eb2016-08-16 20:51:10 -0500316
317Is PNOR Flashing
318 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR
319 ... flashing has started.
320 ${status}= Get Flash BIOS Status
321 should be equal as strings ${status} Flashing
322
323Is PNOR Flash Done
324 [Documentation] Get BIOS 'Flash Done' status. This indicates that the
325 ... PNOR flashing has completed.
Leah McNutt9ba32272016-11-17 15:48:39 +0000326 ${status}= Get Flash BIOS Status
Jay Azurine4c52eb2016-08-16 20:51:10 -0500327 should be equal as strings ${status} Flash Done
328
329Is System State Host Booted
330 [Documentation] Checks whether system state is HOST_BOOTED.
Rahul Maheshwari2c725042017-01-29 22:55:28 -0600331 ${state}= Get BMC State Deprecated
Jay Azurine4c52eb2016-08-16 20:51:10 -0500332 should be equal as strings ${state} HOST_BOOTED
George Keishing5e870cd2016-08-24 10:05:47 -0500333
Rahul Maheshwari2c725042017-01-29 22:55:28 -0600334Is OS Starting
335 [Documentation] Check if boot progress is OS starting.
336 ${boot_progress}= Get Boot Progress
337 Should Be Equal ${boot_progress} FW Progress, Starting OS
338
George Keishing0af24412017-03-10 13:33:23 -0600339Is OS Off
340 [Documentation] Check if boot progress is "Off".
341 ${boot_progress}= Get Boot Progress
342 Should Be Equal ${boot_progress} Off
343
George Keishing5e870cd2016-08-24 10:05:47 -0500344Verify Ping and REST Authentication
Michael Walsha6723f22016-11-22 11:12:01 -0600345 ${l_ping}= Run Keyword And Return Status
George Keishing5e870cd2016-08-24 10:05:47 -0500346 ... Ping Host ${OPENBMC_HOST}
George Keishingc4d3dc02016-09-19 03:45:55 -0500347 Run Keyword If '${l_ping}' == '${False}'
348 ... Fail msg=Ping Failed
George Keishing5e870cd2016-08-24 10:05:47 -0500349
Michael Walsha6723f22016-11-22 11:12:01 -0600350 ${l_rest}= Run Keyword And Return Status
George Keishing5e870cd2016-08-24 10:05:47 -0500351 ... Initialize OpenBMC
George Keishingc4d3dc02016-09-19 03:45:55 -0500352 Run Keyword If '${l_rest}' == '${False}'
353 ... Fail msg=REST Authentication Failed
George Keishing5e870cd2016-08-24 10:05:47 -0500354
355 # Just to make sure the SSH is working for SCP
356 Open Connection And Log In
357 ${system} ${stderr}= Execute Command hostname return_stderr=True
358 Should Be Empty ${stderr}
359
George Keishingc4d3dc02016-09-19 03:45:55 -0500360Check If BMC is Up
361 [Documentation] Wait for Host to be online. Checks every X seconds
362 ... interval for Y minutes and fails if timed out.
363 ... Default MAX timedout is 10 min, interval 10 seconds.
Gunnar Mills38032802016-12-12 13:43:40 -0600364 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min
George Keishingc4d3dc02016-09-19 03:45:55 -0500365 ... ${interval}=10 sec
366
367 Wait Until Keyword Succeeds
368 ... ${max_timeout} ${interval} Verify Ping and REST Authentication
369
George Keishingb3700812016-08-31 03:03:30 -0500370
371Check If warmReset is Initiated
372 [Documentation] Ping would be still alive, so try SSH to connect
373 ... if fails the ports are down indicating reboot
374 ... is in progress
George Keishing3c05d352016-12-13 06:54:50 -0600375
376 # Warm reset adds 3 seconds delay before forcing reboot
377 # To minimize race conditions, we wait for 7 seconds
378 Sleep 7s
George Keishingb3700812016-08-31 03:03:30 -0500379 ${alive}= Run Keyword and Return Status
380 ... Open Connection And Log In
381 Return From Keyword If '${alive}' == '${False}' ${False}
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600382 [Return] ${True}
George Keishingb3700812016-08-31 03:03:30 -0500383
George Keishing06ae4aa2016-08-30 01:41:28 -0500384Flush REST Sessions
385 [Documentation] Removes all the active session objects
386 Delete All Sessions
George Keishingb3700812016-08-31 03:03:30 -0500387
Sivas SRRe1143ae2016-08-26 22:31:02 -0500388Initialize DBUS cmd
389 [Documentation] Initialize dbus string with property string to extract
Gunnar Mills38032802016-12-12 13:43:40 -0600390 [Arguments] ${boot_property}
Michael Walsha6723f22016-11-22 11:12:01 -0600391 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString}
392 ${cmd}= Catenate ${cmd}${boot_property}
Sivas SRRe1143ae2016-08-26 22:31:02 -0500393 Set Global Variable ${dbuscmd} ${cmd}
394
David Shawba2d2c22017-01-23 16:56:38 -0600395Create OS Console File Path
396 [Documentation] Create OS console file path name and return it.
397 [Arguments] ${log_file_path}=${EMPTY}
George Keishing30c12ff2016-09-02 10:25:29 -0500398
David Shawba2d2c22017-01-23 16:56:38 -0600399 # Description of arguements:
400 # file_path The caller's candidate value. If this value is ${EMPTY}, this
401 # keyword will compose a file path name. Otherwise, this
402 # keyword will use the caller's file_path value. In either
403 # case, the value will be returned.
Gunnar Mills9c760342016-12-08 10:12:06 -0600404
David Shawba2d2c22017-01-23 16:56:38 -0600405 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console
406 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}'
407 ... ${default_file_path} ${log_file_path}
Michael Walsh01bd3682017-01-10 11:21:05 -0600408
David Shawba2d2c22017-01-23 16:56:38 -0600409 [Return] ${log_file_path}
Gunnar Mills9c760342016-12-08 10:12:06 -0600410
David Shawba2d2c22017-01-23 16:56:38 -0600411Create OS Console Command String
412 [Documentation] Return a command string to start OS console logging.
Gunnar Mills9c760342016-12-08 10:12:06 -0600413
David Shawba2d2c22017-01-23 16:56:38 -0600414 # First make sure that the ssh_pw program is available.
415 ${cmd_buf}= Catenate which ssh_pw 2>&1
416 Rdpissuing ${cmd_buf}
417 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf}
418 Rdpvars rc output
419 Should Be Equal ${rc} ${0} msg=${output}\n
Gunnar Mills9c760342016-12-08 10:12:06 -0600420
David Shawba2d2c22017-01-23 16:56:38 -0600421 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200
422 ... ${OPENBMC_USERNAME}@${OPENBMC_HOST}
George Keishing30c12ff2016-09-02 10:25:29 -0500423
David Shawba2d2c22017-01-23 16:56:38 -0600424 [Return] ${cmd_buf}
George Keishing30c12ff2016-09-02 10:25:29 -0500425
426Stop SOL Console Logging
David Shawba2d2c22017-01-23 16:56:38 -0600427 [Documentation] Stop system console logging and return log output.
428 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY}
429
430 # If there are muliple system console processes, they will all be stopped.
431 # If there is no existing log file this keyword will return an error
432 # message to that effect (and write that message to targ_file_path, if
433 # specified).
434 # NOTE: This keyword will not fail if there is no running system console
435 # process.
Michael Walsh01bd3682017-01-10 11:21:05 -0600436
437 # Description of arguments:
David Shawba2d2c22017-01-23 16:56:38 -0600438 # log_file_path The file path that was used to call "Start SOL
439 # Console Logging". See that keyword (above) for details.
440 # targ_file_path If specified, the file path to which the source
441 # file path (i.e. "log_file_path") should be copied.
George Keishing30c12ff2016-09-02 10:25:29 -0500442
David Shawba2d2c22017-01-23 16:56:38 -0600443 ${log_file_path}= Create OS Console File Path ${log_file_path}
444 # Find the pid of the active system console logging session (if any).
445 ${search_string}= Create OS Console Command String
446 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}'
447 ... | egrep -v grep | cut -c10-14)
448 Rdpissuing ${cmd_buf}
449 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf}
450 Rdpvars os_con_pid
451 # If rc is not zero it just means that there is no OS Console process
452 # running.
George Keishing30c12ff2016-09-02 10:25:29 -0500453
David Shawba2d2c22017-01-23 16:56:38 -0600454 ${cmd_buf}= Catenate kill -9 ${os_con_pid}
455 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf}
456 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}'
457 ... Run And Return Rc And Output ${cmd_buf}
458 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output
George Keishing30c12ff2016-09-02 10:25:29 -0500459
David Shawba2d2c22017-01-23 16:56:38 -0600460 ${cmd_buf}= Set Variable cat ${log_file_path} 2>&1
461 Rdpissuing ${cmd_buf}
462 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf}
463 Rdpvars rc
George Keishing30c12ff2016-09-02 10:25:29 -0500464
Michael Walsh01bd3682017-01-10 11:21:05 -0600465 Run Keyword If '${targ_file_path}' != '${EMPTY}'
David Shawba2d2c22017-01-23 16:56:38 -0600466 ... Run Keyword And Ignore Error
467 ... Copy File ${log_file_path} ${targ_file_path}
George Keishing30c12ff2016-09-02 10:25:29 -0500468
David Shawba2d2c22017-01-23 16:56:38 -0600469 [Return] ${output}
470
471Start SOL Console Logging
472 [Documentation] Start system console log to file.
473 [Arguments] ${log_file_path}=${EMPTY}
474
475 # This keyword will first call "Stop SOL Console Logging". Only then will
476 # it start SOL console logging. The data returned by "Stop SOL Console
477 # Logging" will in turn be returned by this keyword.
478
479 # Description of arguments:
480 # log_file_path The file path to which system console log data should be
481 # written. Note that this path is taken to be a location on
482 # the machine where this program is running rather than on
483 # the Open BMC system.
484
485 ${log_file_path}= Create OS Console File Path ${log_file_path}
486
487 ${log_output}= Stop SOL Console Logging ${log_file_path}
488
489 # Validate by making sure we can create the file. Problems creating the
490 # file would not be noticed by the subsequent ssh command because we fork
491 # the command.
492 Create File ${log_file_path}
493 ${sub_cmd_buf}= Create OS Console Command String
494 # Routing stderr to stdout so that any startup error text will go to the
495 # output file.
496 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 &
497 Rdpissuing ${cmd_buf}
498 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf}
499 # Because we are forking this command, we essentially will never get a
500 # non-zero return code or any output.
501 Should Be Equal ${rc} ${0}
502
503 [Return] ${log_output}
George Keishing30c12ff2016-09-02 10:25:29 -0500504
505Get Time Stamp
506 [Documentation] Get the current time stamp data
507 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f
Sridevi Rameshb96a0c32016-10-19 05:09:52 -0500508 [Return] ${cur_time}
George Keishing30c12ff2016-09-02 10:25:29 -0500509
George Keishing1b150202016-09-29 08:51:58 -0500510
511Verify BMC State
512 [Documentation] Get the BMC state and verify if the current
513 ... BMC state is as expected.
514 [Arguments] ${expected}
515
Rahul Maheshwari2c725042017-01-29 22:55:28 -0600516 ${current}= Get BMC State Deprecated
George Keishing8db0e1b2016-10-20 13:46:54 -0500517 Should Contain ${expected} ${current}
George Keishing1b150202016-09-29 08:51:58 -0500518
Rahul Maheshwarif684ba72016-10-25 07:24:41 -0500519Start Journal Log
520 [Documentation] Start capturing journal log to a file in /tmp using
521 ... journalctl command. By default journal log is collected
522 ... at /tmp/journal_log else user input location.
523 ... The File is appended with datetime.
524 [Arguments] ${file_path}=/tmp/journal_log
525
526 Open Connection And Log In
527
528 ${cur_time}= Get Time Stamp
529 Set Global Variable ${LOG_TIME} ${cur_time}
530 Start Command
531 ... journalctl -f > ${file_path}-${LOG_TIME}
532 Log Journal Log Started: ${file_path}-${LOG_TIME}
533
David Shawba2d2c22017-01-23 16:56:38 -0600534
Rahul Maheshwarif684ba72016-10-25 07:24:41 -0500535Stop Journal Log
536 [Documentation] Stop journalctl process if its running.
537 ... By default return log from /tmp/journal_log else
538 ... user input location.
539 [Arguments] ${file_path}=/tmp/journal_log
540
541 Open Connection And Log In
542
543 ${rc}=
544 ... Execute Command
545 ... ps ax | grep journalctl | grep -v grep
546 ... return_stdout=False return_rc=True
547
548 Return From Keyword If '${rc}' == '${1}'
549 ... No journal log process running
550
551 ${output} ${stderr}=
552 ... Execute Command killall journalctl
553 ... return_stderr=True
554 Should Be Empty ${stderr}
555
556 ${journal_log} ${stderr}=
557 ... Execute Command
558 ... cat ${file_path}-${LOG_TIME}
559 ... return_stderr=True
560 Should Be Empty ${stderr}
561
562 Log ${journal_log}
563
564 Execute Command rm ${file_path}-${LOG_TIME}
565
566 [Return] ${journal_log}
Rahul Maheshwari757d80c2016-10-17 01:09:55 -0500567
568Mac Address To Hex String
569 [Documentation] Converts MAC address into hex format.
570 ... Example
571 ... Given the following MAC: 00:01:6C:80:02:78
572 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78
573 ... Description of arguments:
Michael Walsh01bd3682017-01-10 11:21:05 -0600574 ... i_macaddress MAC address in the following format
575 ... 00:01:6C:80:02:78
Rahul Maheshwari757d80c2016-10-17 01:09:55 -0500576 [Arguments] ${i_macaddress}
577
578 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')}
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600579 [Return] ${mac_hex}
Rahul Maheshwari757d80c2016-10-17 01:09:55 -0500580
581IP Address To Hex String
582 [Documentation] Converts IP address into hex format.
583 ... Example:
584 ... Given the following IP: 10.3.164.100
585 ... This keyword will return: 0xa 0x3 0xa4 0xa0
586 ... Description of arguments:
Michael Walsh01bd3682017-01-10 11:21:05 -0600587 ... i_ipaddress IP address in the following format
588 ... 10.10.10.10
Rahul Maheshwari757d80c2016-10-17 01:09:55 -0500589 [Arguments] ${i_ipaddress}
590
591 @{ip}= Split String ${i_ipaddress} .
592 ${index}= Set Variable ${0}
593
594 :FOR ${item} IN @{ip}
595 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes
596 \ Set List Value ${ip} ${index} ${hex}
597 \ ${index}= Set Variable ${index + 1}
598 ${ip_hex}= Catenate @{ip}
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600599 [Return] ${ip_hex}
Sivas SRRea85d1f2016-11-13 22:44:28 -0600600
601BMC CPU Performance Check
602 [Documentation] Minimal 10% of proc should be free in this instance
603
604 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd}
605 ... return_stderr=True
606 Should be empty ${stderr}
607 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} %
608 Should be true ${bmc_cpu_percentage} < 90
609
610BMC Mem Performance Check
611 [Documentation] Minimal 10% of memory should be free in this instance
612
613 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd}
614 ... return_stderr=True
615 Should be empty ${stderr}
616
617 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd}
618 ... return_stderr=True
619 Should be empty ${stderr}
620
621 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100
622 ${bmc_mem_percentage}= Evaluate
623 ... ${bmc_mem_percentage}/${bmc_mem_total_output}
624 Should be true ${bmc_mem_percentage} > 10
625
Sivas SRRaca55712016-12-21 04:32:35 -0600626BMC File System Usage Check
627 [Documentation] Check the file system space. None should be 100% full
Sivas SRR352bc162017-02-13 22:12:29 -0600628 ... except /run/initramfs/ro
Sivas SRRaca55712016-12-21 04:32:35 -0600629 ${bmc_fs_usage_output} ${stderr}= Execute Command
630 ... ${bmc_file_system_usage_cmd} return_stderr=True
631 Should Be Empty ${stderr}
632 Should Be True ${bmc_fs_usage_output}==0
633
Sivas SRRea85d1f2016-11-13 22:44:28 -0600634Check BMC CPU Performance
635 [Documentation] Minimal 10% of proc should be free in 3 sample
636 :FOR ${var} IN Range 1 4
637 \ BMC CPU Performance check
638
639Check BMC Mem Performance
640 [Documentation] Minimal 10% of memory should be free
641
642 :FOR ${var} IN Range 1 4
643 \ BMC Mem Performance check
644
Sivas SRRaca55712016-12-21 04:32:35 -0600645Check BMC File System Performance
646 [Documentation] Check for file system usage for 4 times
647
648 :FOR ${var} IN Range 1 4
649 \ BMC File System Usage check
650
Rahul Maheshwari9a8d3b12016-12-05 04:06:16 -0600651Get Endpoint Paths
652 [Documentation] Returns all url paths ending with given endpoint
653 ... Example:
654 ... Given the following endpoint: cpu
Michael Walsh01bd3682017-01-10 11:21:05 -0600655 ... This keyword will return: list of all urls ending with
656 ... cpu -
Rahul Maheshwari9a8d3b12016-12-05 04:06:16 -0600657 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0,
658 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1
659 ... Description of arguments:
660 ... path URL path for enumeration
661 ... endpoint string for which url path ending
662 [Arguments] ${path} ${endpoint}
663
664 ${resp}= Read Properties ${path}/enumerate timeout=30
665 log Dictionary ${resp}
666
667 ${list}= Get Dictionary Keys ${resp}
668 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$
Gunnar Millsc9ea9362016-12-13 16:21:13 -0600669 [Return] ${resp}
Sridevi Rameshb96a0c32016-10-19 05:09:52 -0500670
671Check Zombie Process
672 [Documentation] Check if any defunct process exist or not on BMC
Michael Walsh01bd3682017-01-10 11:21:05 -0600673 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l
Sridevi Rameshb96a0c32016-10-19 05:09:52 -0500674 ... return_stderr=True return_rc=True
675 Should Be True ${count}==0
676 Should Be Empty ${stderr}
George Keishing5327d012016-12-08 08:43:29 -0600677
678Prune Journal Log
679 [Documentation] Prune archived journal logs.
680 [Arguments] ${vacuum_size}=1M
681
682 # This keyword can be used to prevent the journal
683 # log from filling up the /run filesystem.
684 # This command will retain only the latest logs
685 # of the user specified size.
686
687 Open Connection And Log In
688 ${output} ${stderr} ${rc}=
689 ... Execute Command
690 ... journalctl --vacuum-size=${vacuum_size}
691 ... return_stderr=True return_rc=True
692
693 Should Be Equal ${rc} ${0} msg=${stderr}
Sridevi Ramesh1699d372016-12-06 00:20:22 -0600694
695Set BMC Power Policy
696 [Documentation] Set the given BMC power policy.
697 [arguments] ${policy}
698
699 ${valueDict}= create dictionary data=${policy}
700 Write Attribute ${HOST_SETTING} power_policy data=${valueDict}
701 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy
702 Should Be Equal ${currentPolicy} ${policy}
George Keishingcce9df22017-01-24 06:19:33 -0600703
George Keishing4d6f3d62017-02-06 09:50:41 -0600704Get System Power Policy
705 [Documentation] Get the BMC power policy.
706 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy
707 [Return] ${currentPolicy}
George Keishingcce9df22017-01-24 06:19:33 -0600708
Rahul Maheshwari1cbf0042017-02-13 05:17:37 -0600709Get Auto Reboot
710 [Documentation] Returns auto reboot setting.
Michael Walsh81843772017-03-02 14:44:44 -0600711 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot
Rahul Maheshwari1cbf0042017-02-13 05:17:37 -0600712 [Return] ${setting}
713
714
715Set Auto Reboot
716 [Documentation] Set the given auto reboot setting.
717 [Arguments] ${setting}
718 # setting auto reboot's setting, i.e. yes or no
719
720 ${valueDict}= Set Variable ${setting}
721 ${data}= Create Dictionary data=${valueDict}
Michael Walsh81843772017-03-02 14:44:44 -0600722 Write Attribute ${HOST_SETTING} auto_reboot data=${data}
Rahul Maheshwari1cbf0042017-02-13 05:17:37 -0600723 ${current_setting}= Get Auto Reboot
724 Should Be Equal ${current_setting} ${setting}
725
726
George Keishingcce9df22017-01-24 06:19:33 -0600727Set BMC Reset Reference Time
728 [Documentation] Set current boot time as a reference and increment
George Keishing80dd5af2017-02-08 07:01:25 -0600729 ... boot count.
George Keishingcce9df22017-01-24 06:19:33 -0600730
731 ${cur_btime}= Get BMC Boot Time
George Keishing80dd5af2017-02-08 07:01:25 -0600732 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0}
George Keishingcce9df22017-01-24 06:19:33 -0600733 ... Set Global Variable ${BOOT_TIME} ${cur_btime}
George Keishing80dd5af2017-02-08 07:01:25 -0600734 ... ELSE IF ${cur_btime} > ${BOOT_TIME}
735 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime}
George Keishingcce9df22017-01-24 06:19:33 -0600736 ... AND
737 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1}
738
George Keishingcce9df22017-01-24 06:19:33 -0600739Get BMC Boot Time
740 [Documentation] Get boot time from /proc/stat.
741
742 Open Connection And Log In
743 ${output} ${stderr}=
744 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' '
745 ... return_stderr=True
746 Should Be Empty ${stderr}
747 ${btime}= Convert To Integer ${output}
748 [Return] ${btime}
George Keishingede30002017-02-02 09:33:07 -0600749
George Keishingede30002017-02-02 09:33:07 -0600750Execute Command On BMC
751 [Documentation] Execute given command on BMC and return output.
752 [Arguments] ${command}
753 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True
754 Should Be Empty ${stderr}
755 [Return] ${stdout}
David Shawba2d2c22017-01-23 16:56:38 -0600756
George Keishing7a520222017-02-27 09:44:30 -0600757Enable Core Dump On BMC
758 [Documentation] Enable core dump collection.
759 Open Connection And Log In
760 ${core_pattern}= Execute Command On BMC
761 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern
762 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p
Rahul Maheshwarie95622c2017-02-24 10:04:29 -0600763
764Trigger Host Watchdog Error
765 [Documentation] Inject host watchdog error using BMC.
766 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s
767 # Description of arguments:
768 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 1 second).
769 # sleep_time Time delay for host watchdog error to get injected.
770 # Default is 5 seconds.
771
772 Execute Command On BMC
773 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds}
774 Execute Command On BMC
775 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start
776 Sleep ${sleep_time}
Prashanth Kattiae7c2282017-03-15 07:43:46 -0500777
778Login To OS Host
779 [Documentation] Login to OS Host.
780 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME}
781 ... ${os_password}=${OS_PASSWORD}
782 # Desription of arguments:
783 # ${os_host} IP address of the OS Host.
784 # ${os_username} OS Host Login user name.
785 # ${os_password} OS Host Login passwrd.
786
787 ${os_state}= Run Keyword And Return Status Ping Host ${os_host}
788 Run Keyword If '${os_state}' == 'False'
789 ... Run Keywords Initiate Host Reboot AND
790 ... Is Host Running AND
791 ... Wait for OS ${os_host} ${os_username} ${os_password}
792
793 Open Connection ${os_host}
794 Login ${os_username} ${os_password}