lib/openbmc_ffdc_methods.robot - openbmc/openbmc-test-automation - Gitiles

 *** Settings ***
 Documentation      Methods to execute commands on BMC and collect
 ...                data to a list of FFDC files

 Resource           openbmc_ffdc_utils.robot
 Resource           rest_client.robot
 Resource           utils.robot
 Library            SSHLibrary
 Library            OperatingSystem
 Library            Collections

 *** Keywords ***

 ################################################################
 # Method : Call FFDC Methods                                   #
 #          Execute the user define keywords from the FFDC List #
 #          Unlike any other keywords this will call into the   #
 #          list of keywords defined in the FFDC list at one go #
 ################################################################

 Call FFDC Methods
     [Documentation]   Calls into FFDC Keyword index list

     @{entries}=     Get ffdc method index
     :FOR  ${index}  IN   @{entries}
     \     Method Call Keyword List   ${index}
     SSHLibrary.Close All Connections


 Method Call Keyword List
     [Documentation]   Iterate the list through keyword index
     [Arguments]       ${index}

     @{method_list}=      Get ffdc method call   ${index}
     :FOR  ${method}  IN  @{method_list}
     \    Execute Keyword Method   ${method[1]}


 Execute Keyword Method
     [Documentation]   Calls into BMC method keywords. Don't let one
     ...               failure skips the remaining. Get whatever data
     ...               it could gather at worse case scenario.
     [Arguments]   ${keyword_name}

     Run Keyword And Continue On Failure   ${keyword_name}


 ################################################################
 # Method : BMC FFDC Manifest                                   #
 #          Execute command on BMC and write to ffdc_report.txt #
 ################################################################

 BMC FFDC Manifest
     [Documentation]    Get the commands index for the FFDC_BMC_CMD,
     ...                login to BMC and execute commands.
     Open Connection And Log In

     @{entries}=     Get ffdc cmd index
     :FOR  ${index}  IN   @{entries}
     \     Iterate BMC Command List Pairs   ${index}


 Iterate BMC Command List Pairs
     [Documentation]    Feed in key pair list from dictionary to execute
     [Arguments]        ${key_index}

     @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
     Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
     :FOR  ${cmd}  IN  @{cmd_list}
     \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}


 Execute Command and Write FFDC
     [Documentation]    Execute command on BMC or OS and write to ffdc
     ...                By default to ffdc_report.txt file else to
     ...                specified file path.
     [Arguments]        ${key_index}
     ...                ${cmd}
     ...                ${logpath}=${FFDC_FILE_PATH}

     Run Keyword If   '${logpath}' == '${FFDC_FILE_PATH}'
     ...    Write Cmd Output to FFDC File   ${key_index}  ${cmd}

     ${stdout}  ${stderr}=
     ...   Execute Command    ${cmd}   return_stderr=True

     # Write stdout on success and stderr/stdout to the file on failure.
     Run Keyword If  $stderr == '${EMPTY}'
     ...    Write Data To File  ${stdout}${\n}  ${logpath}
     ...  ELSE  Write Data To File
     ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
     ...    ${logpath}


 ################################################################
 # Method : BMC FFDC Files                                      #
 #          Execute command on BMC and write to individual file #
 #          based on the file name pre-defined in the list      #
 ################################################################

 BMC FFDC Files
     [Documentation]    Get the command list and iterate
     Open Connection And Log In
     @{entries}=     Get ffdc file index
     :FOR  ${index}  IN   @{entries}
     \     Create File and Write Data   ${index}


 Create File and Write Data
     [Documentation]    Create files to current FFDC log directory,
     ...                executes command and write to corresponding
     ...                file name in the current FFDC directory.
     [Arguments]        ${key_index}

     # To build IP address in searchable form eg: dummy\.domain\.com
     ${OPENBMC_HOST_REGEX}=  Run  echo ${OPENBMC_HOST} | sed 's/\(\.\)/\\\1/g'
     @{cmd_list}=      Get ffdc bmc file   ${key_index}
     :FOR  ${cmd}  IN  @{cmd_list}
     \   ${logpath}=  Catenate  SEPARATOR=   ${LOG_PREFIX}   ${cmd[0]}.txt
     \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
     # Rename OPENBMC_HOST IP address from given file to DUMMYIP
     \   Run  sed -i 's/'${OPENBMC_HOST_REGEX}'/DUMMYIP/g' ${logpath}


 ################################################################
 # Method : Log Test Case Status                                #
 #          Creates test result history footprint for reference #
 ################################################################

 Log Test Case Status
     [Documentation]  Test case execution result history.
     ...  Create once and append to this file
     ...  logs/test_history.txt
     ...  Format   Date:Test suite:Test case:Status
     ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
     Create Directory   ${FFDC_LOG_PATH}

     ${exist}=   Run Keyword and Return Status
     ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}

     Run Keyword If  '${exist}' == '${False}'
     ...   Create File  ${TEST_HISTORY}

     ${cur_time}=      Get Current Time Stamp

     Append To File    ${TEST_HISTORY}
     ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}


 Log FFDC Get Requests
     [Documentation]    Create file in current FFDC log directory.
     ...                Do openbmc get request and write to
     ...                corresponding file name.
     ...                JSON pretty print for logging to file.
     [Arguments]        ${key_index}

     @{cmd_list}=  Get ffdc get request  ${key_index}
     :FOR  ${cmd}  IN  @{cmd_list}
     \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
     \   ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
     \   ${status}=    Run Keyword and Return Status
     ...   Should Be Equal As Strings    ${resp.status_code}    ${HTTP_OK}
     \   Run Keyword If   '${status}' == '${False}'  Continue For Loop
     \   ${jsondata}=  to json  ${resp.content}    pretty_print=True
     \   Write Data To File  ${\n}${jsondata}${\n}  ${logpath}


 BMC FFDC Get Requests
     [Documentation]    Get the command list and iterate
     Open Connection And Log In
     @{entries}=  Get ffdc get request index
     :FOR  ${index}  IN  @{entries}
     \   Log FFDC Get Requests   ${index}


 Log OS ALL DISTROS FFDC
     [Documentation]    Create file in current FFDC log directory.
     ...                Executes OS command and write to
     ...                corresponding file name.
     [Arguments]        ${key_index}

     @{cmd_list}=  get ffdc os all distros call  ${key_index}
     :FOR  ${cmd}  IN  @{cmd_list}
     \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
     \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}


 Log OS SPECIFIC DISTRO FFDC
     [Documentation]    Create file in current FFDC log directory.
     ...                Executes OS command and write to
     ...                corresponding file name.
     [Arguments]        ${key_index}  ${linux_distro}

     @{cmd_list}=  get ffdc os distro call  ${key_index}  ${linux_distro}
     :FOR  ${cmd}  IN  @{cmd_list}
     \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
     \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}


 OS FFDC Files
     [Documentation]    Get the command list and iterate
     [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
     ...   ${OS_PASSWORD}=${OS_PASSWORD}

     Return From Keyword If  '${OS_HOST}' == '${EMPTY}'
     ...   No OS Host Provided

     # If can't ping, return
     ${rc}=  Run Keyword and Return Status  Ping Host  ${OS_HOST}
     Return From Keyword If  '${rc}' == '${False}'
     ...   Could not ping OS

     Open Connection And Log In  host=${OS_HOST}  username=${OS_USERNAME}
     ...   password=${OS_PASSWORD}

     ${output}  ${stderr}  ${rc}=  Execute Command  uptime  return_stderr=True
     ...   return_rc=True

     # If the return code returned by "Execute Command" is non-zero, return
     Return From Keyword If  '${rc}' != '${0}'
     ...   Could not connect to OS

     @{entries}=  Get ffdc os all distros index
     :FOR  ${index}  IN  @{entries}
     \   Log OS ALL DISTROS FFDC  ${index}

     ${linux_distro}=  Execute Command
     ...   . /etc/os-release; echo $ID
     ...   return_stdout=True  return_stderr=False  return_rc=False

     Return From Keyword If
     ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
     ...  Could not determine Linux Distribution

     @{entries}=  Get ffdc os distro index  ${linux_distro}
     :FOR  ${index}  IN  @{entries}
     \   Log OS SPECIFIC DISTRO FFDC  ${index}  ${linux_distro}


 ##############################################################################
 SCP Coredump Files
     [Documentation]  Copy core dump file from BMC to local system.
     # Check if core dump exist in the /tmp
     Open Connection And Log In
     ${core_files}=  Execute Command  ls /tmp/core_*
     @{core_list} =  Split String    ${core_files}
     # Copy the core files
     Open Connection for SCP
     :FOR  ${index}  IN  @{core_list}
     \  scp.Get File  ${index}  ${LOG_PREFIX}${index.lstrip("/tmp/")}
     # Remove the file from remote to avoid re-copying on next FFDC call
     \  Execute Command On BMC  rm ${index}


 ##############################################################################
 Collect eSEL Log
     [Documentation]  Collect eSEL log from logging entry and convert eSEL data
     ...              to elog formated string text file.
     ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}/enumerate  quiet=${1}
     ${status}=  Run Keyword And Return Status
     ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
     Return From Keyword If  '${status}' == '${False}'

     ${content}=  To Json  ${resp.content}
     # Grab the list of entries from logging/entry/
     # The data shown below is the result of the "Get Dictionary Keys".
     # Example:
     # /xyz/openbmc_project/logging/entry/1
     # /xyz/openbmc_project/logging/entry/2
     ${esel_list}=  Get Dictionary Keys  ${content['data']}

     ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  esel
     Create File  ${logpath}
     # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
     #  "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
     # Sample eSEL entry:
     #  "/xyz/openbmc_project/logging/entry/1": {
     #    "Timestamp": 1487744317025,
     #    "AdditionalData": [
     #        "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
     #    ],
     #    "Message": "org.open_power.Error.Host.Event.Event",
     #    "Id": 1,
     #    "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
     # }

     :FOR  ${entry_path}  IN  @{esel_list}
     \  ${esel_data}=  Read Attribute  ${entry_path}  AdditionalData  quiet=${1}
     \  ${length}=  Get Length  ${esel_data}
     # Skip writting to file if eSEL AdditionalData is empty
     \  Continue For Loop If  ${length} == ${0}
     \  Write Data To File  "${esel_data[0]}"  ${logpath}
     \  Write Data To File  ${\n}  ${logpath}

     ${out}=  Run  which eSEL.pl
     ${status}=  Run Keyword And Return Status
     ...  Should Contain  ${out}  eSEL.pl
     Return From Keyword If  '${status}' == '${False}'

     Convert eSEL To Elog Format  ${logpath}


 ##############################################################################
 Convert eSEL To Elog Format
     [Documentation]  Execute parser tool on the eSEL data file to generate
     ...              formated error log.
     [Arguments]  ${esel_file_path}
     # Desription of arguments:
     # ${esel_file_path}  Absoulte path of the eSEL data.

     Run  eSEL.pl -l ${esel_file_path}

 ##############################################################################
	* Settings *
	Documentation Methods to execute commands on BMC and collect
	... data to a list of FFDC files

	Resource openbmc_ffdc_utils.robot
	Resource rest_client.robot
	Resource utils.robot
	Library SSHLibrary
	Library OperatingSystem
	Library Collections

	* Keywords *

	################################################################
	# Method : Call FFDC Methods #
	# Execute the user define keywords from the FFDC List #
	# Unlike any other keywords this will call into the #
	# list of keywords defined in the FFDC list at one go #
	################################################################

	Call FFDC Methods
	[Documentation] Calls into FFDC Keyword index list

	@{entries}= Get ffdc method index
	:FOR ${index} IN @{entries}
	\ Method Call Keyword List ${index}
	SSHLibrary.Close All Connections


	Method Call Keyword List
	[Documentation] Iterate the list through keyword index
	[Arguments] ${index}

	@{method_list}= Get ffdc method call ${index}
	:FOR ${method} IN @{method_list}
	\ Execute Keyword Method ${method[1]}


	Execute Keyword Method
	[Documentation] Calls into BMC method keywords. Don't let one
	... failure skips the remaining. Get whatever data
	... it could gather at worse case scenario.
	[Arguments] ${keyword_name}

	Run Keyword And Continue On Failure ${keyword_name}


	################################################################
	# Method : BMC FFDC Manifest #
	# Execute command on BMC and write to ffdc_report.txt #
	################################################################

	BMC FFDC Manifest
	[Documentation] Get the commands index for the FFDC_BMC_CMD,
	... login to BMC and execute commands.
	Open Connection And Log In

	@{entries}= Get ffdc cmd index
	:FOR ${index} IN @{entries}
	\ Iterate BMC Command List Pairs ${index}


	Iterate BMC Command List Pairs
	[Documentation] Feed in key pair list from dictionary to execute
	[Arguments] ${key_index}

	@{cmd_list}= Get ffdc bmc cmd ${key_index}
	Set Suite Variable ${ENTRY_INDEX} ${key_index}
	:FOR ${cmd} IN @{cmd_list}
	\ Execute Command and Write FFDC ${cmd[0]} ${cmd[1]}


	Execute Command and Write FFDC
	[Documentation] Execute command on BMC or OS and write to ffdc
	... By default to ffdc_report.txt file else to
	... specified file path.
	[Arguments] ${key_index}
	... ${cmd}
	... ${logpath}=${FFDC_FILE_PATH}

	Run Keyword If '${logpath}' == '${FFDC_FILE_PATH}'
	... Write Cmd Output to FFDC File ${key_index} ${cmd}

	${stdout} ${stderr}=
	... Execute Command ${cmd} return_stderr=True

	# Write stdout on success and stderr/stdout to the file on failure.
	Run Keyword If $stderr == '${EMPTY}'
	... Write Data To File ${stdout}${\n} ${logpath}
	... ELSE Write Data To File
	... ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
	... ${logpath}


	################################################################
	# Method : BMC FFDC Files #
	# Execute command on BMC and write to individual file #
	# based on the file name pre-defined in the list #
	################################################################

	BMC FFDC Files
	[Documentation] Get the command list and iterate
	Open Connection And Log In
	@{entries}= Get ffdc file index
	:FOR ${index} IN @{entries}
	\ Create File and Write Data ${index}


	Create File and Write Data
	[Documentation] Create files to current FFDC log directory,
	... executes command and write to corresponding
	... file name in the current FFDC directory.
	[Arguments] ${key_index}

	# To build IP address in searchable form eg: dummy\.domain\.com
	${OPENBMC_HOST_REGEX}= Run echo ${OPENBMC_HOST} \| sed 's/\(\.\)/\\\1/g'
	@{cmd_list}= Get ffdc bmc file ${key_index}
	:FOR ${cmd} IN @{cmd_list}
	\ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt
	\ Execute Command and Write FFDC ${cmd[0]} ${cmd[1]} ${logpath}
	# Rename OPENBMC_HOST IP address from given file to DUMMYIP
	\ Run sed -i 's/'${OPENBMC_HOST_REGEX}'/DUMMYIP/g' ${logpath}



	################################################################
	# Method : Log Test Case Status #
	# Creates test result history footprint for reference #
	################################################################

	Log Test Case Status
	[Documentation] Test case execution result history.
	... Create once and append to this file
	... logs/test_history.txt
	... Format Date:Test suite:Test case:Status
	... 20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
	Create Directory ${FFDC_LOG_PATH}

	${exist}= Run Keyword and Return Status
	... OperatingSystem.File Should Exist ${TEST_HISTORY}

	Run Keyword If '${exist}' == '${False}'
	... Create File ${TEST_HISTORY}

	${cur_time}= Get Current Time Stamp

	Append To File ${TEST_HISTORY}
	... ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}


	Log FFDC Get Requests
	[Documentation] Create file in current FFDC log directory.
	... Do openbmc get request and write to
	... corresponding file name.
	... JSON pretty print for logging to file.
	[Arguments] ${key_index}

	@{cmd_list}= Get ffdc get request ${key_index}
	:FOR ${cmd} IN @{cmd_list}
	\ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt
	\ ${resp}= OpenBMC Get Request ${cmd[1]} quiet=${1}
	\ ${status}= Run Keyword and Return Status
	... Should Be Equal As Strings ${resp.status_code} ${HTTP_OK}
	\ Run Keyword If '${status}' == '${False}' Continue For Loop
	\ ${jsondata}= to json ${resp.content} pretty_print=True
	\ Write Data To File ${\n}${jsondata}${\n} ${logpath}


	BMC FFDC Get Requests
	[Documentation] Get the command list and iterate
	Open Connection And Log In
	@{entries}= Get ffdc get request index
	:FOR ${index} IN @{entries}
	\ Log FFDC Get Requests ${index}


	Log OS ALL DISTROS FFDC
	[Documentation] Create file in current FFDC log directory.
	... Executes OS command and write to
	... corresponding file name.
	[Arguments] ${key_index}

	@{cmd_list}= get ffdc os all distros call ${key_index}
	:FOR ${cmd} IN @{cmd_list}
	\ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt
	\ Execute Command and Write FFDC ${cmd[0]} ${cmd[1]} ${logpath}


	Log OS SPECIFIC DISTRO FFDC
	[Documentation] Create file in current FFDC log directory.
	... Executes OS command and write to
	... corresponding file name.
	[Arguments] ${key_index} ${linux_distro}

	@{cmd_list}= get ffdc os distro call ${key_index} ${linux_distro}
	:FOR ${cmd} IN @{cmd_list}
	\ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt
	\ Execute Command and Write FFDC ${cmd[0]} ${cmd[1]} ${logpath}



	OS FFDC Files
	[Documentation] Get the command list and iterate
	[Arguments] ${OS_HOST}=${OS_HOST} ${OS_USERNAME}=${OS_USERNAME}
	... ${OS_PASSWORD}=${OS_PASSWORD}

	Return From Keyword If '${OS_HOST}' == '${EMPTY}'
	... No OS Host Provided

	# If can't ping, return
	${rc}= Run Keyword and Return Status Ping Host ${OS_HOST}
	Return From Keyword If '${rc}' == '${False}'
	... Could not ping OS

	Open Connection And Log In host=${OS_HOST} username=${OS_USERNAME}
	... password=${OS_PASSWORD}

	${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True
	... return_rc=True

	# If the return code returned by "Execute Command" is non-zero, return
	Return From Keyword If '${rc}' != '${0}'
	... Could not connect to OS

	@{entries}= Get ffdc os all distros index
	:FOR ${index} IN @{entries}
	\ Log OS ALL DISTROS FFDC ${index}

	${linux_distro}= Execute Command
	... . /etc/os-release; echo $ID
	... return_stdout=True return_stderr=False return_rc=False

	Return From Keyword If
	... '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
	... Could not determine Linux Distribution

	@{entries}= Get ffdc os distro index ${linux_distro}
	:FOR ${index} IN @{entries}
	\ Log OS SPECIFIC DISTRO FFDC ${index} ${linux_distro}


	##############################################################################
	SCP Coredump Files
	[Documentation] Copy core dump file from BMC to local system.
	# Check if core dump exist in the /tmp
	Open Connection And Log In
	${core_files}= Execute Command ls /tmp/core_*
	@{core_list} = Split String ${core_files}
	# Copy the core files
	Open Connection for SCP
	:FOR ${index} IN @{core_list}
	\ scp.Get File ${index} ${LOG_PREFIX}${index.lstrip("/tmp/")}
	# Remove the file from remote to avoid re-copying on next FFDC call
	\ Execute Command On BMC rm ${index}


	##############################################################################
	Collect eSEL Log
	[Documentation] Collect eSEL log from logging entry and convert eSEL data
	... to elog formated string text file.
	${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}/enumerate quiet=${1}
	${status}= Run Keyword And Return Status
	... Should Be Equal As Strings ${resp.status_code} ${HTTP_OK}
	Return From Keyword If '${status}' == '${False}'

	${content}= To Json ${resp.content}
	# Grab the list of entries from logging/entry/
	# The data shown below is the result of the "Get Dictionary Keys".
	# Example:
	# /xyz/openbmc_project/logging/entry/1
	# /xyz/openbmc_project/logging/entry/2
	${esel_list}= Get Dictionary Keys ${content['data']}

	${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} esel
	Create File ${logpath}
	# Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
	# "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
	# Sample eSEL entry:
	# "/xyz/openbmc_project/logging/entry/1": {
	# "Timestamp": 1487744317025,
	# "AdditionalData": [
	# "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
	# ],
	# "Message": "org.open_power.Error.Host.Event.Event",
	# "Id": 1,
	# "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
	# }

	:FOR ${entry_path} IN @{esel_list}
	\ ${esel_data}= Read Attribute ${entry_path} AdditionalData quiet=${1}
	\ ${length}= Get Length ${esel_data}
	# Skip writting to file if eSEL AdditionalData is empty
	\ Continue For Loop If ${length} == ${0}
	\ Write Data To File "${esel_data[0]}" ${logpath}
	\ Write Data To File ${\n} ${logpath}

	${out}= Run which eSEL.pl
	${status}= Run Keyword And Return Status
	... Should Contain ${out} eSEL.pl
	Return From Keyword If '${status}' == '${False}'

	Convert eSEL To Elog Format ${logpath}


	##############################################################################
	Convert eSEL To Elog Format
	[Documentation] Execute parser tool on the eSEL data file to generate
	... formated error log.
	[Arguments] ${esel_file_path}
	# Desription of arguments:
	# ${esel_file_path} Absoulte path of the eSEL data.

	Run eSEL.pl -l ${esel_file_path}

	##############################################################################