Add re-try for redfish GET request for targets

Changes:
    - This retries is added to improve the success rate
      for the GET request in case returns JSON errors
      for the following URI during operation
      /redfish/v1/Systems/ and /redfish/v1/Managers/

    - Added other places in code to handle exceptions.
    - Fix IPMI import paths in test suite

Tested:
   - Ran from sandbox.

Change-Id: Ie37dbdd9c7572e0003a56e65a7b5f0a4c3b26a1a
Signed-off-by: George Keishing <gkeishin@in.ibm.com>
diff --git a/ipmi/test_ipmi_sel.robot b/ipmi/test_ipmi_sel.robot
index 77a45b3..d1c6dc4 100644
--- a/ipmi/test_ipmi_sel.robot
+++ b/ipmi/test_ipmi_sel.robot
@@ -3,10 +3,10 @@
 Documentation    Module to test IPMI SEL functionality.
 Resource         ../lib/ipmi_client.robot
 Resource         ../lib/openbmc_ffdc.robot
-Resource         ../../../lib/logging_utils.robot
+Resource         ../lib/logging_utils.robot
 
 Library          ../lib/ipmi_utils.py
-Library          ../../../lib/logging_utils.py
+Library          ../lib/logging_utils.py
 
 Variables        ../data/ipmi_raw_cmd_table.py
 
diff --git a/lib/bmc_redfish_utils.py b/lib/bmc_redfish_utils.py
index e240d94..8299abd 100644
--- a/lib/bmc_redfish_utils.py
+++ b/lib/bmc_redfish_utils.py
@@ -6,6 +6,7 @@
 
 import json
 import re
+from json.decoder import JSONDecodeError
 
 import gen_print as gp
 from robot.libraries.BuiltIn import BuiltIn
@@ -66,8 +67,13 @@
                                     "/redfish/v1/Systems/1").
         attribute                   Name of the attribute (e.g. 'PowerState').
         """
-
-        resp = self._redfish_.get(resource_path)
+        try:
+            resp = self._redfish_.get(resource_path)
+        except JSONDecodeError as e:
+            BuiltIn().log_to_console(
+                "get_attribute: JSONDecodeError, re-trying"
+            )
+            resp = self._redfish_.get(resource_path)
 
         if verify:
             if resp.dict[attribute] == verify:
@@ -308,9 +314,18 @@
                 ):
                     continue
 
-                self._rest_response_ = self._redfish_.get(
-                    resource, valid_status_codes=[200, 404, 405, 500]
-                )
+                try:
+                    self._rest_response_ = self._redfish_.get(
+                        resource, valid_status_codes=[200, 404, 405, 500]
+                    )
+                except JSONDecodeError as e:
+                    BuiltIn().log_to_console(
+                        "enumerate_request: JSONDecodeError, re-trying"
+                    )
+                    self._rest_response_ = self._redfish_.get(
+                        resource, valid_status_codes=[200, 404, 405, 500]
+                    )
+
                 # Enumeration is done for available resources ignoring the
                 # ones for which response is not obtained.
                 if self._rest_response_.status != 200:
diff --git a/lib/bmc_redfish_utils.robot b/lib/bmc_redfish_utils.robot
index 1e8bb10..c4422e8 100644
--- a/lib/bmc_redfish_utils.robot
+++ b/lib/bmc_redfish_utils.robot
@@ -32,7 +32,8 @@
     #  }
     # }
 
-    ${target}=  redfish_utils.Get Target Actions  /redfish/v1/Systems/${SYSTEM_ID}/  ComputerSystem.Reset
+    ${target}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  redfish_utils.Get Target Actions  /redfish/v1/Systems/${SYSTEM_ID}/  ComputerSystem.Reset
     ${payload}=  Create Dictionary  ResetType=${reset_type}
     ${resp}=  Redfish.Post  ${target}  body=&{payload}
     ...  valid_status_codes=[${HTTP_OK}, ${HTTP_NO_CONTENT}]
@@ -52,7 +53,8 @@
     #  "target": "/redfish/v1/Managers/${MANAGER_ID}/Actions/Manager.Reset"
     # }
 
-    ${target}=  redfish_utils.Get Target Actions  /redfish/v1/Managers/${MANAGER_ID}/  Manager.Reset
+    ${target}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  redfish_utils.Get Target Actions  /redfish/v1/Managers/${MANAGER_ID}/  Manager.Reset
     ${payload}=  Create Dictionary  ResetType=${reset_type}
     Redfish.Post  ${target}  body=&{payload}
 
diff --git a/lib/utils.robot b/lib/utils.robot
index 925b148..084d400 100755
--- a/lib/utils.robot
+++ b/lib/utils.robot
@@ -389,7 +389,8 @@
 Redfish Get Auto Reboot
     [Documentation]  Returns auto reboot setting.
 
-    ${resp}=  Redfish.Get Attribute  /redfish/v1/Systems/${SYSTEM_ID}  Boot
+    ${resp}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  Redfish.Get Attribute  /redfish/v1/Systems/${SYSTEM_ID}  Boot
     RETURN  ${resp["AutomaticRetryConfig"]}
 
 
@@ -906,7 +907,8 @@
     #    "State": "Enabled"
     # },
 
-    ${status}=  Redfish.Get Attribute  /redfish/v1/Managers/${MANAGER_ID}  Status
+    ${status}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  Redfish.Get Attribute  /redfish/v1/Managers/${MANAGER_ID}  Status
     RETURN  ${status["State"]}
 
 
@@ -917,7 +919,8 @@
     # Description of argument(s):
     # match_state    Expected match state (e.g. Enabled, Starting, Error)
 
-    ${Status}=  Redfish.Get Attribute  /redfish/v1/Managers/${MANAGER_ID}  Status
+    ${Status}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  Redfish.Get Attribute  /redfish/v1/Managers/${MANAGER_ID}  Status
 
     Should Be Equal As Strings  ${match_state}  ${Status['State']}
 
@@ -935,7 +938,8 @@
     #    "State": "StandbyOffline"
     # },
 
-    ${chassis}=  Redfish.Get Properties  /redfish/v1/Chassis/${CHASSIS_ID}
+    ${chassis}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  Redfish.Get Properties  /redfish/v1/Chassis/${CHASSIS_ID}
     RETURN  ${chassis["PowerState"]}  ${chassis["Status"]["State"]}
 
 
@@ -947,7 +951,8 @@
     #    "LastState": "OSRunning"
     # },
 
-    ${boot_progress}=  Redfish.Get Properties  /redfish/v1/Systems/${SYSTEM_ID}/
+    ${boot_progress}=  Wait Until Keyword Succeeds  1 min  20 sec
+    ...  Redfish.Get Properties  /redfish/v1/Systems/${SYSTEM_ID}/
 
     Return From Keyword If  "${PLATFORM_ARCH_TYPE}" == "x86"
     ...  NA  ${boot_progress["Status"]["State"]}
diff --git a/redfish/service_root/test_service_root.robot b/redfish/service_root/test_service_root.robot
index b505b6f..1a7888c 100644
--- a/redfish/service_root/test_service_root.robot
+++ b/redfish/service_root/test_service_root.robot
@@ -77,7 +77,8 @@
     [Tags]  Verify_Redfish_Invalid_URL_Response_Code
 
     Redfish.Login
-    Redfish.Get  /redfish/v1/idontexist  valid_status_codes=[${HTTP_NOT_FOUND}]
+    Wait Until Keyword Succeeds  1 min  30 sec
+    ...  Redfish.Get  /redfish/v1/idontexist  valid_status_codes=[${HTTP_NOT_FOUND}]
     Redfish.Logout