Add simple Redfish tests
Added Redfish tests to validate connection between Redfish
Telemetry that is implemented in bmcweb and Telemetry service.
Change-Id: Iacc63aeb7f2852d5acac5d5615f98b402f7c4417
Signed-off-by: Wludzik, Jozef <jozef.wludzik@intel.com>
diff --git a/.gitignore b/.gitignore
index 1ee5855..b36d642 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
build/
xyz.openbmc_project.Telemetry.service
telemetry
+__pycache__
# Created by https://www.toptal.com/developers/gitignore/api/vim,intellij,meson,visualstudiocode
# Edit at https://www.toptal.com/developers/gitignore?templates=vim,intellij,meson,visualstudiocode
diff --git a/redfish-tests/README.md b/redfish-tests/README.md
new file mode 100644
index 0000000..553fe62
--- /dev/null
+++ b/redfish-tests/README.md
@@ -0,0 +1,37 @@
+# Requirements
+
+At least python3.6 is required and following two python packages installed -
+requests, pytest.
+
+```
+$ python3 --version
+Python 3.6.8
+
+$ pytest --version
+This is pytest version 5.4.2, imported from /usr/local/lib/python3.6/site-packages/pytest/__init__.py
+
+$ sudo pip3 install request pytest
+```
+
+# Config
+
+There are few options included to tests:
+- `--host_addr` - address of an OpenBMC together with protocol and port, ex.
+'https://localhost:443', default: 'https://localhost:4443'
+- `--username` - name of the user that is used during tests, user should have
+'ConfigureManager' privilege, default: 'root'
+- `--password` - plain password, default: '0penBmc'
+- `--metric_limit` - allows to limit number of detected metrics in system,
+default: 200
+
+SSL verification is disabled during tests.
+
+# Run
+
+```
+# To run all tests in verbose mode and with stop on first failed test (s option direct all print to stdout)
+ pytest test_telemetry.py -xsv
+
+# To run specific test using statement
+ pytest test_telemetry.py -xsv -k 'test_get_telemetry_service or test_get_metric_report'
+```
diff --git a/redfish-tests/conftest.py b/redfish-tests/conftest.py
new file mode 100644
index 0000000..d5ed4ae
--- /dev/null
+++ b/redfish-tests/conftest.py
@@ -0,0 +1,36 @@
+import pytest
+import redfish_requests
+
+
+def pytest_addoption(parser):
+ parser.addoption('--host_addr', action='store',
+ default='https://localhost:4443')
+ parser.addoption('--username', action='store', default='root')
+ parser.addoption('--password', action='store', default='0penBmc')
+ parser.addoption('--metric_limit', action='store', default=200)
+
+
+@pytest.fixture(scope='session')
+def redfish(request):
+ host_addr = request.config.getoption('--host_addr')
+ username = request.config.getoption('--username')
+ password = request.config.getoption('--password')
+ return redfish_requests.RedfishRequest(host_addr, username, password)
+
+
+@pytest.fixture(scope='session')
+def telemetry(request, redfish):
+ metric_limit = request.config.getoption('--metric_limit')
+ return redfish_requests.TelemetryService(redfish, metric_limit)
+
+
+@pytest.fixture(scope='function')
+def report_definitions(redfish):
+ report_definitions = redfish_requests.ReportDef(redfish)
+ print('Cleaning reports before test')
+ for report in report_definitions.get_collection():
+ report_definitions.delete_report(report)
+ yield report_definitions
+ print('Cleaning reports after test')
+ for report in report_definitions.get_collection():
+ report_definitions.delete_report(report)
diff --git a/redfish-tests/redfish_requests.py b/redfish-tests/redfish_requests.py
new file mode 100644
index 0000000..28dee84
--- /dev/null
+++ b/redfish-tests/redfish_requests.py
@@ -0,0 +1,127 @@
+import enum
+import math
+import re
+import requests
+
+
+class RedfishHttpStatus(enum.IntEnum):
+ ok = 200
+ created = 201
+ no_content = 204
+ bad_request = 400
+ not_found = 404
+ internal_server_error = 500
+
+
+class RedfishRequest:
+ telemetry_service_path = '/redfish/v1/TelemetryService'
+ metric_definition_path = f'{telemetry_service_path}/MetricDefinitions'
+ metric_report_definition_path = \
+ f'{telemetry_service_path}/MetricReportDefinitions'
+ metric_report_path = f'{telemetry_service_path}/MetricReports'
+
+ def __init__(self, host_addr, username, password):
+ self.host_addr = host_addr
+ self.username = username
+ self.password = password
+
+ def get(self, path, code=RedfishHttpStatus.ok):
+ u = self.host_addr + path
+ r = requests.get(u, auth=(self.username, self.password), verify=False)
+ assert r.status_code == code, \
+ f'{r.status_code} == {code} on path {u}\n{r.text}'
+ print(r.text)
+ return r.json()
+
+ def post(self, path, body, code=RedfishHttpStatus.created):
+ u = self.host_addr + path
+ r = requests.post(u, auth=(self.username, self.password), verify=False,
+ json=body)
+ assert r.status_code == code, \
+ f'{r.status_code} == {code} on path {u}\n{r.text}'
+ print(r.text)
+ return r.json()
+
+ def delete(self, path, code=RedfishHttpStatus.no_content):
+ u = self.host_addr + path
+ r = requests.delete(u, auth=(self.username, self.password),
+ verify=False)
+ assert r.status_code == code, \
+ f'{r.status_code} == {code} on path {u}\n{r.text}'
+
+
+class TelemetryService:
+ def __init__(self, redfish, metric_limit):
+ r = redfish.get(redfish.telemetry_service_path)
+ self.min_interval = Duration.to_seconds(r['MinCollectionInterval'])
+ self.max_reports = r['MaxReports']
+ self.metrics = []
+ r = redfish.get(redfish.metric_definition_path)
+ for m in r['Members']:
+ path = m['@odata.id']
+ metricDef = redfish.get(path)
+ self.metrics += [x for x in metricDef['MetricProperties']]
+ self.metrics = self.metrics[:metric_limit]
+
+
+class ReportDef:
+ def __init__(self, redfish):
+ self.redfish = redfish
+
+ def get_collection(self):
+ r = self.redfish.get(self.redfish.metric_report_definition_path)
+ return [x['@odata.id'] for x in r['Members']]
+
+ def add_report(self, id, metrics=None, type='OnRequest', actions=None,
+ interval=None, code=RedfishHttpStatus.created):
+ body = {
+ 'Id': id,
+ 'Metrics': [],
+ 'MetricReportDefinitionType': type,
+ 'ReportActions': ['RedfishEvent', 'LogToMetricReportsCollection']
+ }
+ if metrics is not None:
+ body['Metrics'] = metrics
+ if actions is not None:
+ body['ReportActions'] = actions
+ if interval is not None:
+ body['Schedule'] = {'RecurrenceInterval': interval}
+ return self.redfish.post(self.redfish.metric_report_definition_path,
+ body, code)
+
+ def delete_report(self, path):
+ self.redfish.delete(f'{path}')
+
+
+class Duration:
+ def __init__(self):
+ pass
+
+ def to_iso8061(time):
+ assert time >= 0, 'Invalid argument, time is negative'
+ days = int(time / (24 * 60 * 60))
+ time = math.fmod(time, (24 * 60 * 60))
+ hours = int(time / (60 * 60))
+ time = math.fmod(time, (60 * 60))
+ minutes = int(time / 60)
+ time = round(math.fmod(time, 60), 3)
+ return f'P{str(days)}DT{str(hours)}H{str(minutes)}M{str(time)}S'
+
+ def to_seconds(duration):
+ r = re.fullmatch(r'-?P(\d+D)?(T(\d+H)?(\d+M)?(\d+(.\d+)?S)?)?',
+ duration)
+ assert r, 'Invalid argument, not match with regex'
+ days = r.group(1)
+ hours = r.group(3)
+ minutes = r.group(4)
+ seconds = r.group(5)
+ result = 0
+ if days is not None:
+ result += int(days[:-1]) * 60 * 60 * 24
+ if hours is not None:
+ result += int(hours[:-1]) * 60 * 60
+ if minutes is not None:
+ result += int(minutes[:-1]) * 60
+ if seconds is not None:
+ result += float(seconds[:-1])
+ return result
diff --git a/redfish-tests/test_telemetry.py b/redfish-tests/test_telemetry.py
new file mode 100644
index 0000000..301def3
--- /dev/null
+++ b/redfish-tests/test_telemetry.py
@@ -0,0 +1,241 @@
+from redfish_requests import RedfishHttpStatus, RedfishRequest, Duration
+import pytest
+import time
+
+
+def test_get_telemetry_service(redfish):
+ r = redfish.get(redfish.telemetry_service_path)
+ assert r['Status']['State'] == 'Enabled', 'Invalid status of service'
+ assert Duration.to_seconds(r['MinCollectionInterval']) > 0, \
+ 'Invalid duration format'
+ assert r['MaxReports'] > 0, 'Invalid count of max reports'
+
+
+def test_get_metric_definition_collection(redfish):
+ r = redfish.get(redfish.metric_definition_path)
+ assert 'Members' in r, 'Missing members property'
+ assert 'Members@odata.count' in r, 'Missing members count property'
+
+
+def test_verify_metric_definition_members_if_contains_metrics(redfish):
+ r = redfish.get(redfish.metric_definition_path)
+ for m in r['Members']:
+ path = m['@odata.id']
+ metricDefinition = redfish.get(path)
+ assert 'MetricProperties' in metricDefinition, 'Missing metrics'
+ assert len(metricDefinition['MetricProperties']) > 0, 'Missing metrics'
+
+
+def test_get_metric_definition_that_not_exist_expect_not_found(redfish):
+ r = redfish.get(f'{redfish.metric_definition_path}/NotExisting',
+ code=RedfishHttpStatus.not_found)
+
+
+def test_get_metric_report_definition_collection(redfish):
+ r = redfish.get(redfish.metric_report_definition_path)
+ assert 'Members' in r, 'Missing members property'
+ assert 'Members@odata.count' in r, 'Missing members count property'
+
+
+def test_get_metric_report_definition_that_not_exist_expect_not_found(
+ redfish):
+ r = redfish.get(f'{redfish.metric_report_definition_path}/NotExisting',
+ code=RedfishHttpStatus.not_found)
+
+
+def test_get_metric_report_collection(redfish):
+ r = redfish.get(redfish.metric_report_path)
+ assert 'Members' in r, 'Missing members property'
+ assert 'Members@odata.count' in r, 'Missing members count property'
+
+
+def test_get_metric_report_that_not_exist_expect_not_found(redfish):
+ r = redfish.get(f'{redfish.metric_report_path}/NotExisting',
+ code=RedfishHttpStatus.not_found)
+
+
+def test_post_report_definition_with_empty_body_expect_bad_request(redfish):
+ redfish.post(redfish.metric_report_definition_path, body={},
+ code=RedfishHttpStatus.bad_request)
+
+
+def test_post_report_definition_with_some_body_expect_bad_request(redfish):
+ redfish.post(redfish.metric_report_definition_path, body={'key': 'value'},
+ code=RedfishHttpStatus.bad_request)
+
+
+def test_delete_non_exisiting_metric_report_definition(redfish):
+ redfish.delete(
+ f'{redfish.metric_report_definition_path}/NonExisitingReport',
+ code=RedfishHttpStatus.not_found)
+
+
+def test_add_report(redfish, report_definitions):
+ id = 'Test'
+ path = report_definitions.add_report(id)
+ assert 1 == len(report_definitions.get_collection())
+ r = redfish.get(f'{redfish.metric_report_definition_path}/{id}')
+ assert r['Id'] == id, 'Invalid Id, different then requested'
+ r = redfish.get(f'{redfish.metric_report_path}/{id}')
+ assert r['Id'] == id, 'Invalid Id, different then requested'
+
+
+def test_add_report_above_max_report_expect_bad_request(
+ telemetry, report_definitions):
+ id = 'Test'
+ for i in range(telemetry.max_reports):
+ report_definitions.add_report(id + str(i))
+ assert telemetry.max_reports == len(report_definitions.get_collection())
+ report_definitions.add_report(id + str(telemetry.max_reports),
+ metrics=[], interval=telemetry.min_interval,
+ code=RedfishHttpStatus.bad_request)
+
+
+def test_add_report_long_name(report_definitions):
+ report_definitions.add_report('Test' * 65)
+
+
+def test_add_report_twice_expect_bad_request(report_definitions):
+ report_definitions.add_report('Test')
+ report_definitions.add_report('Test', code=RedfishHttpStatus.bad_request)
+
+
+@pytest.mark.parametrize(
+ 'actions', [[], ['RedfishEvent'], ['LogToMetricReportsCollection'],
+ ['RedfishEvent', 'LogToMetricReportsCollection']])
+def test_add_report_with_actions(actions, redfish, report_definitions):
+ report_definitions.add_report('Test', actions=actions)
+ r = redfish.get(f'{redfish.metric_report_definition_path}/Test')
+ assert r['ReportActions'] == actions, \
+ 'Invalid actions, different then requested'
+
+
+@pytest.mark.parametrize(
+ 'invalid_actions', [['NonExisting'], ['RedfishEvent', 'Partially'],
+ ['LogToMetricNotThisOne']])
+def test_add_report_with_invalid_actions_expect_bad_request(
+ invalid_actions, report_definitions):
+ report_definitions.add_report('Test', actions=invalid_actions,
+ code=RedfishHttpStatus.bad_request)
+
+
+@pytest.mark.parametrize('invalid_id', ['test_-', 't t', 'T.T', 'T,t', 'T:t'])
+def test_add_report_with_invalid_id_expect_bad_request(
+ invalid_id, report_definitions):
+ report_definitions.add_report(invalid_id,
+ code=RedfishHttpStatus.bad_request)
+
+
+def test_add_report_with_metric(redfish, telemetry, report_definitions):
+ if len(telemetry.metrics) <= 0:
+ pytest.skip('Redfish has no sensor available')
+ metric = {'MetricId': 'Id1', 'MetricProperties': [telemetry.metrics[0]]}
+ report_definitions.add_report('Test', metrics=[metric])
+ r = redfish.get(redfish.metric_report_definition_path + '/Test')
+ assert len(r['Metrics']) == 1, 'Invalid Metrics, different then requested'
+ assert r['Metrics'][0]['MetricId'] == metric['MetricId'], \
+ 'Invalid MetricId, different then requested'
+ assert r['Metrics'][0]['MetricProperties'] == metric['MetricProperties'], \
+ 'Invalid MetricProperties, different then requested'
+
+
+def test_add_report_with_invalid_metric_expect_bad_request(report_definitions):
+ metric = {
+ 'MetricId': 'Id1',
+ 'MetricProperties':
+ ['/redfish/v1/Chassis/chassis/Sensors/NonExisting/Reading']
+ }
+ report_definitions.add_report('Test', metrics=[metric],
+ code=RedfishHttpStatus.bad_request)
+
+
+def test_add_report_with_many_metrics(redfish, telemetry, report_definitions):
+ if len(telemetry.metrics) <= 0:
+ pytest.skip('Redfish has no sensor available')
+ metrics = []
+ for i, prop in enumerate(telemetry.metrics):
+ metrics.append({'MetricId': f'Id{str(i)}', 'MetricProperties': [prop]})
+ report_definitions.add_report('Test', metrics=metrics)
+ r = redfish.get(redfish.metric_report_definition_path + '/Test')
+ assert len(r['Metrics']) == len(telemetry.metrics), \
+ 'Invalid Metrics, different then requested'
+
+
+def test_add_report_on_request_with_metric_expect_updated_metric_report(
+ redfish, telemetry, report_definitions):
+ if len(telemetry.metrics) <= 0:
+ pytest.skip('Redfish has no sensor available')
+ metric = {'MetricId': 'Id1', 'MetricProperties': [telemetry.metrics[0]]}
+ report_definitions.add_report('Test', metrics=[metric], type='OnRequest')
+ r = redfish.get(redfish.metric_report_path + '/Test')
+ assert len(r['MetricValues']) > 0, 'Missing MetricValues'
+ metric_value = r['MetricValues'][0]
+ assert metric_value['MetricValue'], 'Missing MetricValues'
+ assert metric_value['MetricId'] == metric['MetricId'], \
+ 'Different Id then set in request'
+ assert metric_value['MetricProperty'] == metric['MetricProperties'][0], \
+ 'Different MetricProperty then set in request'
+
+
+def test_add_report_periodic_with_metric_expect_updated_metric_report(
+ redfish, telemetry, report_definitions):
+ if len(telemetry.metrics) <= 0:
+ pytest.skip('Redfish has no sensor available')
+ metric = {'MetricId': 'Id1', 'MetricProperties': [telemetry.metrics[0]]}
+ report_definitions.add_report(
+ 'Test', metrics=[metric], type='Periodic',
+ interval=Duration.to_iso8061(telemetry.min_interval))
+ time.sleep(telemetry.min_interval + 1)
+ r = redfish.get(redfish.metric_report_path + '/Test')
+ assert len(r['MetricValues']) > 0, 'Missing MetricValues'
+ metric_value = r['MetricValues'][0]
+ assert metric_value['MetricValue'], 'Missing MetricValues'
+ assert metric_value['MetricId'] == metric['MetricId'], \
+ 'Different Id then set in request'
+ assert metric_value['MetricProperty'] == metric['MetricProperties'][0], \
+ 'Different MetricProperty then set in request'
+
+
+@pytest.mark.parametrize('interval', [10, 60, 2400, 90000])
+def test_add_report_check_if_duration_is_set(interval, redfish, telemetry,
+ report_definitions):
+ if interval < telemetry.min_interval:
+ pytest.skip('Interval is below minimal acceptable value, skipping')
+ id = f'Test{str(interval)}'
+ report_definitions.add_report(id, type='Periodic',
+ interval=Duration.to_iso8061(interval))
+ r = redfish.get(f'{redfish.metric_report_definition_path}/{id}')
+ assert r['Schedule']['RecurrenceInterval'], 'Missing RecurrenceInterval'
+ r_interval = Duration.to_seconds(r['Schedule']['RecurrenceInterval'])
+ assert interval == r_interval, 'Invalid interval, different then requested'
+
+
+@pytest.mark.parametrize(
+ 'invalid', ['50000', 'P12ST', 'PT12S12', 'PPP' 'PD222T222H222M222.222S'])
+def test_add_report_with_invalid_duration_response_bad_request(
+ invalid, report_definitions):
+ r = report_definitions.add_report('Test', type='Periodic', interval=invalid,
+ code=RedfishHttpStatus.bad_request)
+ assert r['error']['@Message.ExtendedInfo'][0], \
+ 'Wrong response, not an error'
+ info = r['error']['@Message.ExtendedInfo'][0]
+ assert 'RecurrenceInterval' in info['MessageArgs'], \
+ 'Wrong response, should contain "RecurrenceInterval"'
+
+
+def test_stress_add_reports_with_many_metrics_check_metric_reports(
+ redfish, telemetry, report_definitions):
+ if len(telemetry.metrics) <= 0:
+ pytest.skip('Redfish has no sensor available')
+ metrics = []
+ for i, prop in enumerate(telemetry.metrics):
+ metrics.append({'MetricId': f'Id{str(i)}', 'MetricProperties': [prop]})
+ for i in range(telemetry.max_reports):
+ report_definitions.add_report(f'Test{str(i)}', metrics=metrics)
+ for i in range(telemetry.max_reports):
+ r = redfish.get(f'{redfish.metric_report_definition_path}/Test{str(i)}')
+ assert len(r['Metrics']) == len(telemetry.metrics), \
+ 'Invalid Metrics, different then requested'
+ for i in range(telemetry.max_reports):
+ r = redfish.get(f'{redfish.metric_report_path}/Test{str(i)}')
+ assert len(r['MetricValues']) > 0, 'Missing MetricValues'