blob: df28e1d21e81cff65602b394fb9fcceb76cdcbc6 [file] [log] [blame]
Brad Bishop40320b12019-03-26 16:08:25 -04001# test case management tool - manual execution from testopia test cases
2#
3# Copyright (c) 2018, Intel Corporation.
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Brad Bishop40320b12019-03-26 16:08:25 -04006#
Brad Bishopc342db32019-05-15 21:57:59 -04007
Brad Bishop40320b12019-03-26 16:08:25 -04008import argparse
9import json
10import os
11import sys
12import datetime
13import re
Brad Bishopc342db32019-05-15 21:57:59 -040014import copy
Brad Bishop40320b12019-03-26 16:08:25 -040015from oeqa.core.runner import OETestResultJSONHelper
16
17
Brad Bishopc342db32019-05-15 21:57:59 -040018def load_json_file(f):
19 with open(f, "r") as filedata:
20 return json.load(filedata)
Brad Bishop40320b12019-03-26 16:08:25 -040021
Brad Bishopd89cb5f2019-04-10 09:02:41 -040022def write_json_file(f, json_data):
23 os.makedirs(os.path.dirname(f), exist_ok=True)
24 with open(f, 'w') as filedata:
25 filedata.write(json.dumps(json_data, sort_keys=True, indent=4))
26
Brad Bishop40320b12019-03-26 16:08:25 -040027class ManualTestRunner(object):
Brad Bishop40320b12019-03-26 16:08:25 -040028
Brad Bishopc342db32019-05-15 21:57:59 -040029 def _get_test_module(self, case_file):
30 return os.path.basename(case_file).split('.')[0]
Brad Bishop19323692019-04-05 15:28:33 -040031
Brad Bishop40320b12019-03-26 16:08:25 -040032 def _get_input(self, config):
33 while True:
34 output = input('{} = '.format(config))
Brad Bishop19323692019-04-05 15:28:33 -040035 if re.match('^[a-z0-9-.]+$', output):
Brad Bishop40320b12019-03-26 16:08:25 -040036 break
Brad Bishop19323692019-04-05 15:28:33 -040037 print('Only lowercase alphanumeric, hyphen and dot are allowed. Please try again')
Brad Bishop40320b12019-03-26 16:08:25 -040038 return output
39
Brad Bishopd89cb5f2019-04-10 09:02:41 -040040 def _get_available_config_options(self, config_options, test_module, target_config):
41 avail_config_options = None
42 if test_module in config_options:
43 avail_config_options = config_options[test_module].get(target_config)
44 return avail_config_options
45
46 def _choose_config_option(self, options):
47 while True:
48 output = input('{} = '.format('Option index number'))
49 if output in options:
50 break
51 print('Only integer index inputs from above available configuration options are allowed. Please try again.')
52 return options[output]
53
Brad Bishopc342db32019-05-15 21:57:59 -040054 def _get_config(self, config_options, test_module):
Brad Bishop40320b12019-03-26 16:08:25 -040055 from oeqa.utils.metadata import get_layers
56 from oeqa.utils.commands import get_bb_var
57 from resulttool.resultutils import store_map
58
59 layers = get_layers(get_bb_var('BBLAYERS'))
Brad Bishopc342db32019-05-15 21:57:59 -040060 configurations = {}
61 configurations['LAYERS'] = layers
62 configurations['STARTTIME'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
63 configurations['TEST_TYPE'] = 'manual'
64 configurations['TEST_MODULE'] = test_module
Brad Bishop40320b12019-03-26 16:08:25 -040065
Brad Bishopc342db32019-05-15 21:57:59 -040066 extra_config = set(store_map['manual']) - set(configurations)
Brad Bishop40320b12019-03-26 16:08:25 -040067 for config in sorted(extra_config):
Brad Bishopc342db32019-05-15 21:57:59 -040068 avail_config_options = self._get_available_config_options(config_options, test_module, config)
Brad Bishopd89cb5f2019-04-10 09:02:41 -040069 if avail_config_options:
70 print('---------------------------------------------')
71 print('These are available configuration #%s options:' % config)
72 print('---------------------------------------------')
73 for option, _ in sorted(avail_config_options.items(), key=lambda x: int(x[0])):
74 print('%s: %s' % (option, avail_config_options[option]))
75 print('Please select configuration option, enter the integer index number.')
76 value_conf = self._choose_config_option(avail_config_options)
77 print('---------------------------------------------\n')
78 else:
79 print('---------------------------------------------')
80 print('This is configuration #%s. Please provide configuration value(use "None" if not applicable).' % config)
81 print('---------------------------------------------')
82 value_conf = self._get_input('Configuration Value')
83 print('---------------------------------------------\n')
Brad Bishopc342db32019-05-15 21:57:59 -040084 configurations[config] = value_conf
85 return configurations
Brad Bishop40320b12019-03-26 16:08:25 -040086
Brad Bishopc342db32019-05-15 21:57:59 -040087 def _execute_test_steps(self, case):
Brad Bishop40320b12019-03-26 16:08:25 -040088 test_result = {}
Brad Bishop40320b12019-03-26 16:08:25 -040089 print('------------------------------------------------------------------------')
Brad Bishopc342db32019-05-15 21:57:59 -040090 print('Executing test case: %s' % case['test']['@alias'])
Brad Bishop40320b12019-03-26 16:08:25 -040091 print('------------------------------------------------------------------------')
Brad Bishopc342db32019-05-15 21:57:59 -040092 print('You have total %s test steps to be executed.' % len(case['test']['execution']))
Brad Bishop40320b12019-03-26 16:08:25 -040093 print('------------------------------------------------------------------------\n')
Brad Bishopc342db32019-05-15 21:57:59 -040094 for step, _ in sorted(case['test']['execution'].items(), key=lambda x: int(x[0])):
95 print('Step %s: %s' % (step, case['test']['execution'][step]['action']))
96 expected_output = case['test']['execution'][step]['expected_results']
Brad Bishop19323692019-04-05 15:28:33 -040097 if expected_output:
98 print('Expected output: %s' % expected_output)
Brad Bishop40320b12019-03-26 16:08:25 -040099 while True:
Brad Bishop19323692019-04-05 15:28:33 -0400100 done = input('\nPlease provide test results: (P)assed/(F)ailed/(B)locked/(S)kipped? \n').lower()
Brad Bishop40320b12019-03-26 16:08:25 -0400101 result_types = {'p':'PASSED',
Brad Bishop19323692019-04-05 15:28:33 -0400102 'f':'FAILED',
103 'b':'BLOCKED',
104 's':'SKIPPED'}
Brad Bishop40320b12019-03-26 16:08:25 -0400105 if done in result_types:
106 for r in result_types:
107 if done == r:
108 res = result_types[r]
109 if res == 'FAILED':
110 log_input = input('\nPlease enter the error and the description of the log: (Ex:log:211 Error Bitbake)\n')
Brad Bishopc342db32019-05-15 21:57:59 -0400111 test_result.update({case['test']['@alias']: {'status': '%s' % res, 'log': '%s' % log_input}})
Brad Bishop40320b12019-03-26 16:08:25 -0400112 else:
Brad Bishopc342db32019-05-15 21:57:59 -0400113 test_result.update({case['test']['@alias']: {'status': '%s' % res}})
Brad Bishop40320b12019-03-26 16:08:25 -0400114 break
115 print('Invalid input!')
116 return test_result
117
Brad Bishopc342db32019-05-15 21:57:59 -0400118 def _get_write_dir(self):
119 return os.environ['BUILDDIR'] + '/tmp/log/manual/'
Brad Bishop40320b12019-03-26 16:08:25 -0400120
Brad Bishopc342db32019-05-15 21:57:59 -0400121 def run_test(self, case_file, config_options_file, testcase_config_file):
122 test_module = self._get_test_module(case_file)
123 cases = load_json_file(case_file)
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400124 config_options = {}
125 if config_options_file:
126 config_options = load_json_file(config_options_file)
Brad Bishopc342db32019-05-15 21:57:59 -0400127 configurations = self._get_config(config_options, test_module)
128 result_id = 'manual_%s_%s' % (test_module, configurations['STARTTIME'])
Brad Bishop40320b12019-03-26 16:08:25 -0400129 test_results = {}
Brad Bishopc342db32019-05-15 21:57:59 -0400130 if testcase_config_file:
131 test_case_config = load_json_file(testcase_config_file)
132 test_case_to_execute = test_case_config['testcases']
133 for case in copy.deepcopy(cases) :
134 if case['test']['@alias'] not in test_case_to_execute:
135 cases.remove(case)
136
137 print('\nTotal number of test cases in this test suite: %s\n' % len(cases))
138 for c in cases:
139 test_result = self._execute_test_steps(c)
Brad Bishop40320b12019-03-26 16:08:25 -0400140 test_results.update(test_result)
Brad Bishopc342db32019-05-15 21:57:59 -0400141 return configurations, result_id, self._get_write_dir(), test_results
Brad Bishop40320b12019-03-26 16:08:25 -0400142
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400143 def _get_true_false_input(self, input_message):
144 yes_list = ['Y', 'YES']
145 no_list = ['N', 'NO']
146 while True:
147 more_config_option = input(input_message).upper()
148 if more_config_option in yes_list or more_config_option in no_list:
149 break
150 print('Invalid input!')
151 if more_config_option in no_list:
152 return False
153 return True
154
Brad Bishopc342db32019-05-15 21:57:59 -0400155 def make_config_option_file(self, logger, case_file, config_options_file):
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400156 config_options = {}
157 if config_options_file:
158 config_options = load_json_file(config_options_file)
Brad Bishopc342db32019-05-15 21:57:59 -0400159 new_test_module = self._get_test_module(case_file)
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400160 print('Creating configuration options file for test module: %s' % new_test_module)
161 new_config_options = {}
162
163 while True:
164 config_name = input('\nPlease provide test configuration to create:\n').upper()
165 new_config_options[config_name] = {}
166 while True:
167 config_value = self._get_input('Configuration possible option value')
168 config_option_index = len(new_config_options[config_name]) + 1
169 new_config_options[config_name][config_option_index] = config_value
170 more_config_option = self._get_true_false_input('\nIs there more configuration option input: (Y)es/(N)o\n')
171 if not more_config_option:
172 break
173 more_config = self._get_true_false_input('\nIs there more configuration to create: (Y)es/(N)o\n')
174 if not more_config:
175 break
176
177 if new_config_options:
178 config_options[new_test_module] = new_config_options
179 if not config_options_file:
Brad Bishopc342db32019-05-15 21:57:59 -0400180 config_options_file = os.path.join(self._get_write_dir(), 'manual_config_options.json')
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400181 write_json_file(config_options_file, config_options)
182 logger.info('Configuration option file created at %s' % config_options_file)
183
Brad Bishop40320b12019-03-26 16:08:25 -0400184def manualexecution(args, logger):
185 testrunner = ManualTestRunner()
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400186 if args.make_config_options_file:
187 testrunner.make_config_option_file(logger, args.file, args.config_options_file)
188 return 0
Brad Bishopc342db32019-05-15 21:57:59 -0400189 configurations, result_id, write_dir, test_results = testrunner.run_test(args.file, args.config_options_file, args.testcase_config_file)
Brad Bishop40320b12019-03-26 16:08:25 -0400190 resultjsonhelper = OETestResultJSONHelper()
Brad Bishopc342db32019-05-15 21:57:59 -0400191 resultjsonhelper.dump_testresult_file(write_dir, configurations, result_id, test_results)
Brad Bishop40320b12019-03-26 16:08:25 -0400192 return 0
193
194def register_commands(subparsers):
195 """Register subcommands from this plugin"""
196 parser_build = subparsers.add_parser('manualexecution', help='helper script for results populating during manual test execution.',
197 description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/',
198 group='manualexecution')
199 parser_build.set_defaults(func=manualexecution)
200 parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
Brad Bishopd89cb5f2019-04-10 09:02:41 -0400201 parser_build.add_argument('-c', '--config-options-file', default='',
202 help='the config options file to import and used as available configuration option selection or make config option file')
203 parser_build.add_argument('-m', '--make-config-options-file', action='store_true',
204 help='make the configuration options file based on provided inputs')
Brad Bishopc342db32019-05-15 21:57:59 -0400205 parser_build.add_argument('-t', '--testcase-config-file', default='',
206 help='the testcase configuration file to enable user to run a selected set of test case')