Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1 | def tinder_http_post(server, selector, content_type, body): |
| 2 | import httplib |
| 3 | # now post it |
| 4 | for i in range(0,5): |
| 5 | try: |
| 6 | h = httplib.HTTP(server) |
| 7 | h.putrequest('POST', selector) |
| 8 | h.putheader('content-type', content_type) |
| 9 | h.putheader('content-length', str(len(body))) |
| 10 | h.endheaders() |
| 11 | h.send(body) |
| 12 | errcode, errmsg, headers = h.getreply() |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 13 | #print(errcode, errmsg, headers) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 14 | return (errcode,errmsg, headers, h.file) |
| 15 | except: |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 16 | print("Error sending the report!") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 17 | # try again |
| 18 | pass |
| 19 | |
| 20 | # return some garbage |
| 21 | return (-1, "unknown", "unknown", None) |
| 22 | |
| 23 | def tinder_form_data(bound, dict, log): |
| 24 | output = [] |
| 25 | # for each key in the dictionary |
| 26 | for name in dict: |
| 27 | assert dict[name] |
| 28 | output.append( "--" + bound ) |
| 29 | output.append( 'Content-Disposition: form-data; name="%s"' % name ) |
| 30 | output.append( "" ) |
| 31 | output.append( dict[name] ) |
| 32 | if log: |
| 33 | output.append( "--" + bound ) |
| 34 | output.append( 'Content-Disposition: form-data; name="log"; filename="log.txt"' ) |
| 35 | output.append( '' ) |
| 36 | output.append( log ) |
| 37 | output.append( '--' + bound + '--' ) |
| 38 | output.append( '' ) |
| 39 | |
| 40 | return "\r\n".join(output) |
| 41 | |
| 42 | def tinder_time_string(): |
| 43 | """ |
| 44 | Return the time as GMT |
| 45 | """ |
| 46 | return "" |
| 47 | |
| 48 | def tinder_format_http_post(d,status,log): |
| 49 | """ |
| 50 | Format the Tinderbox HTTP post with the data needed |
| 51 | for the tinderbox to be happy. |
| 52 | """ |
| 53 | |
| 54 | import random |
| 55 | |
| 56 | # the variables we will need to send on this form post |
| 57 | variables = { |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 58 | "tree" : d.getVar('TINDER_TREE'), |
| 59 | "machine_name" : d.getVar('TINDER_MACHINE'), |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 60 | "os" : os.uname()[0], |
| 61 | "os_version" : os.uname()[2], |
| 62 | "compiler" : "gcc", |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 63 | "clobber" : d.getVar('TINDER_CLOBBER') or "0", |
| 64 | "srcdate" : d.getVar('SRCDATE'), |
| 65 | "PN" : d.getVar('PN'), |
| 66 | "PV" : d.getVar('PV'), |
| 67 | "PR" : d.getVar('PR'), |
| 68 | "FILE" : d.getVar('FILE') or "N/A", |
| 69 | "TARGETARCH" : d.getVar('TARGET_ARCH'), |
| 70 | "TARGETFPU" : d.getVar('TARGET_FPU') or "Unknown", |
| 71 | "TARGETOS" : d.getVar('TARGET_OS') or "Unknown", |
| 72 | "MACHINE" : d.getVar('MACHINE') or "Unknown", |
| 73 | "DISTRO" : d.getVar('DISTRO') or "Unknown", |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 74 | "zecke-rocks" : "sure", |
| 75 | } |
| 76 | |
| 77 | # optionally add the status |
| 78 | if status: |
| 79 | variables["status"] = str(status) |
| 80 | |
| 81 | # try to load the machine id |
| 82 | # we only need on build_status.pl but sending it |
| 83 | # always does not hurt |
| 84 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 85 | f = open(d.getVar('TMPDIR')+'/tinder-machine.id', 'r') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 86 | id = f.read() |
| 87 | variables['machine_id'] = id |
| 88 | except: |
| 89 | pass |
| 90 | |
| 91 | # the boundary we will need |
| 92 | boundary = "----------------------------------%d" % int(random.random()*1000000000000) |
| 93 | |
| 94 | # now format the body |
| 95 | body = tinder_form_data( boundary, variables, log ) |
| 96 | |
| 97 | return ("multipart/form-data; boundary=%s" % boundary),body |
| 98 | |
| 99 | |
| 100 | def tinder_build_start(d): |
| 101 | """ |
| 102 | Inform the tinderbox that a build is starting. We do this |
| 103 | by posting our name and tree to the build_start.pl script |
| 104 | on the server. |
| 105 | """ |
| 106 | |
| 107 | # get the body and type |
| 108 | content_type, body = tinder_format_http_post(d,None,None) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 109 | server = d.getVar('TINDER_HOST') |
| 110 | url = d.getVar('TINDER_URL') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 111 | |
| 112 | selector = url + "/xml/build_start.pl" |
| 113 | |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 114 | #print("selector %s and url %s" % (selector, url)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 115 | |
| 116 | # now post it |
| 117 | errcode, errmsg, headers, h_file = tinder_http_post(server,selector,content_type, body) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 118 | #print(errcode, errmsg, headers) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 119 | report = h_file.read() |
| 120 | |
| 121 | # now let us find the machine id that was assigned to us |
| 122 | search = "<machine id='" |
| 123 | report = report[report.find(search)+len(search):] |
| 124 | report = report[0:report.find("'")] |
| 125 | |
| 126 | bb.note("Machine ID assigned by tinderbox: %s" % report ) |
| 127 | |
| 128 | # now we will need to save the machine number |
| 129 | # we will override any previous numbers |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 130 | f = open(d.getVar('TMPDIR')+"/tinder-machine.id", 'w') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 131 | f.write(report) |
| 132 | |
| 133 | |
| 134 | def tinder_send_http(d, status, _log): |
| 135 | """ |
| 136 | Send this log as build status |
| 137 | """ |
| 138 | |
| 139 | # get the body and type |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 140 | server = d.getVar('TINDER_HOST') |
| 141 | url = d.getVar('TINDER_URL') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 142 | |
| 143 | selector = url + "/xml/build_status.pl" |
| 144 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 145 | # now post it - in chunks of 10.000 characters |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 146 | new_log = _log |
| 147 | while len(new_log) > 0: |
| 148 | content_type, body = tinder_format_http_post(d,status,new_log[0:18000]) |
| 149 | errcode, errmsg, headers, h_file = tinder_http_post(server,selector,content_type, body) |
Patrick Williams | c0f7c04 | 2017-02-23 20:41:17 -0600 | [diff] [blame] | 150 | #print(errcode, errmsg, headers) |
| 151 | #print(h.file.read()) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 152 | new_log = new_log[18000:] |
| 153 | |
| 154 | |
| 155 | def tinder_print_info(d): |
| 156 | """ |
| 157 | Print the TinderBox Info |
| 158 | Including informations of the BaseSystem and the Tree |
| 159 | we use. |
| 160 | """ |
| 161 | |
| 162 | # get the local vars |
| 163 | time = tinder_time_string() |
| 164 | ops = os.uname()[0] |
| 165 | version = os.uname()[2] |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 166 | url = d.getVar('TINDER_URL') |
| 167 | tree = d.getVar('TINDER_TREE') |
| 168 | branch = d.getVar('TINDER_BRANCH') |
| 169 | srcdate = d.getVar('SRCDATE') |
| 170 | machine = d.getVar('MACHINE') |
| 171 | distro = d.getVar('DISTRO') |
| 172 | bbfiles = d.getVar('BBFILES') |
| 173 | tarch = d.getVar('TARGET_ARCH') |
| 174 | fpu = d.getVar('TARGET_FPU') |
| 175 | oerev = d.getVar('OE_REVISION') or "unknown" |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 176 | |
| 177 | # there is a bug with tipple quoted strings |
| 178 | # i will work around but will fix the original |
| 179 | # bug as well |
| 180 | output = [] |
| 181 | output.append("== Tinderbox Info" ) |
| 182 | output.append("Time: %(time)s" ) |
| 183 | output.append("OS: %(ops)s" ) |
| 184 | output.append("%(version)s" ) |
| 185 | output.append("Compiler: gcc" ) |
| 186 | output.append("Tinderbox Client: 0.1" ) |
| 187 | output.append("Tinderbox Client Last Modified: yesterday" ) |
| 188 | output.append("Tinderbox Protocol: 0.1" ) |
| 189 | output.append("URL: %(url)s" ) |
| 190 | output.append("Tree: %(tree)s" ) |
| 191 | output.append("Config:" ) |
| 192 | output.append("branch = '%(branch)s'" ) |
| 193 | output.append("TARGET_ARCH = '%(tarch)s'" ) |
| 194 | output.append("TARGET_FPU = '%(fpu)s'" ) |
| 195 | output.append("SRCDATE = '%(srcdate)s'" ) |
| 196 | output.append("MACHINE = '%(machine)s'" ) |
| 197 | output.append("DISTRO = '%(distro)s'" ) |
| 198 | output.append("BBFILES = '%(bbfiles)s'" ) |
| 199 | output.append("OEREV = '%(oerev)s'" ) |
| 200 | output.append("== End Tinderbox Client Info" ) |
| 201 | |
| 202 | # now create the real output |
| 203 | return "\n".join(output) % vars() |
| 204 | |
| 205 | |
| 206 | def tinder_print_env(): |
| 207 | """ |
| 208 | Print the environment variables of this build |
| 209 | """ |
| 210 | time_start = tinder_time_string() |
| 211 | time_end = tinder_time_string() |
| 212 | |
| 213 | # build the environment |
| 214 | env = "" |
| 215 | for var in os.environ: |
| 216 | env += "%s=%s\n" % (var, os.environ[var]) |
| 217 | |
| 218 | output = [] |
| 219 | output.append( "---> TINDERBOX RUNNING env %(time_start)s" ) |
| 220 | output.append( env ) |
| 221 | output.append( "<--- TINDERBOX FINISHED (SUCCESS) %(time_end)s" ) |
| 222 | |
| 223 | return "\n".join(output) % vars() |
| 224 | |
| 225 | def tinder_tinder_start(d, event): |
| 226 | """ |
| 227 | PRINT the configuration of this build |
| 228 | """ |
| 229 | |
| 230 | time_start = tinder_time_string() |
| 231 | config = tinder_print_info(d) |
| 232 | #env = tinder_print_env() |
| 233 | time_end = tinder_time_string() |
| 234 | packages = " ".join( event.getPkgs() ) |
| 235 | |
| 236 | output = [] |
| 237 | output.append( "---> TINDERBOX PRINTING CONFIGURATION %(time_start)s" ) |
| 238 | output.append( config ) |
| 239 | #output.append( env ) |
| 240 | output.append( "<--- TINDERBOX FINISHED PRINTING CONFIGURATION %(time_end)s" ) |
| 241 | output.append( "---> TINDERBOX BUILDING '%(packages)s'" ) |
| 242 | output.append( "<--- TINDERBOX STARTING BUILD NOW" ) |
| 243 | |
| 244 | output.append( "" ) |
| 245 | |
| 246 | return "\n".join(output) % vars() |
| 247 | |
| 248 | def tinder_do_tinder_report(event): |
| 249 | """ |
| 250 | Report to the tinderbox: |
| 251 | On the BuildStart we will inform the box directly |
| 252 | On the other events we will write to the TINDER_LOG and |
| 253 | when the Task is finished we will send the report. |
| 254 | |
| 255 | The above is not yet fully implemented. Currently we send |
| 256 | information immediately. The caching/queuing needs to be |
| 257 | implemented. Also sending more or less information is not |
| 258 | implemented yet. |
| 259 | |
| 260 | We have two temporary files stored in the TMP directory. One file |
| 261 | contains the assigned machine id for the tinderclient. This id gets |
| 262 | assigned when we connect the box and start the build process the second |
| 263 | file is used to workaround an EventHandler limitation. If BitBake is ran |
| 264 | with the continue option we want the Build to fail even if we get the |
| 265 | BuildCompleted Event. In this case we have to look up the status and |
| 266 | send it instead of 100/success. |
| 267 | """ |
| 268 | import glob |
| 269 | |
| 270 | # variables |
| 271 | name = bb.event.getName(event) |
| 272 | log = "" |
| 273 | status = 1 |
| 274 | # Check what we need to do Build* shows we start or are done |
| 275 | if name == "BuildStarted": |
| 276 | tinder_build_start(event.data) |
| 277 | log = tinder_tinder_start(event.data,event) |
| 278 | |
| 279 | try: |
| 280 | # truncate the tinder log file |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 281 | f = open(event.data.getVar('TINDER_LOG'), 'w') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 282 | f.write("") |
| 283 | f.close() |
| 284 | except: |
| 285 | pass |
| 286 | |
| 287 | try: |
| 288 | # write a status to the file. This is needed for the -k option |
| 289 | # of BitBake |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 290 | g = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 291 | g.write("") |
| 292 | g.close() |
| 293 | except IOError: |
| 294 | pass |
| 295 | |
| 296 | # Append the Task-Log (compile,configure...) to the log file |
| 297 | # we will send to the server |
| 298 | if name == "TaskSucceeded" or name == "TaskFailed": |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 299 | log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T'), event.task)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 300 | |
| 301 | if len(log_file) != 0: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 302 | to_file = event.data.getVar('TINDER_LOG') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 303 | log += "".join(open(log_file[0], 'r').readlines()) |
| 304 | |
| 305 | # set the right 'HEADER'/Summary for the TinderBox |
| 306 | if name == "TaskStarted": |
| 307 | log += "---> TINDERBOX Task %s started\n" % event.task |
| 308 | elif name == "TaskSucceeded": |
| 309 | log += "<--- TINDERBOX Task %s done (SUCCESS)\n" % event.task |
| 310 | elif name == "TaskFailed": |
| 311 | log += "<--- TINDERBOX Task %s failed (FAILURE)\n" % event.task |
| 312 | elif name == "PkgStarted": |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 313 | log += "---> TINDERBOX Package %s started\n" % event.data.getVar('PF') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 314 | elif name == "PkgSucceeded": |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 315 | log += "<--- TINDERBOX Package %s done (SUCCESS)\n" % event.data.getVar('PF') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 316 | elif name == "PkgFailed": |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 317 | if not event.data.getVar('TINDER_AUTOBUILD') == "0": |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 318 | build.exec_task('do_clean', event.data) |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 319 | log += "<--- TINDERBOX Package %s failed (FAILURE)\n" % event.data.getVar('PF') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 320 | status = 200 |
| 321 | # remember the failure for the -k case |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 322 | h = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 323 | h.write("200") |
| 324 | elif name == "BuildCompleted": |
| 325 | log += "Build Completed\n" |
| 326 | status = 100 |
| 327 | # Check if we have a old status... |
| 328 | try: |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 329 | h = open(event.data.getVar('TMPDIR')+'/tinder-status', 'r') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 330 | status = int(h.read()) |
| 331 | except: |
| 332 | pass |
| 333 | |
| 334 | elif name == "MultipleProviders": |
| 335 | log += "---> TINDERBOX Multiple Providers\n" |
| 336 | log += "multiple providers are available (%s);\n" % ", ".join(event.getCandidates()) |
| 337 | log += "consider defining PREFERRED_PROVIDER_%s\n" % event.getItem() |
| 338 | log += "is runtime: %d\n" % event.isRuntime() |
| 339 | log += "<--- TINDERBOX Multiple Providers\n" |
| 340 | elif name == "NoProvider": |
| 341 | log += "Error: No Provider for: %s\n" % event.getItem() |
| 342 | log += "Error:Was Runtime: %d\n" % event.isRuntime() |
| 343 | status = 200 |
| 344 | # remember the failure for the -k case |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 345 | h = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 346 | h.write("200") |
| 347 | |
| 348 | # now post the log |
| 349 | if len(log) == 0: |
| 350 | return |
| 351 | |
| 352 | # for now we will use the http post method as it is the only one |
| 353 | log_post_method = tinder_send_http |
| 354 | log_post_method(event.data, status, log) |
| 355 | |
| 356 | |
| 357 | # we want to be an event handler |
| 358 | addhandler tinderclient_eventhandler |
| 359 | python tinderclient_eventhandler() { |
| 360 | if e.data is None or bb.event.getName(e) == "MsgNote": |
| 361 | return |
| 362 | |
Brad Bishop | 6e60e8b | 2018-02-01 10:27:11 -0500 | [diff] [blame^] | 363 | do_tinder_report = e.data.getVar('TINDER_REPORT') |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 364 | if do_tinder_report and do_tinder_report == "1": |
| 365 | tinder_do_tinder_report(e) |
| 366 | |
| 367 | return |
| 368 | } |