code.vuplus.com Git - vuplus_bitbake/blob - lib/bb/runqueue.py

   1                 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 """
   5 BitBake 'RunQueue' implementation
   6
   7 Handles preparation and execution of a queue of tasks
   8
   9 Copyright (C) 2006  Richard Purdie
  10
  11 This program is free software; you can redistribute it and/or modify it under
  12 the terms of the GNU General Public License version 2 as published by the Free
  13 Software Foundation
  14
  15 This program is distributed in the hope that it will be useful, but WITHOUT
  16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  18
  19 You should have received a copy of the GNU General Public License along with
  20 """
  21
  22 from bb import msg, data, fetch, event, mkdirhier, utils
  23 from sets import Set
  24 import bb, os, sys
  25
  26 class TaskFailure(Exception):
  27     """Exception raised when a task in a runqueue fails"""
  28
  29     def __init__(self, fnid, fn, taskname):
  30         self.args = fnid, fn, taskname
  31
  32 class RunQueue:
  33     """
  34     BitBake Run Queue implementation
  35     """
  36     def __init__(self):
  37         self.reset_runqueue()
  38
  39     def reset_runqueue(self):
  40         self.runq_fnid = []
  41         self.runq_task = []
  42         self.runq_depends = []
  43         self.runq_revdeps = []
  44         self.runq_weight = []
  45         self.prio_map = []
  46
  47     def get_user_idstring(self, task, taskData):
  48         fn = taskData.fn_index[self.runq_fnid[task]]
  49         taskname = self.runq_task[task]
  50         return "%s, %s" % (fn, taskname)
  51
  52     def prepare_runqueue(self, cfgData, dataCache, taskData, targets):
  53         """
  54         Turn a set of taskData into a RunQueue and compute data needed
  55         to optimise the execution order.
  56         targets is list of paired values - a provider name and the task to run
  57         """
  58
  59         depends = []
  60         runq_weight1 = []
  61         runq_build = []
  62         runq_done = []
  63
  64         bb.msg.note(1, bb.msg.domain.RunQueue, "Preparing Runqueue")
  65
  66         for task in range(len(taskData.tasks_name)):
  67             fnid = taskData.tasks_fnid[task]
  68             fn = taskData.fn_index[fnid]
  69             task_deps = dataCache.task_deps[fn]
  70
  71             if fnid not in taskData.failed_fnids:
  72
  73                 depends = taskData.tasks_tdepends[task]
  74
  75                 # Resolve Depends
  76                 if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']:
  77                     taskname = task_deps['deptask'][taskData.tasks_name[task]]
  78                     for depid in taskData.depids[fnid]:
  79                         if depid in taskData.build_targets:
  80                             depdata = taskData.build_targets[depid][0]
  81                             if depdata:
  82                                 dep = taskData.fn_index[depdata]
  83                                 depends.append(taskData.gettask_id(dep, taskname))
  84
  85                 # Resolve Runtime Depends
  86                 if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']:
  87                     taskname = task_deps['rdeptask'][taskData.tasks_name[task]]
  88                     for depid in taskData.rdepids[fnid]:
  89                         if depid in taskData.run_targets:
  90                             depdata = taskData.run_targets[depid][0]
  91                             if depdata:
  92                                 dep = taskData.fn_index[depdata]
  93                                 depends.append(taskData.gettask_id(dep, taskname))
  94
  95                 def add_recursive_build(depid):
  96                     """
  97                     Add build depends of depid to depends
  98                     (if we've not see it before)
  99                     (calls itself recursively)
 100                     """
 101                     if str(depid) in dep_seen:
 102                         return
 103                     dep_seen.append(depid)
 104                     if depid in taskData.build_targets:
 105                         depdata = taskData.build_targets[depid][0]
 106                         if depdata:
 107                             dep = taskData.fn_index[depdata]
 108                             taskid = taskData.gettask_id(dep, taskname)
 109                             depends.append(taskid)
 110                             fnid = taskData.tasks_fnid[taskid]
 111                             for nextdepid in taskData.depids[fnid]:
 112                                 if nextdepid not in dep_seen:
 113                                     add_recursive_build(nextdepid)
 114                             for nextdepid in taskData.rdepids[fnid]:
 115                                 if nextdepid not in rdep_seen:
 116                                     add_recursive_run(nextdepid)
 117
 118                 def add_recursive_run(rdepid):
 119                     """
 120                     Add runtime depends of rdepid to depends
 121                     (if we've not see it before)
 122                     (calls itself recursively)
 123                     """
 124                     if str(rdepid) in rdep_seen:
 125                         return
 126                     rdep_seen.append(rdepid)
 127                     if rdepid in taskData.run_targets:
 128                         depdata = taskData.run_targets[rdepid][0]
 129                         if depdata:
 130                             dep = taskData.fn_index[depdata]
 131                             taskid = taskData.gettask_id(dep, taskname)
 132                             depends.append(taskid)
 133                             fnid = taskData.tasks_fnid[taskid]
 134                             for nextdepid in taskData.depids[fnid]:
 135                                 if nextdepid not in dep_seen:
 136                                     add_recursive_build(nextdepid)
 137                             for nextdepid in taskData.rdepids[fnid]:
 138                                 if nextdepid not in rdep_seen:
 139                                     add_recursive_run(nextdepid)
 140
 141
 142                 # Resolve Recursive Runtime Depends
 143                 # Also includes all Build Depends (and their runtime depends)
 144                 if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']:
 145                     dep_seen = []
 146                     rdep_seen = []
 147                     taskname = task_deps['recrdeptask'][taskData.tasks_name[task]]
 148                     for depid in taskData.depids[fnid]:
 149                         add_recursive_build(depid)
 150                     for rdepid in taskData.rdepids[fnid]:
 151                         add_recursive_run(rdepid)
 152
 153                 #Prune self references
 154                 if task in depends:
 155                     newdep = []
 156                     bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends))
 157                     for dep in depends:
 158                        if task != dep:
 159                           newdep.append(dep)
 160                     depends = newdep
 161
 162
 163             self.runq_fnid.append(taskData.tasks_fnid[task])
 164             self.runq_task.append(taskData.tasks_name[task])
 165             self.runq_depends.append(Set(depends))
 166             self.runq_revdeps.append(Set())
 167             self.runq_weight.append(0)
 168
 169             runq_weight1.append(0)
 170             runq_build.append(0)
 171             runq_done.append(0)
 172
 173         bb.msg.note(2, bb.msg.domain.RunQueue, "Marking Active Tasks")
 174
 175         def mark_active(listid, depth):
 176             """
 177             Mark an item as active along with its depends
 178             (calls itself recursively)
 179             """
 180
 181             if runq_build[listid] == 1:
 182                 return
 183
 184             runq_build[listid] = 1
 185
 186             depends = self.runq_depends[listid]
 187             for depend in depends:
 188                 mark_active(depend, depth+1)
 189
 190         for target in targets:
 191             targetid = taskData.getbuild_id(target[0])
 192             if targetid in taskData.failed_deps:
 193                 continue
 194
 195             if targetid not in taskData.build_targets:
 196                 continue
 197
 198             fnid = taskData.build_targets[targetid][0]
 199             if fnid in taskData.failed_fnids:
 200                 continue
 201
 202             fnids = taskData.matches_in_list(self.runq_fnid, fnid)
 203             tasks = taskData.matches_in_list(self.runq_task, target[1])
 204
 205             listid = taskData.both_contain(fnids, tasks)
 206
 207             mark_active(listid, 1)
 208
 209         # Prune inactive tasks
 210         maps = []
 211         delcount = 0
 212         for listid in range(len(self.runq_fnid)):
 213             if runq_build[listid-delcount] == 1:
 214                 maps.append(listid-delcount)
 215             else:
 216                 del self.runq_fnid[listid-delcount]
 217                 del self.runq_task[listid-delcount]
 218                 del self.runq_depends[listid-delcount]
 219                 del self.runq_weight[listid-delcount]
 220                 del runq_weight1[listid-delcount]
 221                 del runq_build[listid-delcount]
 222                 del runq_done[listid-delcount]
 223                 del self.runq_revdeps[listid-delcount]
 224                 delcount = delcount + 1
 225                 maps.append(-1)
 226
 227         if len(self.runq_fnid) == 0:
 228             if not taskData.abort:
 229                 bb.msg.note(1, bb.msg.domain.RunQueue, "All possible tasks have been run but build incomplete (--continue mode). See errors above for incomplete tasks.")
 230                 return
 231             bb.msg.fatal(bb.msg.domain.RunQueue, "No active tasks and not in --continue mode?! Please report this bug.")
 232
 233         bb.msg.note(2, bb.msg.domain.RunQueue, "Pruned %s inactive tasks, %s left" % (delcount, len(self.runq_fnid)))
 234
 235         for listid in range(len(self.runq_fnid)):
 236             newdeps = []
 237             origdeps = self.runq_depends[listid]
 238             for origdep in origdeps:
 239                 if maps[origdep] == -1:
 240                     bb.msg.fatal(bb.msg.domain.RunQueue, "Invalid mapping - Should never happen!")
 241                 newdeps.append(maps[origdep])
 242             self.runq_depends[listid] = Set(newdeps)
 243
 244         bb.msg.note(2, bb.msg.domain.RunQueue, "Assign Weightings")
 245
 246         for listid in range(len(self.runq_fnid)):
 247             for dep in self.runq_depends[listid]:
 248                 self.runq_revdeps[dep].add(listid)
 249
 250         endpoints = []
 251         for listid in range(len(self.runq_fnid)):
 252             revdeps = self.runq_revdeps[listid]
 253             if len(revdeps) == 0:
 254                 runq_done[listid] = 1
 255                 self.runq_weight[listid] = 1
 256                 endpoints.append(listid)
 257             for dep in revdeps:
 258                 if dep in self.runq_depends[listid]:
 259                     #self.dump_data(taskData)
 260                     bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep] , taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid]))
 261             runq_weight1[listid] = len(revdeps)
 262
 263         bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints))
 264
 265         while 1:
 266             next_points = []
 267             for listid in endpoints:
 268                 for revdep in self.runq_depends[listid]:
 269                     self.runq_weight[revdep] = self.runq_weight[revdep] + self.runq_weight[listid]
 270                     runq_weight1[revdep] = runq_weight1[revdep] - 1
 271                     if runq_weight1[revdep] == 0:
 272                         next_points.append(revdep)
 273                         runq_done[revdep] = 1
 274             endpoints = next_points
 275             if len(next_points) == 0:
 276                 break
 277
 278         # Sanity Checks
 279         for task in range(len(self.runq_fnid)):
 280             if runq_done[task] == 0:
 281                 seen = []
 282                 def print_chain(taskid):
 283                     seen.append(taskid)
 284                     for revdep in self.runq_revdeps[taskid]:
 285                         if runq_done[revdep] == 0:
 286                             bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) (depends: %s)" % (revdep, self.get_user_idstring(revdep, taskData), self.runq_depends[revdep]))
 287                             if revdep not in seen:
 288                                 print_chain(revdep)
 289                 print_chain(task)
 290                 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) not processed!\nThis is probably a circular dependency (the chain might be printed above)." % (task, self.get_user_idstring(task, taskData)))
 291             if runq_weight1[task] != 0:
 292                 bb.msg.fatal(bb.msg.domain.RunQueue, "Task %s (%s) count not zero!" % (task, self.get_user_idstring(task, taskData)))
 293
 294         # Make a weight sorted map
 295         from copy import deepcopy
 296
 297         sortweight = deepcopy(self.runq_weight)
 298         sortweight.sort()
 299         copyweight = deepcopy(self.runq_weight)
 300         self.prio_map = []
 301
 302         for weight in sortweight:
 303             idx = copyweight.index(weight)
 304             self.prio_map.append(idx)
 305             copyweight[idx] = -1
 306         self.prio_map.reverse()
 307
 308     def execute_runqueue(self, cooker, cfgData, dataCache, taskData, runlist):
 309         """
 310         Run the tasks in a queue prepared by prepare_runqueue
 311         Upon failure, optionally try to recover the build using any alternate providers
 312         (if the abort on failure configuration option isn't set)
 313         """
 314
 315         failures = 0
 316         while 1:
 317             try:
 318                 self.execute_runqueue_internal(cooker, cfgData, dataCache, taskData)
 319                 return failures
 320             except bb.runqueue.TaskFailure, (fnid, taskData.fn_index[fnid], taskname):
 321                 if taskData.abort:
 322                     raise
 323                 taskData.fail_fnid(fnid)
 324                 self.reset_runqueue()
 325                 self.prepare_runqueue(cfgData, dataCache, taskData, runlist)
 326                 failures = failures + 1
 327
 328     def execute_runqueue_internal(self, cooker, cfgData, dataCache, taskData):
 329         """
 330         Run the tasks in a queue prepared by prepare_runqueue
 331         """
 332
 333         bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
 334
 335         runq_buildable = []
 336         runq_running = []
 337         runq_complete = []
 338         active_builds = 0
 339         build_pids = {}
 340
 341         if len(self.runq_fnid) == 0:
 342             # nothing to do
 343             return
 344
 345         def get_next_task(data):
 346             """
 347             Return the id of the highest priority task that is buildable
 348             """
 349             for task1 in range(len(data.runq_fnid)):
 350                 task = data.prio_map[task1]
 351                 if runq_running[task] == 1:
 352                     continue
 353                 if runq_buildable[task] == 1:
 354                     return task
 355             return None
 356
 357         def task_complete(data, task):
 358             """
 359             Mark a task as completed
 360             Look at the reverse dependencies and mark any task with
 361             completed dependencies as buildable
 362             """
 363             runq_complete[task] = 1
 364             for revdep in data.runq_revdeps[task]:
 365                 if runq_running[revdep] == 1:
 366                     continue
 367                 if runq_buildable[revdep] == 1:
 368                     continue
 369                 alldeps = 1
 370                 for dep in data.runq_depends[revdep]:
 371                     if runq_complete[dep] != 1:
 372                         alldeps = 0
 373                 if alldeps == 1:
 374                     runq_buildable[revdep] = 1
 375                     fn = taskData.fn_index[self.runq_fnid[revdep]]
 376                     taskname = self.runq_task[revdep]
 377                     bb.msg.debug(1, bb.msg.domain.RunQueue, "Marking task %s (%s, %s) as buildable" % (revdep, fn, taskname))
 378
 379         # Mark initial buildable tasks
 380         for task in range(len(self.runq_fnid)):
 381             runq_running.append(0)
 382             runq_complete.append(0)
 383             if len(self.runq_depends[task]) == 0:
 384                 runq_buildable.append(1)
 385             else:
 386                 runq_buildable.append(0)
 387
 388
 389         number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1)
 390
 391         try:
 392             while 1:
 393                 task = get_next_task(self)
 394                 if task is not None:
 395                     fn = taskData.fn_index[self.runq_fnid[task]]
 396                     taskname = self.runq_task[task]
 397
 398                     if bb.build.stamp_is_current_cache(dataCache, fn, taskname):
 399                         targetid = taskData.gettask_id(fn, taskname)
 400                         if not (targetid in taskData.external_targets and cooker.configuration.force):
 401                             bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task, taskData)))
 402                             runq_running[task] = 1
 403                             task_complete(self, task)
 404                             continue
 405
 406                     bb.msg.debug(1, bb.msg.domain.RunQueue, "Running task %s (%s)" % (task, self.get_user_idstring(task, taskData)))
 407                     try:
 408                         pid = os.fork()
 409                     except OSError, e:
 410                         bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror))
 411                     if pid == 0:
 412                         cooker.configuration.cmd = taskname[3:]
 413                         try:
 414                             cooker.tryBuild(fn, False)
 415                         except bb.build.EventException:
 416                             bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
 417                             sys.exit(1)
 418                         except:
 419                             bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
 420                             raise
 421                         sys.exit(0)
 422                     build_pids[pid] = task
 423                     runq_running[task] = 1
 424                     active_builds = active_builds + 1
 425                     if active_builds < number_tasks:
 426                         continue
 427                 if active_builds > 0:
 428                     result = os.waitpid(-1, 0)
 429                     active_builds = active_builds - 1
 430                     if result[1] != 0:
 431                         bb.msg.error(bb.msg.domain.RunQueue, "Task %s (%s) failed" % (build_pids[result[0]], self.get_user_idstring(build_pids[result[0]], taskData)))
 432                         raise bb.runqueue.TaskFailure(self.runq_fnid[build_pids[result[0]]], taskData.fn_index[self.runq_fnid[build_pids[result[0]]]], self.runq_task[build_pids[result[0]]])
 433                     task_complete(self, build_pids[result[0]])
 434                     del build_pids[result[0]]
 435                     continue
 436                 break
 437         except SystemExit:
 438             raise
 439         except:
 440             bb.msg.error(bb.msg.domain.RunQueue, "Exception received")
 441             if active_builds > 0:
 442                 while active_builds > 0:
 443                     bb.msg.note(1, bb.msg.domain.RunQueue, "Waiting for %s active tasks to finish" % active_builds)
 444                     tasknum = 1
 445                     for k, v in build_pids.iteritems():
 446                         bb.msg.note(1, bb.msg.domain.RunQueue, "%s: %s (%s)" % (tasknum, self.get_user_idstring(v, taskData), k))
 447                         tasknum = tasknum + 1
 448                     result = os.waitpid(-1, 0)
 449                     del build_pids[result[0]]
 450                     active_builds = active_builds - 1
 451             raise
 452
 453         # Sanity Checks
 454         for task in range(len(self.runq_fnid)):
 455             if runq_buildable[task] == 0:
 456                 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never buildable!" % task)
 457             if runq_running[task] == 0:
 458                 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never ran!" % task)
 459             if runq_complete[task] == 0:
 460                 bb.msg.error(bb.msg.domain.RunQueue, "Task %s never completed!" % task)
 461
 462         return 0
 463
 464     def dump_data(self, taskQueue):
 465         """
 466         Dump some debug information on the internal data structures
 467         """
 468         bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:")
 469         for task in range(len(self.runq_fnid)):
 470                 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task,
 471                         taskQueue.fn_index[self.runq_fnid[task]],
 472                         self.runq_task[task],
 473                         self.runq_weight[task],
 474                         self.runq_depends[task],
 475                         self.runq_revdeps[task]))
 476
 477         bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:")
 478         for task1 in range(len(self.runq_fnid)):
 479             if task1 in self.prio_map:
 480                 task = self.prio_map[task1]
 481                 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s   Deps %s RevDeps %s" % (task,
 482                         taskQueue.fn_index[self.runq_fnid[task]],
 483                         self.runq_task[task],
 484                         self.runq_weight[task],
 485                         self.runq_depends[task],
 486                         self.runq_revdeps[task]))