code.vuplus.com Git - vuplus_bitbake/blob - lib/bb/taskdata.py

   1 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 """
   5 BitBake 'TaskData' implementation
   6
   7 Task data collection and handling
   8
   9 Copyright (C) 2006  Richard Purdie
  10
  11 This program is free software; you can redistribute it and/or modify it under
  12 the terms of the GNU General Public License version 2 as published by the Free
  13 Software Foundation
  14
  15 This program is distributed in the hope that it will be useful, but WITHOUT
  16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  18
  19 You should have received a copy of the GNU General Public License along with
  20 """
  21
  22 from bb import data, fetch, event, mkdirhier, utils
  23 import bb, os
  24
  25 class TaskData:
  26     """
  27     BitBake Task Data implementation
  28     """
  29     def __init__(self, abort = True):
  30         self.build_names_index = []
  31         self.run_names_index = []
  32         self.fn_index = []
  33
  34         self.build_targets = {}
  35         self.run_targets = {}
  36
  37         self.external_targets = []
  38
  39         self.tasks_fnid = []
  40         self.tasks_name = []
  41         self.tasks_tdepends = []
  42         # Cache to speed up task ID lookups
  43         self.tasks_lookup = {}
  44
  45         self.depids = {}
  46         self.rdepids = {}
  47
  48         self.consider_msgs_cache = []
  49
  50         self.failed_deps = []
  51         self.failed_rdeps = []
  52         self.failed_fnids = []
  53
  54         self.abort = abort
  55
  56     def getbuild_id(self, name):
  57         """
  58         Return an ID number for the build target name.
  59         If it doesn't exist, create one.
  60         """
  61         if not name in self.build_names_index:
  62             self.build_names_index.append(name)
  63
  64         return self.build_names_index.index(name)
  65
  66     def getrun_id(self, name):
  67         """
  68         Return an ID number for the run target name.
  69         If it doesn't exist, create one.
  70         """
  71         if not name in self.run_names_index:
  72             self.run_names_index.append(name)
  73
  74         return self.run_names_index.index(name)
  75
  76     def getfn_id(self, name):
  77         """
  78         Return an ID number for the filename.
  79         If it doesn't exist, create one.
  80         """
  81         if not name in self.fn_index:
  82             self.fn_index.append(name)
  83
  84         return self.fn_index.index(name)
  85
  86     def gettask_id(self, fn, task):
  87         """
  88         Return an ID number for the task matching fn and task.
  89         If it doesn't exist, create one.
  90         """
  91         fnid = self.getfn_id(fn)
  92
  93         if fnid in self.tasks_lookup:
  94             if task in self.tasks_lookup[fnid]:
  95                 return self.tasks_lookup[fnid][task]
  96
  97         self.tasks_name.append(task)
  98         self.tasks_fnid.append(fnid)
  99         self.tasks_tdepends.append([])
 100
 101         listid = len(self.tasks_name) - 1
 102
 103         if fnid not in self.tasks_lookup:
 104             self.tasks_lookup[fnid] = {}
 105         self.tasks_lookup[fnid][task] = listid
 106
 107         return listid
 108
 109     def add_tasks(self, fn, dataCache):
 110         """
 111         Add tasks for a given fn to the database
 112         """
 113
 114         task_graph = dataCache.task_queues[fn]
 115         task_deps = dataCache.task_deps[fn]
 116
 117         fnid = self.getfn_id(fn)
 118
 119         if fnid in self.failed_fnids:
 120             bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
 121
 122         # Check if we've already seen this fn
 123         if fnid in self.tasks_fnid:
 124             return
 125
 126         # Work out task dependencies
 127         for task in task_graph.allnodes():
 128             parentids = []
 129             for dep in task_graph.getparents(task):
 130                 parentid = self.gettask_id(fn, dep)
 131                 parentids.append(parentid)
 132             taskid = self.gettask_id(fn, task)
 133             self.tasks_tdepends[taskid].extend(parentids)
 134
 135         # Work out build dependencies
 136         if not fnid in self.depids:
 137             dependids = {}
 138             for depend in dataCache.deps[fn]:
 139                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
 140                 dependids[self.getbuild_id(depend)] = None
 141             self.depids[fnid] = dependids.keys()
 142
 143         # Work out runtime dependencies
 144         if not fnid in self.rdepids:
 145             rdependids = {}
 146             rdepends = dataCache.rundeps[fn]
 147             rrecs = dataCache.runrecs[fn]
 148             for package in rdepends:
 149                 for rdepend in rdepends[package]:
 150                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
 151                     rdependids[self.getrun_id(rdepend)] = None
 152             for package in rrecs:
 153                 for rdepend in rrecs[package]:
 154                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
 155                     rdependids[self.getrun_id(rdepend)] = None
 156             self.rdepids[fnid] = rdependids.keys()
 157
 158         for dep in self.depids[fnid]:
 159             if dep in self.failed_deps:
 160                 self.fail_fnid(fnid)
 161                 return
 162         for dep in self.rdepids[fnid]:
 163             if dep in self.failed_rdeps:
 164                 self.fail_fnid(fnid)
 165                 return
 166
 167     def have_build_target(self, target):
 168         """
 169         Have we a build target matching this name?
 170         """
 171         targetid = self.getbuild_id(target)
 172
 173         if targetid in self.build_targets:
 174             return True
 175         return False
 176
 177     def have_runtime_target(self, target):
 178         """
 179         Have we a runtime target matching this name?
 180         """
 181         targetid = self.getrun_id(target)
 182
 183         if targetid in self.run_targets:
 184             return True
 185         return False
 186
 187     def add_build_target(self, fn, item):
 188         """
 189         Add a build target.
 190         If already present, append the provider fn to the list
 191         """
 192         targetid = self.getbuild_id(item)
 193         fnid = self.getfn_id(fn)
 194
 195         if targetid in self.build_targets:
 196             if fnid in self.build_targets[targetid]:
 197                 return
 198             self.build_targets[targetid].append(fnid)
 199             return
 200         self.build_targets[targetid] = [fnid]
 201
 202     def add_runtime_target(self, fn, item):
 203         """
 204         Add a runtime target.
 205         If already present, append the provider fn to the list
 206         """
 207         targetid = self.getrun_id(item)
 208         fnid = self.getfn_id(fn)
 209
 210         if targetid in self.run_targets:
 211             if fnid in self.run_targets[targetid]:
 212                 return
 213             self.run_targets[targetid].append(fnid)
 214             return
 215         self.run_targets[targetid] = [fnid]
 216
 217     def mark_external_target(self, item):
 218         """
 219         Mark a build target as being externally requested
 220         """
 221         targetid = self.getbuild_id(item)
 222
 223         if targetid not in self.external_targets:
 224             self.external_targets.append(targetid)
 225
 226     def get_unresolved_build_targets(self, dataCache):
 227         """
 228         Return a list of build targets who's providers
 229         are unknown.
 230         """
 231         unresolved = []
 232         for target in self.build_names_index:
 233             if target in dataCache.ignored_dependencies:
 234                 continue
 235             if self.build_names_index.index(target) in self.failed_deps:
 236                 continue
 237             if not self.have_build_target(target):
 238                 unresolved.append(target)
 239         return unresolved
 240
 241     def get_unresolved_run_targets(self, dataCache):
 242         """
 243         Return a list of runtime targets who's providers
 244         are unknown.
 245         """
 246         unresolved = []
 247         for target in self.run_names_index:
 248             if target in dataCache.ignored_dependencies:
 249                 continue
 250             if self.run_names_index.index(target) in self.failed_rdeps:
 251                 continue
 252             if not self.have_runtime_target(target):
 253                 unresolved.append(target)
 254         return unresolved
 255
 256     def get_provider(self, item):
 257         """
 258         Return a list of providers of item
 259         """
 260         targetid = self.getbuild_id(item)
 261
 262         return self.build_targets[targetid]
 263
 264     def get_dependees(self, itemid):
 265         """
 266         Return a list of targets which depend on item
 267         """
 268         dependees = []
 269         for fnid in self.depids:
 270             if itemid in self.depids[fnid]:
 271                 dependees.append(fnid)
 272         return dependees
 273
 274     def get_dependees_str(self, item):
 275         """
 276         Return a list of targets which depend on item as a user readable string
 277         """
 278         itemid = self.getbuild_id(item)
 279         dependees = []
 280         for fnid in self.depids:
 281             if itemid in self.depids[fnid]:
 282                 dependees.append(self.fn_index[fnid])
 283         return dependees
 284
 285     def get_rdependees(self, itemid):
 286         """
 287         Return a list of targets which depend on runtime item
 288         """
 289         dependees = []
 290         for fnid in self.rdepids:
 291             if itemid in self.rdepids[fnid]:
 292                 dependees.append(fnid)
 293         return dependees
 294
 295     def get_rdependees_str(self, item):
 296         """
 297         Return a list of targets which depend on runtime item as a user readable string
 298         """
 299         itemid = self.getrun_id(item)
 300         dependees = []
 301         for fnid in self.rdepids:
 302             if itemid in self.rdepids[fnid]:
 303                 dependees.append(self.fn_index[fnid])
 304         return dependees
 305
 306     def add_provider(self, cfgData, dataCache, item):
 307         try:
 308             self.add_provider_internal(cfgData, dataCache, item)
 309         except bb.providers.NoProvider:
 310             if self.abort:
 311                 bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
 312                 raise
 313             targetid = self.getbuild_id(item)
 314             self.remove_buildtarget(targetid)
 315
 316         self.mark_external_target(item)
 317
 318     def add_provider_internal(self, cfgData, dataCache, item):
 319         """
 320         Add the providers of item to the task data
 321         Mark entries were specifically added externally as against dependencies
 322         added internally during dependency resolution
 323         """
 324
 325         if item in dataCache.ignored_dependencies:
 326             return
 327
 328         if not item in dataCache.providers:
 329             bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
 330             bb.event.fire(bb.event.NoProvider(item, cfgData))
 331             raise bb.providers.NoProvider(item)
 332
 333         if self.have_build_target(item):
 334             return
 335
 336         all_p = dataCache.providers[item]
 337
 338         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 339
 340         for p in eligible:
 341             fnid = self.getfn_id(p)
 342             if fnid in self.failed_fnids:
 343                 eligible.remove(p)
 344
 345         if not eligible:
 346             bb.msg.debug(1, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
 347             bb.event.fire(bb.event.NoProvider(item, cfgData))
 348             raise bb.providers.NoProvider(item)
 349
 350         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
 351         if prefervar:
 352             dataCache.preferred[item] = prefervar
 353
 354         discriminated = False
 355         if item in dataCache.preferred:
 356             for p in eligible:
 357                 pn = dataCache.pkg_fn[p]
 358                 if dataCache.preferred[item] == pn:
 359                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
 360                     eligible.remove(p)
 361                     eligible = [p] + eligible
 362                     discriminated = True
 363                     break
 364
 365         if len(eligible) > 1 and discriminated == False:
 366             if item not in self.consider_msgs_cache:
 367                 providers_list = []
 368                 for fn in eligible:
 369                     providers_list.append(dataCache.pkg_fn[fn])
 370                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
 371                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
 372                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
 373             self.consider_msgs_cache.append(item)
 374
 375         for fn in eligible:
 376             fnid = self.getfn_id(fn)
 377             if fnid in self.failed_fnids:
 378                 continue
 379             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
 380             self.add_build_target(fn, item)
 381             self.add_tasks(fn, dataCache)
 382
 383
 384             #item = dataCache.pkg_fn[fn]
 385
 386     def add_rprovider(self, cfgData, dataCache, item):
 387         """
 388         Add the runtime providers of item to the task data
 389         (takes item names from RDEPENDS/PACKAGES namespace)
 390         """
 391
 392         if item in dataCache.ignored_dependencies:
 393             return
 394
 395         if self.have_runtime_target(item):
 396             return
 397
 398         all_p = bb.providers.getRuntimeProviders(dataCache, item)
 399
 400         if not all_p:
 401             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
 402             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 403             raise bb.providers.NoRProvider(item)
 404
 405         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 406
 407         for p in eligible:
 408             fnid = self.getfn_id(p)
 409             if fnid in self.failed_fnids:
 410                 eligible.remove(p)
 411
 412         if not eligible:
 413             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
 414             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 415             raise bb.providers.NoRProvider(item)
 416
 417         # Should use dataCache.preferred here?
 418         preferred = []
 419         for p in eligible:
 420             pn = dataCache.pkg_fn[p]
 421             provides = dataCache.pn_provides[pn]
 422             for provide in provides:
 423                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
 424                 if prefervar == pn:
 425                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
 426                     eligible.remove(p)
 427                     eligible = [p] + eligible
 428                     preferred.append(p)
 429
 430         if len(eligible) > 1 and len(preferred) == 0:
 431             if item not in self.consider_msgs_cache:
 432                 providers_list = []
 433                 for fn in eligible:
 434                     providers_list.append(dataCache.pkg_fn[fn])
 435                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
 436                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
 437                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 438             self.consider_msgs_cache.append(item)
 439
 440         if len(preferred) > 1:
 441             if item not in self.consider_msgs_cache:
 442                 providers_list = []
 443                 for fn in preferred:
 444                     providers_list.append(dataCache.pkg_fn[fn])
 445                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
 446                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
 447                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 448             self.consider_msgs_cache.append(item)
 449
 450         # run through the list until we find one that we can build
 451         for fn in eligible:
 452             fnid = self.getfn_id(fn)
 453             if fnid in self.failed_fnids:
 454                 continue
 455             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
 456             self.add_runtime_target(fn, item)
 457             self.add_tasks(fn, dataCache)
 458
 459     def fail_fnid(self, fnid):
 460         """
 461         Mark a file as failed (unbuildable)
 462         Remove any references from build and runtime provider lists
 463         """
 464         if fnid in self.failed_fnids:
 465             return
 466         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
 467         self.failed_fnids.append(fnid)
 468         for target in self.build_targets:
 469             if fnid in self.build_targets[target]:
 470                 self.build_targets[target].remove(fnid)
 471                 if len(self.build_targets[target]) == 0:
 472                     self.remove_buildtarget(target)
 473         for target in self.run_targets:
 474             if fnid in self.run_targets[target]:
 475                 self.run_targets[target].remove(fnid)
 476                 if len(self.run_targets[target]) == 0:
 477                     self.remove_runtarget(target)
 478
 479     def remove_buildtarget(self, targetid):
 480         """
 481         Mark a build target as failed (unbuildable)
 482         Trigger removal of any files that have this as a dependency
 483         """
 484         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
 485         self.failed_deps.append(targetid)
 486         dependees = self.get_dependees(targetid)
 487         for fnid in dependees:
 488             self.fail_fnid(fnid)
 489         if self.abort and targetid in self.external_targets:
 490             bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s" % self.build_names_index[targetid])
 491             raise bb.providers.NoProvider
 492
 493     def remove_runtarget(self, targetid):
 494         """
 495         Mark a run target as failed (unbuildable)
 496         Trigger removal of any files that have this as a dependency
 497         """
 498         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
 499         self.failed_rdeps.append(targetid)
 500         dependees = self.get_rdependees(targetid)
 501         for fnid in dependees:
 502             self.fail_fnid(fnid)
 503
 504     def add_unresolved(self, cfgData, dataCache):
 505         """
 506         Resolve all unresolved build and runtime targets
 507         """
 508         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
 509         while 1:
 510             added = 0
 511             for target in self.get_unresolved_build_targets(dataCache):
 512                 try:
 513                     self.add_provider_internal(cfgData, dataCache, target)
 514                     added = added + 1
 515                 except bb.providers.NoProvider:
 516                     targetid = self.getbuild_id(target)
 517                     if self.abort and targetid in self.external_targets:
 518                         raise
 519                     self.remove_buildtarget(targetid)
 520             for target in self.get_unresolved_run_targets(dataCache):
 521                 try:
 522                     self.add_rprovider(cfgData, dataCache, target)
 523                     added = added + 1
 524                 except bb.providers.NoRProvider:
 525                     self.remove_runtarget(self.getrun_id(target))
 526             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
 527             if added == 0:
 528                 break
 529
 530     def dump_data(self):
 531         """
 532         Dump some debug information on the internal data structures
 533         """
 534         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
 535         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index))
 536         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
 537         bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index))
 538         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
 539         for target in self.build_targets.keys():
 540             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
 541         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
 542         for target in self.run_targets.keys():
 543             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
 544         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
 545         for task in range(len(self.tasks_name)):
 546             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
 547                 task,
 548                 self.fn_index[self.tasks_fnid[task]],
 549                 self.tasks_name[task],
 550                 self.tasks_tdepends[task]))
 551         bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):")
 552         for fnid in self.rdepids:
 553             bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))
 554
 555