code.vuplus.com Git - vuplus_bitbake/blob - lib/bb/taskdata.py

   1 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 """
   5 BitBake 'TaskData' implementation
   6
   7 Task data collection and handling
   8
   9 Copyright (C) 2006  Richard Purdie
  10
  11 This program is free software; you can redistribute it and/or modify it under
  12 the terms of the GNU General Public License version 2 as published by the Free
  13 Software Foundation
  14
  15 This program is distributed in the hope that it will be useful, but WITHOUT
  16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  18
  19 You should have received a copy of the GNU General Public License along with
  20 """
  21
  22 from bb import data, fetch, event, mkdirhier, utils
  23 import bb, os
  24
  25 class TaskData:
  26     """
  27     BitBake Task Data implementation
  28     """
  29     def __init__(self):
  30         self.build_names_index = []
  31         self.run_names_index = []
  32         self.fn_index = []
  33
  34         self.build_targets = {}
  35         self.run_targets = {}
  36
  37         self.external_targets = []
  38
  39         self.tasks_fnid = []
  40         self.tasks_name = []
  41         self.tasks_tdepends = []
  42
  43         self.depids = {}
  44         self.rdepids = {}
  45
  46         self.consider_msgs_cache = []
  47
  48         self.failed_deps = []
  49         self.failed_rdeps = []
  50         self.failed_fnids = []
  51
  52
  53     def matches_in_list(self, data, substring):
  54         """
  55         Return a list of the positions of substring in list data
  56         """
  57         matches = []
  58         start = 0
  59         while 1:
  60             try:
  61                 start = data.index(substring, start)
  62             except ValueError:
  63                 return matches
  64             matches.append(start)
  65             start = start + 1
  66
  67     def both_contain(self, list1, list2):
  68         """
  69         Return the items present in both list1 and list2
  70         """
  71         matches = []
  72         for data in list1:
  73             if data in list2:
  74                 return data
  75         return None
  76
  77
  78     def getbuild_id(self, name):
  79         """
  80         Return an ID number for the build target name.
  81         If it doesn't exist, create one.
  82         """
  83         if not name in self.build_names_index:
  84             self.build_names_index.append(name)
  85
  86         return self.build_names_index.index(name)
  87
  88     def getrun_id(self, name):
  89         """
  90         Return an ID number for the run target name.
  91         If it doesn't exist, create one.
  92         """
  93         if not name in self.run_names_index:
  94             self.run_names_index.append(name)
  95
  96         return self.run_names_index.index(name)
  97
  98     def getfn_id(self, name):
  99         """
 100         Return an ID number for the filename.
 101         If it doesn't exist, create one.
 102         """
 103         if not name in self.fn_index:
 104             self.fn_index.append(name)
 105
 106         return self.fn_index.index(name)
 107
 108     def gettask_id(self, fn, task):
 109         """
 110         Return an ID number for the task matching fn and task.
 111         If it doesn't exist, create one.
 112         """
 113         fnid = self.getfn_id(fn)
 114
 115         fnids = self.matches_in_list(self.tasks_fnid, fnid)
 116         names = self.matches_in_list(self.tasks_name, task)
 117
 118         listid = self.both_contain(fnids, names)
 119
 120         if listid is not None:
 121             return listid
 122
 123         self.tasks_name.append(task)
 124         self.tasks_fnid.append(fnid)
 125         self.tasks_tdepends.append([])
 126
 127         return len(self.tasks_name)-1
 128
 129     def add_tasks(self, fn, dataCache):
 130         """
 131         Add tasks for a given fn to the database
 132         """
 133
 134         task_graph = dataCache.task_queues[fn]
 135         task_deps = dataCache.task_deps[fn]
 136
 137         fnid = self.getfn_id(fn)
 138
 139         if fnid in self.failed_fnids:
 140             bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
 141
 142         # Check if we've already seen this fn
 143         if fnid in self.tasks_fnid:
 144             return
 145
 146         # Work out task dependencies
 147         for task in task_graph.allnodes():
 148             parentids = []
 149             for dep in task_graph.getparents(task):
 150                 parentid = self.gettask_id(fn, dep)
 151                 parentids.append(parentid)
 152             taskid = self.gettask_id(fn, task)
 153             self.tasks_tdepends[taskid].extend(parentids)
 154
 155         # Work out build dependencies
 156         if not fnid in self.depids:
 157             dependids = []
 158             for depend in dataCache.deps[fn]:
 159                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
 160                 dependids.append(self.getbuild_id(depend))
 161             self.depids[fnid] = dependids
 162
 163         # Work out runtime dependencies
 164         if not fnid in self.rdepids:
 165             rdependids = []
 166             rdepends = dataCache.rundeps[fn]
 167             rrecs = dataCache.runrecs[fn]
 168             for package in rdepends:
 169                 for rdepend in rdepends[package]:
 170                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
 171                     rdependids.append(self.getrun_id(rdepend))
 172             for package in rrecs:
 173                 for rdepend in rrecs[package]:
 174                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
 175                     rdependids.append(self.getrun_id(rdepend))
 176             self.rdepids[fnid] = rdependids
 177
 178     def have_build_target(self, target):
 179         """
 180         Have we a build target matching this name?
 181         """
 182         targetid = self.getbuild_id(target)
 183
 184         if targetid in self.build_targets:
 185             return True
 186         return False
 187
 188     def have_runtime_target(self, target):
 189         """
 190         Have we a runtime target matching this name?
 191         """
 192         targetid = self.getrun_id(target)
 193
 194         if targetid in self.run_targets:
 195             return True
 196         return False
 197
 198     def add_build_target(self, fn, item):
 199         """
 200         Add a build target.
 201         If already present, append the provider fn to the list
 202         """
 203         targetid = self.getbuild_id(item)
 204         fnid = self.getfn_id(fn)
 205
 206         if targetid in self.build_targets:
 207             if fnid in self.build_targets[targetid]:
 208                 return
 209             self.build_targets[targetid].append(fnid)
 210             return
 211         self.build_targets[targetid] = [fnid]
 212
 213     def add_runtime_target(self, fn, item):
 214         """
 215         Add a runtime target.
 216         If already present, append the provider fn to the list
 217         """
 218         targetid = self.getrun_id(item)
 219         fnid = self.getfn_id(fn)
 220
 221         if targetid in self.run_targets:
 222             if fnid in self.run_targets[targetid]:
 223                 return
 224             self.run_targets[targetid].append(fnid)
 225             return
 226         self.run_targets[targetid] = [fnid]
 227
 228     def mark_external_target(self, item):
 229         """
 230         Mark a build target as being externally requested
 231         """
 232         targetid = self.getbuild_id(item)
 233
 234         if targetid not in self.external_targets:
 235             self.external_targets.append(targetid)
 236
 237     def get_unresolved_build_targets(self, dataCache):
 238         """
 239         Return a list of build targets who's providers
 240         are unknown.
 241         """
 242         unresolved = []
 243         for target in self.build_names_index:
 244             if target in dataCache.ignored_dependencies:
 245                 continue
 246             if self.build_names_index.index(target) in self.failed_deps:
 247                 continue
 248             if not self.have_build_target(target):
 249                 unresolved.append(target)
 250         return unresolved
 251
 252     def get_unresolved_run_targets(self, dataCache):
 253         """
 254         Return a list of runtime targets who's providers
 255         are unknown.
 256         """
 257         unresolved = []
 258         for target in self.run_names_index:
 259             if target in dataCache.ignored_dependencies:
 260                 continue
 261             if self.run_names_index.index(target) in self.failed_rdeps:
 262                 continue
 263             if not self.have_runtime_target(target):
 264                 unresolved.append(target)
 265         return unresolved
 266
 267     def get_provider(self, item):
 268         """
 269         Return a list of providers of item
 270         """
 271         targetid = self.getbuild_id(item)
 272
 273         return self.build_targets[targetid]
 274
 275     def get_dependees(self, itemid):
 276         """
 277         Return a list of targets which depend on item
 278         """
 279         dependees = []
 280         for fnid in self.depids:
 281             if itemid in self.depids[fnid]:
 282                 dependees.append(fnid)
 283         return dependees
 284
 285     def get_dependees_str(self, item):
 286         """
 287         Return a list of targets which depend on item as a user readable string
 288         """
 289         itemid = self.getbuild_id(item)
 290         dependees = []
 291         for fnid in self.depids:
 292             if itemid in self.depids[fnid]:
 293                 dependees.append(self.fn_index[fnid])
 294         return dependees
 295
 296     def get_rdependees(self, itemid):
 297         """
 298         Return a list of targets which depend on runtime item
 299         """
 300         dependees = []
 301         for fnid in self.rdepids:
 302             if itemid in self.rdepids[fnid]:
 303                 dependees.append(fnid)
 304         return dependees
 305
 306     def get_rdependees_str(self, item):
 307         """
 308         Return a list of targets which depend on runtime item as a user readable string
 309         """
 310         itemid = self.getrun_id(item)
 311         dependees = []
 312         for fnid in self.rdepids:
 313             if itemid in self.rdepids[fnid]:
 314                 dependees.append(self.fn_index[fnid])
 315         return dependees
 316
 317     def add_provider(self, cfgData, dataCache, item, external = True):
 318         """
 319         Add the providers of item to the task data
 320         Mark entries were specifically added externally as against dependencies
 321         added internally during dependency resolution
 322         """
 323
 324         if item in dataCache.ignored_dependencies:
 325             return True
 326
 327         if not item in dataCache.providers:
 328             msg = "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item))
 329             if external:
 330                 bb.msg.error(bb.msg.domain.Provider, msg)
 331             else:
 332                 bb.msg.debug(1, bb.msg.domain.Provider, msg)
 333             bb.event.fire(bb.event.NoProvider(item, cfgData))
 334             raise bb.providers.NoProvider(item)
 335
 336         if self.have_build_target(item):
 337             return True
 338
 339         all_p = dataCache.providers[item]
 340
 341         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 342
 343         for p in eligible:
 344             fnid = self.getfn_id(p)
 345             if fnid in self.failed_fnids:
 346                 eligible.remove(p)
 347
 348         if not eligible:
 349             msg = "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item))
 350             if external:
 351                 bb.msg.error(bb.msg.domain.Provider, msg)
 352             else:
 353                 bb.msg.debug(1, bb.msg.domain.Provider, msg)
 354             bb.event.fire(bb.event.NoProvider(item, cfgData))
 355             raise bb.providers.NoProvider(item)
 356
 357         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
 358         if prefervar:
 359             dataCache.preferred[item] = prefervar
 360
 361         discriminated = False
 362         if item in dataCache.preferred:
 363             for p in eligible:
 364                 pn = dataCache.pkg_fn[p]
 365                 if dataCache.preferred[item] == pn:
 366                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
 367                     eligible.remove(p)
 368                     eligible = [p] + eligible
 369                     discriminated = True
 370                     break
 371
 372         if len(eligible) > 1 and discriminated == False:
 373             if item not in self.consider_msgs_cache:
 374                 providers_list = []
 375                 for fn in eligible:
 376                     providers_list.append(dataCache.pkg_fn[fn])
 377                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
 378                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
 379                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
 380             self.consider_msgs_cache.append(item)
 381
 382         for fn in eligible:
 383             fnid = self.getfn_id(fn)
 384             if fnid in self.failed_fnids:
 385                 continue
 386             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
 387             self.add_tasks(fn, dataCache)
 388             self.add_build_target(fn, item)
 389
 390             item = dataCache.pkg_fn[fn]
 391
 392         self.mark_external_target(item)
 393
 394         return True
 395
 396     def add_rprovider(self, cfgData, dataCache, item):
 397         """
 398         Add the runtime providers of item to the task data
 399         (takes item names from RDEPENDS/PACKAGES namespace)
 400         """
 401
 402         if item in dataCache.ignored_dependencies:
 403             return True
 404
 405         if self.have_runtime_target(item):
 406             return True
 407
 408         all_p = bb.providers.getRuntimeProviders(dataCache, item)
 409
 410         if not all_p:
 411             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
 412             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 413             raise bb.providers.NoRProvider(item)
 414
 415         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 416
 417         for p in eligible:
 418             fnid = self.getfn_id(p)
 419             if fnid in self.failed_fnids:
 420                 eligible.remove(p)
 421
 422         if not eligible:
 423             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
 424             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 425             raise bb.providers.NoRProvider(item)
 426
 427         # Should use dataCache.preferred here?
 428         preferred = []
 429         for p in eligible:
 430             pn = dataCache.pkg_fn[p]
 431             provides = dataCache.pn_provides[pn]
 432             for provide in provides:
 433                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
 434                 if prefervar == pn:
 435                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
 436                     eligible.remove(p)
 437                     eligible = [p] + eligible
 438                     preferred.append(p)
 439
 440         if len(eligible) > 1 and len(preferred) == 0:
 441             if item not in self.consider_msgs_cache:
 442                 providers_list = []
 443                 for fn in eligible:
 444                     providers_list.append(dataCache.pkg_fn[fn])
 445                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
 446                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
 447                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 448             self.consider_msgs_cache.append(item)
 449
 450         if len(preferred) > 1:
 451             if item not in self.consider_msgs_cache:
 452                 providers_list = []
 453                 for fn in preferred:
 454                     providers_list.append(dataCache.pkg_fn[fn])
 455                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available (%s);" % ", ".join(providers_list))
 456                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
 457                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 458             self.consider_msgs_cache.append(item)
 459
 460         # run through the list until we find one that we can build
 461         for fn in eligible:
 462             fnid = self.getfn_id(fn)
 463             if fnid in self.failed_fnids:
 464                 continue
 465             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
 466             self.add_tasks(fn, dataCache)
 467             self.add_runtime_target(fn, item)
 468
 469         return True
 470
 471     def fail_fnid(self, fnid):
 472         """
 473         Mark a file as failed (unbuildable)
 474         Remove any references from build and runtime provider lists
 475         """
 476         if fnid in self.failed_fnids:
 477             return
 478         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
 479         self.failed_fnids.append(fnid)
 480         for target in self.build_targets:
 481             if fnid in self.build_targets[target]:
 482                 self.build_targets[target].remove(fnid)
 483                 if len(self.build_targets[target]) == 0:
 484                     self.remove_buildtarget(target)
 485         for target in self.run_targets:
 486             if fnid in self.run_targets[target]:
 487                 self.run_targets[target].remove(fnid)
 488                 if len(self.run_targets[target]) == 0:
 489                     self.remove_runtarget(target)
 490
 491     def remove_buildtarget(self, targetid):
 492         """
 493         Mark a build target as failed (unbuildable)
 494         Trigger removal of any files that have this as a dependency
 495         """
 496         bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
 497         self.failed_deps.append(targetid)
 498         dependees = self.get_dependees(targetid)
 499         for fnid in dependees:
 500             self.fail_fnid(fnid)
 501
 502     def remove_runtarget(self, targetid):
 503         """
 504         Mark a run target as failed (unbuildable)
 505         Trigger removal of any files that have this as a dependency
 506         """
 507         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
 508         self.failed_rdeps.append(targetid)
 509         dependees = self.get_rdependees(targetid)
 510         for fnid in dependees:
 511             self.fail_fnid(fnid)
 512
 513     def add_unresolved(self, cfgData, dataCache):
 514         """
 515         Resolve all unresolved build and runtime targets
 516         """
 517         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
 518         while 1:
 519             added = 0
 520             for target in self.get_unresolved_build_targets(dataCache):
 521                 try:
 522                     self.add_provider(cfgData, dataCache, target, False)
 523                     added = added + 1
 524                 except bb.providers.NoProvider:
 525                     targetid = self.getbuild_id(target)
 526                     if targetid in self.external_targets:
 527                         # FIXME - should look at configuration.abort here and only raise if set
 528                         raise
 529                     self.remove_buildtarget(targetid)
 530             for target in self.get_unresolved_run_targets(dataCache):
 531                 try:
 532                     self.add_rprovider(cfgData, dataCache, target)
 533                     added = added + 1
 534                 except bb.providers.NoRProvider:
 535                     self.remove_runtarget(self.getrun_id(target))
 536             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
 537             if added == 0:
 538                 break
 539
 540
 541     def dump_data(self):
 542         """
 543         Dump some debug information on the internal data structures
 544         """
 545         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
 546         bb.msg.debug(3, bb.msg.domain.TaskData, self.build_names_index)
 547         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
 548         bb.msg.debug(3, bb.msg.domain.TaskData, self.run_names_index)
 549         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
 550         for target in self.build_targets.keys():
 551             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
 552         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
 553         for target in self.run_targets.keys():
 554             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
 555         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
 556         for task in range(len(self.tasks_name)):
 557             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
 558                 task,
 559                 self.fn_index[self.tasks_fnid[task]],
 560                 self.tasks_name[task],
 561                 self.tasks_tdepends[task]))
 562
 563