code.vuplus.com Git - vuplus_bitbake/blob - lib/bb/taskdata.py

   1 #!/usr/bin/env python
   2 # ex:ts=4:sw=4:sts=4:et
   3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
   4 """
   5 BitBake 'TaskData' implementation
   6
   7 Task data collection and handling
   8
   9 Copyright (C) 2006  Richard Purdie
  10
  11 This program is free software; you can redistribute it and/or modify it under
  12 the terms of the GNU General Public License version 2 as published by the Free
  13 Software Foundation
  14
  15 This program is distributed in the hope that it will be useful, but WITHOUT
  16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  18
  19 You should have received a copy of the GNU General Public License along with
  20 """
  21
  22 from bb import data, fetch, event, mkdirhier, utils
  23 import bb, os
  24
  25 class TaskData:
  26     """
  27     BitBake Task Data implementation
  28     """
  29     def __init__(self):
  30         self.build_names_index = []
  31         self.run_names_index = []
  32         self.fn_index = []
  33
  34         self.build_targets = {}
  35         self.run_targets = {}
  36
  37         self.tasks_fnid = []
  38         self.tasks_name = []
  39         self.tasks_tdepends = []
  40
  41         self.depids = {}
  42         self.rdepids = {}
  43
  44         self.consider_msgs_cache = []
  45
  46         self.failed_deps = []
  47         self.failed_rdeps = []
  48         self.failed_fnids = []
  49
  50
  51     def matches_in_list(self, data, substring):
  52         """
  53         Return a list of the positions of substring in list data
  54         """
  55         matches = []
  56         start = 0
  57         while 1:
  58             try:
  59                 start = data.index(substring, start)
  60             except ValueError:
  61                 return matches
  62             matches.append(start)
  63             start = start + 1
  64
  65     def both_contain(self, list1, list2):
  66         """
  67         Return the items present in both list1 and list2
  68         """
  69         matches = []
  70         for data in list1:
  71             if data in list2:
  72                 return data
  73         return None
  74
  75
  76     def getbuild_id(self, name):
  77         """
  78         Return an ID number for the build target name.
  79         If it doesn't exist, create one.
  80         """
  81         if not name in self.build_names_index:
  82             self.build_names_index.append(name)
  83
  84         return self.build_names_index.index(name)
  85
  86     def getrun_id(self, name):
  87         """
  88         Return an ID number for the run target name.
  89         If it doesn't exist, create one.
  90         """
  91         if not name in self.run_names_index:
  92             self.run_names_index.append(name)
  93
  94         return self.run_names_index.index(name)
  95
  96     def getfn_id(self, name):
  97         """
  98         Return an ID number for the filename.
  99         If it doesn't exist, create one.
 100         """
 101         if not name in self.fn_index:
 102             self.fn_index.append(name)
 103
 104         return self.fn_index.index(name)
 105
 106     def gettask_id(self, fn, task):
 107         """
 108         Return an ID number for the task matching fn and task.
 109         If it doesn't exist, create one.
 110         """
 111         fnid = self.getfn_id(fn)
 112
 113         fnids = self.matches_in_list(self.tasks_fnid, fnid)
 114         names = self.matches_in_list(self.tasks_name, task)
 115
 116         listid = self.both_contain(fnids, names)
 117
 118         if listid is not None:
 119             return listid
 120
 121         self.tasks_name.append(task)
 122         self.tasks_fnid.append(fnid)
 123         self.tasks_tdepends.append([])
 124
 125         return len(self.tasks_name)-1
 126
 127     def add_tasks(self, fn, dataCache):
 128         """
 129         Add tasks for a given fn to the database
 130         """
 131
 132         task_graph = dataCache.task_queues[fn]
 133         task_deps = dataCache.task_deps[fn]
 134
 135         fnid = self.getfn_id(fn)
 136
 137         if fnid in self.failed_fnids:
 138             bb.fatal("Trying to re-add a failed file? Something is broken...")
 139
 140         # Check if we've already seen this fn
 141         if fnid in self.tasks_fnid:
 142             return
 143
 144         # Work out task dependencies
 145         for task in task_graph.allnodes():
 146             parentids = []
 147             for dep in task_graph.getparents(task):
 148                 parentid = self.gettask_id(fn, dep)
 149                 parentids.append(parentid)
 150             taskid = self.gettask_id(fn, task)
 151             self.tasks_tdepends[taskid].extend(parentids)
 152
 153         # Work out build dependencies
 154         if not fnid in self.depids:
 155             dependids = []
 156             for depend in dataCache.deps[fn]:
 157                 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
 158                 dependids.append(self.getbuild_id(depend))
 159             self.depids[fnid] = dependids
 160
 161         # Work out runtime dependencies
 162         if not fnid in self.rdepids:
 163             rdependids = []
 164             rdepends = dataCache.rundeps[fn]
 165             rrecs = dataCache.runrecs[fn]
 166             for package in rdepends:
 167                 for rdepend in rdepends[package]:
 168                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
 169                     rdependids.append(self.getrun_id(rdepend))
 170             for package in rrecs:
 171                 for rdepend in rrecs[package]:
 172                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
 173                     rdependids.append(self.getrun_id(rdepend))
 174             self.rdepids[fnid] = rdependids
 175
 176     def have_build_target(self, target):
 177         """
 178         Have we a build target matching this name?
 179         """
 180         targetid = self.getbuild_id(target)
 181
 182         if targetid in self.build_targets:
 183             return True
 184         return False
 185
 186     def have_runtime_target(self, target):
 187         """
 188         Have we a runtime target matching this name?
 189         """
 190         targetid = self.getrun_id(target)
 191
 192         if targetid in self.run_targets:
 193             return True
 194         return False
 195
 196     def add_build_target(self, fn, item):
 197         """
 198         Add a build target.
 199         If already present, append the provider fn to the list
 200         """
 201         targetid = self.getbuild_id(item)
 202         fnid = self.getfn_id(fn)
 203
 204         if targetid in self.build_targets:
 205             if fnid in self.build_targets[targetid]:
 206                 return
 207             self.build_targets[targetid].append(fnid)
 208             return
 209         self.build_targets[targetid] = [fnid]
 210
 211     def add_runtime_target(self, fn, item):
 212         """
 213         Add a runtime target.
 214         If already present, append the provider fn to the list
 215         """
 216         targetid = self.getrun_id(item)
 217         fnid = self.getfn_id(fn)
 218
 219         if targetid in self.run_targets:
 220             if fnid in self.run_targets[targetid]:
 221                 return
 222             self.run_targets[targetid].append(fnid)
 223             return
 224         self.run_targets[targetid] = [fnid]
 225
 226     def get_unresolved_build_targets(self, dataCache):
 227         """
 228         Return a list of build targets who's providers
 229         are unknown.
 230         """
 231         unresolved = []
 232         for target in self.build_names_index:
 233             if target in dataCache.ignored_dependencies:
 234                 continue
 235             if target in self.failed_deps:
 236                 continue
 237             if not self.have_build_target(target):
 238                 unresolved.append(target)
 239         return unresolved
 240
 241     def get_unresolved_run_targets(self, dataCache):
 242         """
 243         Return a list of runtime targets who's providers
 244         are unknown.
 245         """
 246         unresolved = []
 247         for target in self.run_names_index:
 248             if target in dataCache.ignored_dependencies:
 249                 continue
 250             if target in self.failed_rdeps:
 251                 continue
 252             if not self.have_runtime_target(target):
 253                 unresolved.append(target)
 254         return unresolved
 255
 256     def get_provider(self, item):
 257         """
 258         Return a list of providers of item
 259         """
 260         targetid = self.getbuild_id(item)
 261
 262         return self.build_targets[targetid]
 263
 264     def get_dependees(self, itemid):
 265         """
 266         Return a list of targets which depend on item
 267         """
 268         dependees = []
 269         for fnid in self.depids:
 270             if itemid in self.depids[fnid]:
 271                 dependees.append(fnid)
 272         return dependees
 273
 274     def get_dependees_str(self, item):
 275         """
 276         Return a list of targets which depend on item as a user readable string
 277         """
 278         itemid = self.getbuild_id(item)
 279         dependees = []
 280         for fnid in self.depids:
 281             if itemid in self.depids[fnid]:
 282                 dependees.append(self.fn_index[fnid])
 283         return dependees
 284
 285     def get_rdependees(self, itemid):
 286         """
 287         Return a list of targets which depend on runtime item
 288         """
 289         dependees = []
 290         for fnid in self.rdepids:
 291             if itemid in self.rdepids[fnid]:
 292                 dependees.append(fnid)
 293         return dependees
 294
 295     def get_rdependees_str(self, item):
 296         """
 297         Return a list of targets which depend on runtime item as a user readable string
 298         """
 299         itemid = self.getrun_id(item)
 300         dependees = []
 301         for fnid in self.rdepids:
 302             if itemid in self.rdepids[fnid]:
 303                 dependees.append(self.fn_index[fnid])
 304         return dependees
 305
 306     def add_provider(self, cfgData, dataCache, item):
 307         """
 308         Add the providers of item to the task data
 309         """
 310
 311         if item in dataCache.ignored_dependencies:
 312             return True
 313
 314         if not item in dataCache.providers:
 315             bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
 316             bb.event.fire(bb.event.NoProvider(item, cfgData))
 317             raise bb.providers.NoProvider(item)
 318
 319         if self.have_build_target(item):
 320             return True
 321
 322         all_p = dataCache.providers[item]
 323
 324         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 325
 326         for p in eligible:
 327             fnid = self.getfn_id(p)
 328             if fnid in self.failed_fnids:
 329                 eligible.remove(p)
 330
 331         if not eligible:
 332             bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
 333             bb.event.fire(bb.event.NoProvider(item, cfgData))
 334             raise bb.providers.NoProvider(item)
 335
 336         prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, cfgData, 1)
 337         if prefervar:
 338             dataCache.preferred[item] = prefervar
 339
 340         discriminated = False
 341         if item in dataCache.preferred:
 342             for p in eligible:
 343                 pn = dataCache.pkg_fn[p]
 344                 if dataCache.preferred[item] == pn:
 345                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
 346                     eligible.remove(p)
 347                     eligible = [p] + eligible
 348                     discriminated = True
 349                     break
 350
 351         if len(eligible) > 1 and discriminated == False:
 352             if item not in self.consider_msgs_cache:
 353                 providers_list = []
 354                 for fn in eligible:
 355                     providers_list.append(dataCache.pkg_fn[fn])
 356                 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
 357                 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
 358                 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
 359             self.consider_msgs_cache.append(item)
 360
 361         for fn in eligible:
 362             fnid = self.getfn_id(fn)
 363             if fnid in self.failed_fnids:
 364                 continue
 365             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
 366             self.add_tasks(fn, dataCache)
 367             self.add_build_target(fn, item)
 368
 369             item = dataCache.pkg_fn[fn]
 370
 371         return True
 372
 373     def add_rprovider(self, cfgData, dataCache, item):
 374         """
 375         Add the runtime providers of item to the task data
 376         (takes item names from RDEPENDS/PACKAGES namespace)
 377         """
 378
 379         if item in dataCache.ignored_dependencies:
 380             return True
 381
 382         if self.have_runtime_target(item):
 383             return True
 384
 385         all_p = bb.providers.getRuntimeProviders(dataCache, item)
 386
 387         if not all_p:
 388             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
 389             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 390             raise bb.providers.NoRProvider(item)
 391
 392         eligible = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
 393
 394         for p in eligible:
 395             fnid = self.getfn_id(p)
 396             if fnid in self.failed_fnids:
 397                 eligible.remove(p)
 398
 399         if not eligible:
 400             bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
 401             bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
 402             raise bb.providers.NoRProvider(item)
 403
 404         # Should use dataCache.preferred here?
 405         preferred = []
 406         for p in eligible:
 407             pn = dataCache.pkg_fn[p]
 408             provides = dataCache.pn_provides[pn]
 409             for provide in provides:
 410                 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, cfgData, 1)
 411                 if prefervar == pn:
 412                     bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
 413                     eligible.remove(p)
 414                     eligible = [p] + eligible
 415                     preferred.append(p)
 416
 417         if len(eligible) > 1 and len(preferred) == 0:
 418             if item not in self.consider_msgs_cache:
 419                 providers_list = []
 420                 for fn in eligible:
 421                     providers_list.append(dataCache.pkg_fn[fn])
 422                 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
 423                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
 424                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 425             self.consider_msgs_cache.append(item)
 426
 427         if len(preferred) > 1:
 428             if item not in self.consider_msgs_cache:
 429                 providers_list = []
 430                 for fn in preferred:
 431                     providers_list.append(dataCache.pkg_fn[fn])
 432                 bb.msg.note(2, bb.msg.domain.Provider, "multiple preferred providers are available (%s);" % ", ".join(providers_list))
 433                 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
 434                 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
 435             self.consider_msgs_cache.append(item)
 436
 437         # run through the list until we find one that we can build
 438         for fn in eligible:
 439             fnid = self.getfn_id(fn)
 440             if fnid in self.failed_fnids:
 441                 continue
 442             bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
 443             self.add_tasks(fn, dataCache)
 444             self.add_runtime_target(fn, item)
 445
 446         return True
 447
 448     def fail_fnid(self, fnid):
 449         """
 450         Mark a file as failed (unbuildable)
 451         Remove any references from build and runtime provider lists
 452         """
 453         if fnid in self.failed_fnids:
 454             return
 455         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
 456         self.failed_fnids.append(fnid)
 457         for target in self.build_targets:
 458             if fnid in self.build_targets[target]:
 459                 self.build_targets[target].remove(fnid)
 460                 if len(self.build_targets[target]) == 0:
 461                     self.remove_buildtarget(target)
 462         for target in self.run_targets:
 463             if fnid in self.run_targets[target]:
 464                 self.run_targets[target].remove(fnid)
 465                 if len(self.run_targets[target]) == 0:
 466                     self.remove_runtarget(target)
 467
 468     def remove_buildtarget(self, targetid):
 469         """
 470         Mark a build target as failed (unbuildable)
 471         Trigger removal of any files that have this as a dependency
 472         """
 473         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
 474         self.failed_deps.append(targetid)
 475         dependees = self.get_dependees(targetid)
 476         for fnid in dependees:
 477             self.fail_fnid(fnid)
 478
 479     def remove_runtarget(self, targetid):
 480         """
 481         Mark a run target as failed (unbuildable)
 482         Trigger removal of any files that have this as a dependency
 483         """
 484         bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s" % self.run_names_index[targetid])
 485         self.failed_rdeps.append(targetid)
 486         dependees = self.get_rdependees(targetid)
 487         for fnid in dependees:
 488             self.fail_fnid(fnid)
 489
 490     def add_unresolved(self, cfgData, dataCache):
 491         """
 492         Resolve all unresolved build and runtime targets
 493         """
 494         bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
 495         while 1:
 496             added = 0
 497             for target in self.get_unresolved_build_targets(dataCache):
 498                 try:
 499                     self.add_provider(cfgData, dataCache, target)
 500                     added = added + 1
 501                 except bb.providers.NoProvider:
 502                     # FIXME - should look at configuration.abort here and raise if set
 503                     self.remove_buildtarget(self.getbuild_id(target))
 504             for target in self.get_unresolved_run_targets(dataCache):
 505                 try:
 506                     self.add_rprovider(cfgData, dataCache, target)
 507                     added = added + 1
 508                 except bb.providers.NoRProvider:
 509                     # FIXME - should look at configuration.abort here and raise if set
 510                     self.remove_runtarget(self.getrun_id(target))
 511             bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
 512             if added == 0:
 513                 break
 514
 515
 516
 517     def dump_data(self):
 518         """
 519         Dump some debug information on the internal data structures
 520         """
 521         bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
 522         bb.msg.debug(3, bb.msg.domain.TaskData, self.build_names_index)
 523         bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
 524         bb.msg.debug(3, bb.msg.domain.TaskData, self.run_names_index)
 525         bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
 526         for target in self.build_targets.keys():
 527             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.build_names_index[target], self.build_targets[target]))
 528         bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
 529         for target in self.run_targets.keys():
 530             bb.msg.debug(3, bb.msg.domain.TaskData, " %s: %s" % (self.run_names_index[target], self.run_targets[target]))
 531         bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
 532         for task in range(len(self.tasks_name)):
 533             bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
 534                 task,
 535                 self.fn_index[self.tasks_fnid[task]],
 536                 self.tasks_name[task],
 537                 self.tasks_tdepends[task]))
 538
 539