2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 BitBake 'TaskData' implementation
7 Task data collection and handling
11 # Copyright (C) 2006 Richard Purdie
13 # This program is free software; you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License version 2 as
15 # published by the Free Software Foundation.
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
22 # You should have received a copy of the GNU General Public License along
23 # with this program; if not, write to the Free Software Foundation, Inc.,
24 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
26 from bb import data, event, mkdirhier, utils
31 BitBake Task Data implementation
33 def __init__(self, abort = True):
34 self.build_names_index = []
35 self.run_names_index = []
38 self.build_targets = {}
41 self.external_targets = []
45 self.tasks_tdepends = []
46 self.tasks_idepends = []
47 # Cache to speed up task ID lookups
48 self.tasks_lookup = {}
53 self.consider_msgs_cache = []
56 self.failed_rdeps = []
57 self.failed_fnids = []
61 def getbuild_id(self, name):
63 Return an ID number for the build target name.
64 If it doesn't exist, create one.
66 if not name in self.build_names_index:
67 self.build_names_index.append(name)
68 return len(self.build_names_index) - 1
70 return self.build_names_index.index(name)
72 def getrun_id(self, name):
74 Return an ID number for the run target name.
75 If it doesn't exist, create one.
77 if not name in self.run_names_index:
78 self.run_names_index.append(name)
79 return len(self.run_names_index) - 1
81 return self.run_names_index.index(name)
83 def getfn_id(self, name):
85 Return an ID number for the filename.
86 If it doesn't exist, create one.
88 if not name in self.fn_index:
89 self.fn_index.append(name)
90 return len(self.fn_index) - 1
92 return self.fn_index.index(name)
94 def gettask_id(self, fn, task, create = True):
96 Return an ID number for the task matching fn and task.
97 If it doesn't exist, create one by default.
98 Optionally return None instead.
100 fnid = self.getfn_id(fn)
102 if fnid in self.tasks_lookup:
103 if task in self.tasks_lookup[fnid]:
104 return self.tasks_lookup[fnid][task]
109 self.tasks_name.append(task)
110 self.tasks_fnid.append(fnid)
111 self.tasks_tdepends.append([])
112 self.tasks_idepends.append([])
114 listid = len(self.tasks_name) - 1
116 if fnid not in self.tasks_lookup:
117 self.tasks_lookup[fnid] = {}
118 self.tasks_lookup[fnid][task] = listid
122 def add_tasks(self, fn, dataCache):
124 Add tasks for a given fn to the database
127 task_graph = dataCache.task_queues[fn]
128 task_deps = dataCache.task_deps[fn]
130 fnid = self.getfn_id(fn)
132 if fnid in self.failed_fnids:
133 bb.msg.fatal(bb.msg.domain.TaskData, "Trying to re-add a failed file? Something is broken...")
135 # Check if we've already seen this fn
136 if fnid in self.tasks_fnid:
139 for task in task_graph.allnodes():
141 # Work out task dependencies
143 for dep in task_graph.getparents(task):
144 parentid = self.gettask_id(fn, dep)
145 parentids.append(parentid)
146 taskid = self.gettask_id(fn, task)
147 self.tasks_tdepends[taskid].extend(parentids)
149 # Touch all intertask dependencies
150 if 'depends' in task_deps and task in task_deps['depends']:
152 for dep in task_deps['depends'][task].split(" "):
154 ids.append(str(self.getbuild_id(dep.split(":")[0])) + ":" + dep.split(":")[1])
155 self.tasks_idepends[taskid].extend(ids)
157 # Work out build dependencies
158 if not fnid in self.depids:
160 for depend in dataCache.deps[fn]:
161 bb.msg.debug(2, bb.msg.domain.TaskData, "Added dependency %s for %s" % (depend, fn))
162 dependids[self.getbuild_id(depend)] = None
163 self.depids[fnid] = dependids.keys()
165 # Work out runtime dependencies
166 if not fnid in self.rdepids:
168 rdepends = dataCache.rundeps[fn]
169 rrecs = dataCache.runrecs[fn]
170 for package in rdepends:
171 for rdepend in rdepends[package]:
172 bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
173 rdependids[self.getrun_id(rdepend)] = None
174 for package in rrecs:
175 for rdepend in rrecs[package]:
176 bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
177 rdependids[self.getrun_id(rdepend)] = None
178 self.rdepids[fnid] = rdependids.keys()
180 for dep in self.depids[fnid]:
181 if dep in self.failed_deps:
184 for dep in self.rdepids[fnid]:
185 if dep in self.failed_rdeps:
189 def have_build_target(self, target):
191 Have we a build target matching this name?
193 targetid = self.getbuild_id(target)
195 if targetid in self.build_targets:
199 def have_runtime_target(self, target):
201 Have we a runtime target matching this name?
203 targetid = self.getrun_id(target)
205 if targetid in self.run_targets:
209 def add_build_target(self, fn, item):
212 If already present, append the provider fn to the list
214 targetid = self.getbuild_id(item)
215 fnid = self.getfn_id(fn)
217 if targetid in self.build_targets:
218 if fnid in self.build_targets[targetid]:
220 self.build_targets[targetid].append(fnid)
222 self.build_targets[targetid] = [fnid]
224 def add_runtime_target(self, fn, item):
226 Add a runtime target.
227 If already present, append the provider fn to the list
229 targetid = self.getrun_id(item)
230 fnid = self.getfn_id(fn)
232 if targetid in self.run_targets:
233 if fnid in self.run_targets[targetid]:
235 self.run_targets[targetid].append(fnid)
237 self.run_targets[targetid] = [fnid]
239 def mark_external_target(self, item):
241 Mark a build target as being externally requested
243 targetid = self.getbuild_id(item)
245 if targetid not in self.external_targets:
246 self.external_targets.append(targetid)
248 def get_unresolved_build_targets(self, dataCache):
250 Return a list of build targets who's providers
254 for target in self.build_names_index:
255 if target in dataCache.ignored_dependencies:
257 if self.build_names_index.index(target) in self.failed_deps:
259 if not self.have_build_target(target):
260 unresolved.append(target)
263 def get_unresolved_run_targets(self, dataCache):
265 Return a list of runtime targets who's providers
269 for target in self.run_names_index:
270 if target in dataCache.ignored_dependencies:
272 if self.run_names_index.index(target) in self.failed_rdeps:
274 if not self.have_runtime_target(target):
275 unresolved.append(target)
278 def get_provider(self, item):
280 Return a list of providers of item
282 targetid = self.getbuild_id(item)
284 return self.build_targets[targetid]
286 def get_dependees(self, itemid):
288 Return a list of targets which depend on item
291 for fnid in self.depids:
292 if itemid in self.depids[fnid]:
293 dependees.append(fnid)
296 def get_dependees_str(self, item):
298 Return a list of targets which depend on item as a user readable string
300 itemid = self.getbuild_id(item)
302 for fnid in self.depids:
303 if itemid in self.depids[fnid]:
304 dependees.append(self.fn_index[fnid])
307 def get_rdependees(self, itemid):
309 Return a list of targets which depend on runtime item
312 for fnid in self.rdepids:
313 if itemid in self.rdepids[fnid]:
314 dependees.append(fnid)
317 def get_rdependees_str(self, item):
319 Return a list of targets which depend on runtime item as a user readable string
321 itemid = self.getrun_id(item)
323 for fnid in self.rdepids:
324 if itemid in self.rdepids[fnid]:
325 dependees.append(self.fn_index[fnid])
328 def add_provider(self, cfgData, dataCache, item):
330 self.add_provider_internal(cfgData, dataCache, item)
331 except bb.providers.NoProvider:
333 bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
335 targetid = self.getbuild_id(item)
336 self.remove_buildtarget(targetid)
338 self.mark_external_target(item)
340 def add_provider_internal(self, cfgData, dataCache, item):
342 Add the providers of item to the task data
343 Mark entries were specifically added externally as against dependencies
344 added internally during dependency resolution
347 if item in dataCache.ignored_dependencies:
350 if not item in dataCache.providers:
351 bb.msg.note(2, bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (item, self.get_dependees_str(item)))
352 bb.event.fire(bb.event.NoProvider(item, cfgData))
353 raise bb.providers.NoProvider(item)
355 if self.have_build_target(item):
358 all_p = dataCache.providers[item]
360 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
363 fnid = self.getfn_id(p)
364 if fnid in self.failed_fnids:
368 bb.msg.note(2, bb.msg.domain.Provider, "No providers of build target %s after filtering (for %s)" % (item, self.get_dependees_str(item)))
369 bb.event.fire(bb.event.NoProvider(item, cfgData))
370 raise bb.providers.NoProvider(item)
372 if len(eligible) > 1 and foundUnique == False:
373 if item not in self.consider_msgs_cache:
376 providers_list.append(dataCache.pkg_fn[fn])
377 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available for %s (%s);" % (item, ", ".join(providers_list)))
378 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
379 bb.event.fire(bb.event.MultipleProviders(item,providers_list,cfgData))
380 self.consider_msgs_cache.append(item)
383 fnid = self.getfn_id(fn)
384 if fnid in self.failed_fnids:
386 bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy %s" % (fn, item))
387 self.add_build_target(fn, item)
388 self.add_tasks(fn, dataCache)
391 #item = dataCache.pkg_fn[fn]
393 def add_rprovider(self, cfgData, dataCache, item):
395 Add the runtime providers of item to the task data
396 (takes item names from RDEPENDS/PACKAGES namespace)
399 if item in dataCache.ignored_dependencies:
402 if self.have_runtime_target(item):
405 all_p = bb.providers.getRuntimeProviders(dataCache, item)
408 bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s (for %s)" % (item, self.get_rdependees_str(item)))
409 bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
410 raise bb.providers.NoRProvider(item)
412 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
415 fnid = self.getfn_id(p)
416 if fnid in self.failed_fnids:
420 bb.msg.error(bb.msg.domain.Provider, "No providers of runtime build target %s after filtering (for %s)" % (item, self.get_rdependees_str(item)))
421 bb.event.fire(bb.event.NoProvider(item, cfgData, runtime=True))
422 raise bb.providers.NoRProvider(item)
424 if len(eligible) > 1 and numberPreferred == 0:
425 if item not in self.consider_msgs_cache:
428 providers_list.append(dataCache.pkg_fn[fn])
429 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (%s);" % (item, ", ".join(providers_list)))
430 bb.msg.note(2, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER entry to match runtime %s" % item)
431 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
432 self.consider_msgs_cache.append(item)
434 if numberPreferred > 1:
435 if item not in self.consider_msgs_cache:
438 providers_list.append(dataCache.pkg_fn[fn])
439 bb.msg.note(2, bb.msg.domain.Provider, "multiple providers are available for runtime %s (top %s entries preferred) (%s);" % (item, numberPreferred, ", ".join(providers_list)))
440 bb.msg.note(2, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER entry to match runtime %s" % item)
441 bb.event.fire(bb.event.MultipleProviders(item,providers_list, cfgData, runtime=True))
442 self.consider_msgs_cache.append(item)
444 # run through the list until we find one that we can build
446 fnid = self.getfn_id(fn)
447 if fnid in self.failed_fnids:
449 bb.msg.debug(2, bb.msg.domain.Provider, "adding %s to satisfy runtime %s" % (fn, item))
450 self.add_runtime_target(fn, item)
451 self.add_tasks(fn, dataCache)
453 def fail_fnid(self, fnid, missing_list = []):
455 Mark a file as failed (unbuildable)
456 Remove any references from build and runtime provider lists
458 missing_list, A list of missing requirements for this target
460 if fnid in self.failed_fnids:
462 bb.msg.debug(1, bb.msg.domain.Provider, "Removing failed file %s" % self.fn_index[fnid])
463 self.failed_fnids.append(fnid)
464 for target in self.build_targets:
465 if fnid in self.build_targets[target]:
466 self.build_targets[target].remove(fnid)
467 if len(self.build_targets[target]) == 0:
468 self.remove_buildtarget(target, missing_list)
469 for target in self.run_targets:
470 if fnid in self.run_targets[target]:
471 self.run_targets[target].remove(fnid)
472 if len(self.run_targets[target]) == 0:
473 self.remove_runtarget(target, missing_list)
475 def remove_buildtarget(self, targetid, missing_list = []):
477 Mark a build target as failed (unbuildable)
478 Trigger removal of any files that have this as a dependency
480 bb.msg.note(2, bb.msg.domain.Provider, "Removing failed build target %s" % self.build_names_index[targetid])
481 self.failed_deps.append(targetid)
482 dependees = self.get_dependees(targetid)
483 for fnid in dependees:
484 self.fail_fnid(fnid, [self.build_names_index[targetid]]+missing_list)
485 if self.abort and targetid in self.external_targets:
486 bb.msg.error(bb.msg.domain.Provider, "No buildable providers available for required build target %s ('%s')" % (self.build_names_index[targetid], missing_list))
487 raise bb.providers.NoProvider
489 def remove_runtarget(self, targetid, missing_list = []):
491 Mark a run target as failed (unbuildable)
492 Trigger removal of any files that have this as a dependency
494 bb.msg.note(1, bb.msg.domain.Provider, "Removing failed runtime build target %s ('%s')" % (self.run_names_index[targetid], missing_list))
495 self.failed_rdeps.append(targetid)
496 dependees = self.get_rdependees(targetid)
497 for fnid in dependees:
498 self.fail_fnid(fnid, [self.run_names_index[targetid]]+missing_list)
500 def add_unresolved(self, cfgData, dataCache):
502 Resolve all unresolved build and runtime targets
504 bb.msg.note(1, bb.msg.domain.TaskData, "Resolving missing task queue dependencies")
507 for target in self.get_unresolved_build_targets(dataCache):
509 self.add_provider_internal(cfgData, dataCache, target)
511 except bb.providers.NoProvider:
512 targetid = self.getbuild_id(target)
513 if self.abort and targetid in self.external_targets:
514 bb.msg.error(bb.msg.domain.Provider, "No providers of build target %s (for %s)" % (target, self.get_dependees_str(target)))
516 self.remove_buildtarget(targetid)
517 for target in self.get_unresolved_run_targets(dataCache):
519 self.add_rprovider(cfgData, dataCache, target)
521 except bb.providers.NoRProvider:
522 self.remove_runtarget(self.getrun_id(target))
523 bb.msg.debug(1, bb.msg.domain.TaskData, "Resolved " + str(added) + " extra dependecies")
530 Dump some debug information on the internal data structures
532 bb.msg.debug(3, bb.msg.domain.TaskData, "build_names:")
533 bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.build_names_index))
535 bb.msg.debug(3, bb.msg.domain.TaskData, "run_names:")
536 bb.msg.debug(3, bb.msg.domain.TaskData, ", ".join(self.run_names_index))
538 bb.msg.debug(3, bb.msg.domain.TaskData, "build_targets:")
539 for buildid in range(len(self.build_names_index)):
540 target = self.build_names_index[buildid]
542 if buildid in self.build_targets:
543 targets = self.build_targets[buildid]
544 bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s: %s" % (buildid, target, targets))
546 bb.msg.debug(3, bb.msg.domain.TaskData, "run_targets:")
547 for runid in range(len(self.run_names_index)):
548 target = self.run_names_index[runid]
550 if runid in self.run_targets:
551 targets = self.run_targets[runid]
552 bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s: %s" % (runid, target, targets))
554 bb.msg.debug(3, bb.msg.domain.TaskData, "tasks:")
555 for task in range(len(self.tasks_name)):
556 bb.msg.debug(3, bb.msg.domain.TaskData, " (%s)%s - %s: %s" % (
558 self.fn_index[self.tasks_fnid[task]],
559 self.tasks_name[task],
560 self.tasks_tdepends[task]))
562 bb.msg.debug(3, bb.msg.domain.TaskData, "runtime ids (per fn):")
563 for fnid in self.rdepids:
564 bb.msg.debug(3, bb.msg.domain.TaskData, " %s %s: %s" % (fnid, self.fn_index[fnid], self.rdepids[fnid]))