2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re, time
25 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
27 from bb import utils, data, parse, debug, event, fatal, cache
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
50 self.packages_dynamic = {}
51 self.bbfile_priority = {}
52 self.bbfile_config_priorities = []
53 self.ignored_dependencies = None
54 self.possible_world = []
55 self.world_target = Set()
61 self.all_depends = Set()
67 def handle_bb_data(self, file_name, bb_cache, cached):
69 We will fill the dictionaries with the stuff we
70 need for building the tree more fast
73 pn = bb_cache.getVar('PN', file_name, True)
74 pv = bb_cache.getVar('PV', file_name, True)
75 pr = bb_cache.getVar('PR', file_name, True)
76 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
77 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
78 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
79 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
80 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
81 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
83 # build PackageName to FileName lookup table
84 if pn not in self.pkg_pn:
86 self.pkg_pn[pn].append(file_name)
88 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
89 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
91 # build FileName to PackageName lookup table
92 self.pkg_fn[file_name] = pn
93 self.pkg_pvpr[file_name] = (pv,pr)
94 self.pkg_dp[file_name] = dp
96 # Build forward and reverse provider hashes
97 # Forward: virtual -> [filenames]
98 # Reverse: PN -> [virtuals]
99 if pn not in self.pn_provides:
100 self.pn_provides[pn] = Set()
101 self.pn_provides[pn] |= provides
103 for provide in provides:
104 if provide not in self.providers:
105 self.providers[provide] = []
106 self.providers[provide].append(file_name)
109 self.all_depends.add(dep)
111 # Build reverse hash for PACKAGES, so runtime dependencies
112 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
113 for package in packages:
114 if not package in self.packages:
115 self.packages[package] = []
116 self.packages[package].append(file_name)
117 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
119 for package in packages_dynamic:
120 if not package in self.packages_dynamic:
121 self.packages_dynamic[package] = []
122 self.packages_dynamic[package].append(file_name)
124 for rprovide in rprovides:
125 if not rprovide in self.rproviders:
126 self.rproviders[rprovide] = []
127 self.rproviders[rprovide].append(file_name)
129 # Build hash of runtime depeneds and rececommends
131 def add_dep(deplist, deps):
133 if not dep in deplist:
136 if not file_name in self.rundeps:
137 self.rundeps[file_name] = {}
138 if not file_name in self.runrecs:
139 self.runrecs[file_name] = {}
141 for package in packages + [pn]:
142 if not package in self.rundeps[file_name]:
143 self.rundeps[file_name][package] = {}
144 if not package in self.runrecs[file_name]:
145 self.runrecs[file_name][package] = {}
147 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
148 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
149 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
150 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
152 # Collect files we may need for possible world-dep
154 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
155 self.possible_world.append(file_name)
158 #============================================================================#
160 #============================================================================#
163 Manage build statistics for one run
172 print "Build statistics:"
173 print " Attempted builds: %d" % self.attempt
175 print " Failed builds: %d" % self.fail
177 print " Dependencies not satisfied: %d" % self.deps
178 if self.fail or self.deps: return 1
182 #============================================================================#
184 #============================================================================#
185 class BBConfiguration( object ):
187 Manages build options and configurations for one run
189 def __init__( self, options ):
190 for key, val in options.__dict__.items():
191 setattr( self, key, val )
193 #============================================================================#
195 #============================================================================#
198 Manages one bitbake build run
201 ParsingStatus = BBParsingStatus # make it visible from the shell
202 Statistics = BBStatistics # make it visible from the shell
204 def __init__( self ):
205 self.build_cache_fail = []
206 self.build_cache = []
207 self.rbuild_cache = []
208 self.building_list = []
210 self.consider_msgs_cache = []
212 self.stats = BBStatistics()
218 def tryBuildPackage( self, fn, item, the_data ):
219 """Build one package"""
220 bb.event.fire(bb.event.PkgStarted(item, the_data))
222 self.stats.attempt += 1
223 if self.configuration.force:
224 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
225 if not self.configuration.dry_run:
226 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
227 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
228 self.build_cache.append(fn)
230 except bb.build.FuncFailed:
232 bb.error("task stack execution failed")
233 bb.event.fire(bb.event.PkgFailed(item, the_data))
234 self.build_cache_fail.append(fn)
236 except bb.build.EventException, e:
239 bb.error("%s event exception, aborting" % bb.event.getName(event))
240 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn)
244 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
246 Build a provider and its dependencies.
247 build_depends is a list of previous build dependencies (not runtime)
248 If build_depends is empty, we're dealing with a runtime depends
251 the_data = self.bb_cache.loadDataFull(fn, self)
253 # Only follow all (runtime) dependencies if doing a build
254 if not buildAllDeps and self.configuration.cmd is "build":
255 buildAllDeps = self.status.build_all[fn]
257 # Error on build time dependency loops
258 if build_depends and build_depends.count(fn) > 1:
259 bb.error("%s depends on itself (eventually)" % fn)
260 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
263 # See if this is a runtime dependency we've already built
264 # Or a build dependency being handled in a different build chain
265 if fn in self.building_list:
266 return self.addRunDeps(fn, virtual , buildAllDeps)
268 item = self.status.pkg_fn[fn]
270 self.building_list.append(fn)
272 pathstr = "%s (%s)" % (item, virtual)
273 self.build_path.append(pathstr)
275 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
277 if self.configuration.verbose:
278 bb.note("current path: %s" % (" -> ".join(self.build_path)))
279 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
284 depcmd = self.configuration.cmd
285 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
286 if bbdepcmd is not None:
293 oldcmd = self.configuration.cmd
294 self.configuration.cmd = depcmd
296 for dependency in depends_list:
297 if dependency in self.status.ignored_dependencies:
301 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
302 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
304 if self.configuration.abort:
308 self.configuration.cmd = oldcmd
314 if not self.addRunDeps(fn, virtual , buildAllDeps):
317 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
318 self.build_cache.append(fn)
321 return self.tryBuildPackage( fn, item, the_data )
324 self.building_list.remove(fn)
325 self.build_path.remove(pathstr)
327 def findBestProvider( self, pn, pkg_pn = None):
329 If there is a PREFERRED_VERSION, find the highest-priority bbfile
330 providing that version. If not, find the latest version provided by
331 an bbfile in the highest-priority set.
334 pkg_pn = self.status.pkg_pn
339 priority = self.status.bbfile_priority[f]
340 if priority not in priorities:
341 priorities[priority] = []
342 priorities[priority].append(f)
343 p_list = priorities.keys()
344 p_list.sort(lambda a, b: a - b)
347 tmp_pn = [priorities[p]] + tmp_pn
349 preferred_file = None
351 localdata = data.createCopy(self.configuration.data)
352 bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
353 bb.data.update_data(localdata)
355 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
357 m = re.match('(.*)_(.*)', preferred_v)
359 preferred_v = m.group(1)
360 preferred_r = m.group(2)
364 for file_set in tmp_pn:
366 pv,pr = self.status.pkg_pvpr[f]
367 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
369 preferred_ver = (pv, pr)
374 pv_str = '%s-%s' % (preferred_v, preferred_r)
377 if preferred_file is None:
378 bb.note("preferred version %s of %s not available" % (pv_str, pn))
380 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
384 # get highest priority file set
389 for file_name in files:
390 pv,pr = self.status.pkg_pvpr[file_name]
391 dp = self.status.pkg_dp[file_name]
393 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
397 if preferred_file is None:
398 preferred_file = latest_f
399 preferred_ver = latest
401 return (latest,latest_f,preferred_ver, preferred_file)
403 def showVersions( self ):
404 pkg_pn = self.status.pkg_pn
405 preferred_versions = {}
409 for pn in pkg_pn.keys():
410 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
411 preferred_versions[pn] = (pref_ver, pref_file)
412 latest_versions[pn] = (last_ver, last_file)
414 pkg_list = pkg_pn.keys()
418 pref = preferred_versions[p]
419 latest = latest_versions[p]
422 prefstr = pref[0][0] + "-" + pref[0][1]
426 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
430 def showEnvironment( self ):
431 """Show the outer or per-package environment"""
432 if self.configuration.buildfile:
434 self.bb_cache = bb.cache.init(self)
436 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
438 fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
441 # emit variables and shell functions
443 data.update_data( self.configuration.data )
444 data.emit_env(sys.__stdout__, self.configuration.data, True)
447 # emit the metadata which isnt valid shell
448 for e in self.configuration.data.keys():
449 if data.getVarFlag( e, 'python', self.configuration.data ):
450 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
452 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
454 Generate two graphs one for the DEPENDS and RDEPENDS. The current
455 implementation creates crappy graphs ;)
457 pkgs_to_build A list of packages that needs to be built
458 ignore_deps A list of names where processing of dependencies
459 should be stopped. e.g. dependencies that get
462 def myFilterProvider( providers, item):
464 Take a list of providers and filter according to environment
465 variables. In contrast to filterProviders we do not discriminate
466 and take PREFERRED_PROVIDER into account.
469 preferred_versions = {}
471 # Collate providers by PN
474 pn = self.status.pkg_fn[p]
479 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
481 for pn in pkg_pn.keys():
482 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
483 eligible.append(preferred_versions[pn][1])
486 if p in self.build_cache_fail:
487 bb.debug(1, "rejecting already-failed %s" % p)
490 if len(eligible) == 0:
491 bb.error("no eligible providers for %s" % item)
494 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % package, self.configuration.data, 1)
496 # try the preferred provider first
499 if prefervar == self.status.pkg_fn[p]:
500 bb.note("Selecting PREFERRED_PROVIDER %s" % prefervar)
502 elligible = [p] + elligible
509 rdepends_file = file('rdepends.dot', 'w' )
510 depends_file = file('depends.dot', 'w' )
511 alldepends_file = file('alldepends.dot', 'w' )
515 print >> depends_file, "digraph depends {"
516 print >> rdepends_file, "digraph rdepends {"
517 print >> alldepends_file, "digraph alldepends {"
519 # try to avoid adding the same rdepends over an over again
521 added_depends_error = False
523 def add_depends(package_list):
525 Add all depends of all packages from this list
527 for package in package_list:
528 if package in seen_depends or package in ignore_deps:
531 seen_depends[package] = 1
533 if not package in self.status.providers:
535 We have not seen this name -> error in
538 print >> depends_file, '"%(package)s" -> ERROR' % vars()
541 # get all providers for this package
542 providers = self.status.providers[package]
544 # now let us find the bestProvider for it
545 fn = self.filterProviders(providers, package)[0]
547 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
548 rdepends = bb.utils.explode_deps(self.bb_cache.getVar('RDEPENDS', fn, True) or "")
549 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
550 add_depends ( depends )
551 add_rdepends( rdepends )
553 # now create the node
554 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
556 depends = filter( (lambda x: x not in ignore_deps), depends )
557 for depend in depends:
558 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
560 def add_rdepends(package_list):
562 for each package of the package_list
563 we will see if we have handled it already
567 # start with the initial list
568 add_depends( pkgs_to_build )
571 print >> depends_file, "}"
572 print >> rdepends_file, "}"
573 print >> alldepends_file, "}"
575 def filterProviders(self, providers, item):
577 Take a list of providers and filter/reorder according to the
578 environment variables and previous build results
581 preferred_versions = {}
583 # Collate providers by PN
586 pn = self.status.pkg_fn[p]
591 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
593 for pn in pkg_pn.keys():
594 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
595 eligible.append(preferred_versions[pn][1])
598 if p in self.build_cache_fail:
599 bb.debug(1, "rejecting already-failed %s" % p)
602 if len(eligible) == 0:
603 bb.error("no eligible providers for %s" % item)
606 # look to see if one of them is already staged, or marked as preferred.
607 # if so, bump it to the head of the queue
609 pn = self.status.pkg_fn[p]
610 pv, pr = self.status.pkg_pvpr[p]
612 stamp = '%s.do_populate_staging' % self.status.stamp[p]
613 if os.path.exists(stamp):
614 (newvers, fn) = preferred_versions[pn]
615 if not fn in eligible:
616 # package was made ineligible by already-failed check
618 oldver = "%s-%s" % (pv, pr)
619 newver = '-'.join(newvers)
620 if (newver != oldver):
621 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
623 extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
624 if self.configuration.verbose:
625 bb.note("%s" % extra_chat)
627 eligible = [fn] + eligible
633 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
635 Build something to provide a named build requirement
636 (takes item names from DEPENDS namespace)
640 discriminated = False
642 if not item in self.status.providers:
643 bb.error("Nothing provides dependency %s" % item)
644 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
647 all_p = self.status.providers[item]
650 if p in self.build_cache:
651 bb.debug(1, "already built %s in this run\n" % p)
654 eligible = self.filterProviders(all_p, item)
659 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
661 self.preferred[item] = prefervar
663 if item in self.preferred:
665 pn = self.status.pkg_fn[p]
666 if self.preferred[item] == pn:
667 if self.configuration.verbose:
668 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
670 eligible = [p] + eligible
674 if len(eligible) > 1 and discriminated == False:
675 if item not in self.consider_msgs_cache:
678 providers_list.append(self.status.pkg_fn[fn])
679 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
680 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
681 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
682 self.consider_msgs_cache.append(item)
685 # run through the list until we find one that we can build
687 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
688 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
691 bb.note("no buildable providers for %s" % item)
692 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
695 def buildRProvider( self, item , buildAllDeps ):
697 Build something to provide a named runtime requirement
698 (takes item names from RDEPENDS/PACKAGES namespace)
703 discriminated = False
708 all_p = self.getProvidersRun(item)
711 bb.error("Nothing provides runtime dependency %s" % (item))
712 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True))
716 if p in self.rbuild_cache:
717 bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
719 if p in self.build_cache:
720 bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
721 return self.addRunDeps(p, item , buildAllDeps)
723 eligible = self.filterProviders(all_p, item)
729 pn = self.status.pkg_fn[p]
730 provides = self.status.pn_provides[pn]
731 for provide in provides:
732 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
734 if self.configuration.verbose:
735 bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
737 eligible = [p] + eligible
740 if len(eligible) > 1 and len(preferred) == 0:
741 if item not in self.consider_msgs_cache:
744 providers_list.append(self.status.pkg_fn[fn])
745 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
746 bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
747 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
748 self.consider_msgs_cache.append(item)
750 if len(preferred) > 1:
751 if item not in self.consider_msgs_cache:
754 providers_list.append(self.status.pkg_fn[fn])
755 bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
756 bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
757 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
758 self.consider_msgs_cache.append(item)
760 # run through the list until we find one that we can build
762 bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
763 if self.tryBuild(fn, item, buildAllDeps):
766 bb.error("No buildable providers for runtime %s" % item)
767 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
770 def getProvidersRun(self, rdepend):
772 Return any potential providers of runtime rdepend
776 if rdepend in self.status.rproviders:
777 rproviders += self.status.rproviders[rdepend]
779 if rdepend in self.status.packages:
780 rproviders += self.status.packages[rdepend]
785 # Only search dynamic packages if we can't find anything in other variables
786 for pattern in self.status.packages_dynamic:
787 regexp = re.compile(pattern)
788 if regexp.match(rdepend):
789 rproviders += self.status.packages_dynamic[pattern]
793 def addRunDeps(self , fn, item , buildAllDeps):
795 Add any runtime dependencies of runtime item provided by fn
796 as long as item has't previously been processed by this function.
799 if item in self.rbuild_cache:
806 self.rbuild_cache.append(item)
808 if fn in self.status.rundeps and item in self.status.rundeps[fn]:
809 rdepends += self.status.rundeps[fn][item].keys()
810 if fn in self.status.runrecs and item in self.status.runrecs[fn]:
811 rdepends += self.status.runrecs[fn][item].keys()
813 bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
815 for rdepend in rdepends:
816 if rdepend in self.status.ignored_dependencies:
818 if not self.buildRProvider(rdepend, buildAllDeps):
822 def buildDepgraph( self ):
823 all_depends = self.status.all_depends
824 pn_provides = self.status.pn_provides
826 localdata = data.createCopy(self.configuration.data)
827 bb.data.update_data(localdata)
829 def calc_bbfile_priority(filename):
830 for (regex, pri) in self.status.bbfile_config_priorities:
831 if regex.match(filename):
835 # Handle PREFERRED_PROVIDERS
836 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
837 (providee, provider) = p.split(':')
838 if providee in self.preferred and self.preferred[providee] != provider:
839 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
840 self.preferred[providee] = provider
842 # Calculate priorities for each file
843 for p in self.status.pkg_fn.keys():
844 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
846 def buildWorldTargetList(self):
848 Build package list for "bitbake world"
850 all_depends = self.status.all_depends
851 pn_provides = self.status.pn_provides
852 bb.debug(1, "collating packages for \"world\"")
853 for f in self.status.possible_world:
855 pn = self.status.pkg_fn[f]
857 for p in pn_provides[pn]:
858 if p.startswith('virtual/'):
859 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
862 for pf in self.status.providers[p]:
863 if self.status.pkg_fn[pf] != pn:
864 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
868 self.status.world_target.add(pn)
870 # drop reference count now
871 self.status.possible_world = None
872 self.status.all_depends = None
874 def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
875 # feed the status with new input
877 self.status.handle_bb_data(f, bb_cache, from_cache)
881 if os.isatty(sys.stdout.fileno()):
882 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
886 sys.stdout.write("Parsing .bb files, please wait...")
889 sys.stdout.write("done.")
892 def interactiveMode( self ):
893 """Drop off into a shell"""
896 except ImportError, details:
897 bb.fatal("Sorry, shell not available (%s)" % details )
899 bb.data.update_data( self.configuration.data )
903 def parseConfigurationFile( self, afile ):
905 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
907 # Add the handlers we inherited by INHERIT
908 # we need to do this manually as it is not guranteed
909 # we will pick up these classes... as we only INHERIT
910 # on .inc and .bb files but not on .conf
911 data = bb.data.createCopy( self.configuration.data )
912 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
913 for inherit in inherits:
914 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
916 # FIXME: This assumes that we included at least one .inc file
917 for var in bb.data.keys(data):
918 if bb.data.getVarFlag(var, 'handler', data):
919 bb.event.register(var,bb.data.getVar(var, data))
922 bb.fatal( "Unable to open %s" % afile )
923 except bb.parse.ParseError, details:
924 bb.fatal( "Unable to parse %s (%s)" % (afile, details) )
926 def handleCollections( self, collections ):
927 """Handle collections"""
929 collection_list = collections.split()
930 for c in collection_list:
931 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
933 bb.error("BBFILE_PATTERN_%s not defined" % c)
935 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
937 bb.error("BBFILE_PRIORITY_%s not defined" % c)
940 cre = re.compile(regex)
942 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
946 self.status.bbfile_config_priorities.append((cre, pri))
948 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
951 def cook( self, configuration, args ):
953 We are building stuff here. We do the building
954 from here. By default we try to execute task
958 self.configuration = configuration
960 if not self.configuration.cmd:
961 self.configuration.cmd = "build"
963 if self.configuration.debug:
964 bb.debug_level = self.configuration.debug
966 self.configuration.data = bb.data.init()
968 for f in self.configuration.file:
969 self.parseConfigurationFile( f )
971 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
975 # Special updated configuration we use for firing events
977 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
978 bb.data.update_data(self.configuration.event_data)
980 if self.configuration.show_environment:
981 self.showEnvironment()
984 # inject custom variables
985 if not bb.data.getVar("BUILDNAME", self.configuration.data):
986 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
987 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
989 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
991 if self.configuration.interactive:
992 self.interactiveMode()
994 if self.configuration.buildfile is not None:
995 bf = os.path.abspath( self.configuration.buildfile )
997 bbfile_data = bb.parse.handle(bf, self.configuration.data)
999 bb.fatal("Unable to open %s" % bf)
1001 item = bb.data.getVar('PN', bbfile_data, 1)
1003 self.tryBuildPackage( bf, item, bbfile_data )
1004 except bb.build.EventException:
1005 bb.error( "Build of '%s' failed" % item )
1007 sys.exit( self.stats.show() )
1009 # initialise the parsing status now we know we will need deps
1010 self.status = BBParsingStatus()
1012 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
1013 self.status.ignored_dependencies = Set( ignore.split() )
1015 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
1017 pkgs_to_build = None
1019 if not pkgs_to_build:
1021 pkgs_to_build.extend(args)
1022 if not pkgs_to_build:
1023 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
1025 pkgs_to_build = bbpkgs.split()
1026 if not pkgs_to_build and not self.configuration.show_versions \
1027 and not self.configuration.interactive \
1028 and not self.configuration.show_environment:
1029 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
1030 print "for usage information."
1033 # Import Psyco if available and not disabled
1034 if not self.configuration.disable_psyco:
1039 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
1041 psyco.bind( self.collect_bbfiles )
1043 bb.note("You have disabled Psyco. This decreases performance.")
1046 bb.debug(1, "collecting .bb files")
1047 self.collect_bbfiles( self.myProgressCallback )
1048 bb.debug(1, "parsing complete")
1051 if self.configuration.parse_only:
1052 print "Requested parsing .bb files only. Exiting."
1056 self.buildDepgraph()
1058 if self.configuration.show_versions:
1061 if 'world' in pkgs_to_build:
1062 self.buildWorldTargetList()
1063 pkgs_to_build.remove('world')
1064 for t in self.status.world_target:
1065 pkgs_to_build.append(t)
1067 if self.configuration.dot_graph:
1068 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
1072 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
1075 for k in pkgs_to_build:
1078 if self.buildProvider( k , False ) == 0:
1081 except bb.build.EventException:
1082 bb.error("Build of " + k + " failed")
1086 failures += failures
1087 if self.configuration.abort:
1090 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
1092 sys.exit( self.stats.show() )
1094 except KeyboardInterrupt:
1095 print "\nNOTE: KeyboardInterrupt - Build not completed."
1098 def get_bbfiles( self, path = os.getcwd() ):
1099 """Get list of default .bb files by reading out the current directory"""
1100 contents = os.listdir(path)
1103 (root, ext) = os.path.splitext(f)
1105 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
1108 def find_bbfiles( self, path ):
1109 """Find all the .bb files in a directory (uses find)"""
1110 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
1112 finddata = os.popen(findcmd)
1115 return finddata.readlines()
1117 def collect_bbfiles( self, progressCallback ):
1118 """Collect all available .bb build files"""
1119 self.cb = progressCallback
1120 parsed, cached, skipped, masked = 0, 0, 0, 0
1121 self.bb_cache = bb.cache.init(self)
1123 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
1124 data.setVar("BBFILES", " ".join(files), self.configuration.data)
1127 files = self.get_bbfiles()
1130 bb.error("no files to build.")
1134 if os.path.isdir(f):
1135 dirfiles = self.find_bbfiles(f)
1137 newfiles += dirfiles
1139 newfiles += glob.glob(f) or [ f ]
1141 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
1143 bbmask_compiled = re.compile(bbmask)
1144 except sre_constants.error:
1145 bb.fatal("BBMASK is not a valid regular expression.")
1147 for i in xrange( len( newfiles ) ):
1149 if bbmask and bbmask_compiled.search(f):
1150 bb.debug(1, "bbmake: skipping %s" % f)
1153 debug(1, "bbmake: parsing %s" % f)
1155 # read a file's metadata
1157 fromCache, skip = self.bb_cache.loadData(f, self)
1160 #bb.note("Skipping %s" % f)
1161 self.bb_cache.skip(f)
1163 elif fromCache: cached += 1
1167 # allow metadata files to add items to BBFILES
1168 #data.update_data(self.pkgdata[f])
1169 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1171 for aof in addbbfiles.split():
1172 if not files.count(aof):
1173 if not os.path.isabs(aof):
1174 aof = os.path.join(os.path.dirname(f),aof)
1177 # now inform the caller
1178 if self.cb is not None:
1179 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
1182 self.bb_cache.remove(f)
1183 bb.error("opening %s: %s" % (f, e))
1185 except KeyboardInterrupt:
1186 self.bb_cache.sync()
1188 except Exception, e:
1189 self.bb_cache.remove(f)
1190 bb.error("%s while parsing %s" % (e, f))
1192 self.bb_cache.remove(f)
1195 if self.cb is not None:
1196 print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ),
1198 self.bb_cache.sync()
1200 #============================================================================#
1202 #============================================================================#
1205 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
1206 usage = """%prog [options] [package ...]
1208 Executes the specified task (default is 'build') for a given set of BitBake files.
1209 It expects that BBFILES is defined, which is a space seperated list of files to
1210 be executed. BBFILES does support wildcards.
1211 Default BBFILES are the .bb files in the current directory.""" )
1213 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
1214 action = "store", dest = "buildfile", default = None )
1216 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
1217 action = "store_false", dest = "abort", default = True )
1219 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1220 action = "store_true", dest = "force", default = False )
1222 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1223 action = "store_true", dest = "interactive", default = False )
1225 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1226 action = "store", dest = "cmd", default = "build" )
1228 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1229 action = "append", dest = "file", default = [] )
1231 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1232 action = "store_true", dest = "verbose", default = False )
1234 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1235 action = "count", dest="debug", default = 0)
1237 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
1238 action = "store_true", dest = "dry_run", default = False )
1240 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
1241 action = "store_true", dest = "parse_only", default = False )
1243 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
1244 action = "store_true", dest = "disable_psyco", default = False )
1246 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
1247 action = "store_true", dest = "show_versions", default = False )
1249 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1250 action = "store_true", dest = "show_environment", default = False )
1252 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
1253 action = "store_true", dest = "dot_graph", default = False )
1254 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
1255 action = "append", dest = "ignored_dot_deps", default = [] )
1258 options, args = parser.parse_args( sys.argv )
1261 cooker.cook( BBConfiguration( options ), args[1:] )
1265 if __name__ == "__main__":
1266 print """WARNING, WARNING, WARNING
1267 This is a Bitbake from the Unstable/Development Branch.
1268 You might want to use the bitbake-1.4 stable branch (if you are not a BitBake developer or tester). I'm going to sleep 5 seconds now to make sure you see that."""