2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re, time
25 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
27 from bb import utils, data, parse, event, cache, providers
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
35 #============================================================================#
37 #============================================================================#
38 class BBParsingStatus:
40 The initial idea for this status class is to use the data when it is
41 already loaded instead of loading it from various place over and over
49 self.packages_dynamic = {}
50 self.bbfile_priority = {}
51 self.bbfile_config_priorities = []
52 self.ignored_dependencies = None
53 self.possible_world = []
54 self.world_target = Set()
60 self.all_depends = Set()
70 def handle_bb_data(self, file_name, bb_cache, cached):
72 We will fill the dictionaries with the stuff we
73 need for building the tree more fast
76 pn = bb_cache.getVar('PN', file_name, True)
77 pv = bb_cache.getVar('PV', file_name, True)
78 pr = bb_cache.getVar('PR', file_name, True)
79 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
80 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
81 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
82 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
83 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
84 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
86 self.task_queues[file_name] = bb_cache.getVar("_task_graph", file_name, True)
87 self.task_deps[file_name] = bb_cache.getVar("_task_deps", file_name, True)
89 # build PackageName to FileName lookup table
90 if pn not in self.pkg_pn:
92 self.pkg_pn[pn].append(file_name)
94 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
95 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
97 # build FileName to PackageName lookup table
98 self.pkg_fn[file_name] = pn
99 self.pkg_pvpr[file_name] = (pv,pr)
100 self.pkg_dp[file_name] = dp
102 # Build forward and reverse provider hashes
103 # Forward: virtual -> [filenames]
104 # Reverse: PN -> [virtuals]
105 if pn not in self.pn_provides:
106 self.pn_provides[pn] = Set()
107 self.pn_provides[pn] |= provides
109 for provide in provides:
110 if provide not in self.providers:
111 self.providers[provide] = []
112 self.providers[provide].append(file_name)
114 self.deps[file_name] = Set()
116 self.all_depends.add(dep)
117 self.deps[file_name].add(dep)
119 # Build reverse hash for PACKAGES, so runtime dependencies
120 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
121 for package in packages:
122 if not package in self.packages:
123 self.packages[package] = []
124 self.packages[package].append(file_name)
125 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
127 for package in packages_dynamic:
128 if not package in self.packages_dynamic:
129 self.packages_dynamic[package] = []
130 self.packages_dynamic[package].append(file_name)
132 for rprovide in rprovides:
133 if not rprovide in self.rproviders:
134 self.rproviders[rprovide] = []
135 self.rproviders[rprovide].append(file_name)
137 # Build hash of runtime depends and rececommends
139 def add_dep(deplist, deps):
141 if not dep in deplist:
144 if not file_name in self.rundeps:
145 self.rundeps[file_name] = {}
146 if not file_name in self.runrecs:
147 self.runrecs[file_name] = {}
149 for package in packages + [pn]:
150 if not package in self.rundeps[file_name]:
151 self.rundeps[file_name][package] = {}
152 if not package in self.runrecs[file_name]:
153 self.runrecs[file_name][package] = {}
155 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
156 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
157 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
158 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
160 # Collect files we may need for possible world-dep
162 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
163 self.possible_world.append(file_name)
166 #============================================================================#
168 #============================================================================#
171 Manage build statistics for one run
180 print "Build statistics:"
181 print " Attempted builds: %d" % self.attempt
183 print " Failed builds: %d" % self.fail
185 print " Dependencies not satisfied: %d" % self.deps
186 if self.fail or self.deps: return 1
190 #============================================================================#
192 #============================================================================#
193 class BBConfiguration( object ):
195 Manages build options and configurations for one run
197 def __init__( self, options ):
198 for key, val in options.__dict__.items():
199 setattr( self, key, val )
201 #============================================================================#
203 #============================================================================#
206 Manages one bitbake build run
209 ParsingStatus = BBParsingStatus # make it visible from the shell
210 Statistics = BBStatistics # make it visible from the shell
212 def __init__( self ):
213 self.build_cache_fail = []
214 self.build_cache = []
215 self.rbuild_cache = []
216 self.building_list = []
218 self.consider_msgs_cache = []
219 self.stats = BBStatistics()
225 def tryBuildPackage( self, fn, item, the_data ):
226 """Build one package"""
227 bb.event.fire(bb.event.PkgStarted(item, the_data))
229 self.stats.attempt += 1
230 if self.configuration.force:
231 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
232 if not self.configuration.dry_run:
233 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
234 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
235 self.build_cache.append(fn)
237 except bb.build.FuncFailed:
239 bb.msg.error(bb.msg.domain.Build, "task stack execution failed")
240 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn)
243 except bb.build.EventException, e:
246 bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event))
247 bb.event.fire(bb.event.PkgFailed(item, the_data))
248 self.build_cache_fail.append(fn)
251 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
253 Build a provider and its dependencies.
254 build_depends is a list of previous build dependencies (not runtime)
255 If build_depends is empty, we're dealing with a runtime depends
258 the_data = self.bb_cache.loadDataFull(fn, self)
260 # Only follow all (runtime) dependencies if doing a build
261 if not buildAllDeps and self.configuration.cmd is "build":
262 buildAllDeps = self.status.build_all[fn]
264 # Error on build time dependency loops
265 if build_depends and build_depends.count(fn) > 1:
266 bb.msg.error(bb.msg.domain.Depends, "%s depends on itself (eventually)" % fn)
267 bb.msg.error(bb.msg.domain.Depends, "upwards chain is: %s" % (" -> ".join(self.build_path)))
270 # See if this is a runtime dependency we've already built
271 # Or a build dependency being handled in a different build chain
272 if fn in self.building_list:
273 return self.addRunDeps(fn, virtual , buildAllDeps)
275 item = self.status.pkg_fn[fn]
277 self.building_list.append(fn)
279 pathstr = "%s (%s)" % (item, virtual)
280 self.build_path.append(pathstr)
282 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
284 bb.msg.note(2, bb.msg.domain.Depends, "current path: %s" % (" -> ".join(self.build_path)))
285 bb.msg.note(2, bb.msg.domain.Depends, "dependencies for %s are: %s" % (item, " ".join(depends_list)))
290 depcmd = self.configuration.cmd
291 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
292 if bbdepcmd is not None:
299 oldcmd = self.configuration.cmd
300 self.configuration.cmd = depcmd
302 for dependency in depends_list:
303 if dependency in self.status.ignored_dependencies:
307 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
308 bb.msg.error(bb.msg.domain.Depends, "dependency %s (for %s) not satisfied" % (dependency,item))
310 if self.configuration.abort:
314 self.configuration.cmd = oldcmd
320 if not self.addRunDeps(fn, virtual , buildAllDeps):
323 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
324 self.build_cache.append(fn)
327 return self.tryBuildPackage( fn, item, the_data )
330 self.building_list.remove(fn)
331 self.build_path.remove(pathstr)
334 def showVersions( self ):
335 pkg_pn = self.status.pkg_pn
336 preferred_versions = {}
340 for pn in pkg_pn.keys():
341 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn, self.configuration.data, self.status)
342 preferred_versions[pn] = (pref_ver, pref_file)
343 latest_versions[pn] = (last_ver, last_file)
345 pkg_list = pkg_pn.keys()
349 pref = preferred_versions[p]
350 latest = latest_versions[p]
353 prefstr = pref[0][0] + "-" + pref[0][1]
357 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
361 def showEnvironment( self ):
362 """Show the outer or per-package environment"""
363 if self.configuration.buildfile:
365 self.bb_cache = bb.cache.init(self)
367 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
369 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to read %s: %s" % ( self.configuration.buildfile, e ))
371 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
372 # emit variables and shell functions
374 data.update_data( self.configuration.data )
375 data.emit_env(sys.__stdout__, self.configuration.data, True)
377 bb.msg.fatal(bb.msg.domain.Parsing, "%s" % e)
378 # emit the metadata which isnt valid shell
379 for e in self.configuration.data.keys():
380 if data.getVarFlag( e, 'python', self.configuration.data ):
381 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
383 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
385 Generate two graphs one for the DEPENDS and RDEPENDS. The current
386 implementation creates crappy graphs ;)
388 pkgs_to_build A list of packages that needs to be built
389 ignore_deps A list of names where processing of dependencies
390 should be stopped. e.g. dependencies that get
393 def myFilterProvider( providers, item):
395 Take a list of providers and filter according to environment
396 variables. In contrast to filterProviders we do not discriminate
397 and take PREFERRED_PROVIDER into account.
400 preferred_versions = {}
402 # Collate providers by PN
405 pn = self.status.pkg_fn[p]
410 bb.msg.debug(1, bb.msg.domain.Provider, "providers for %s are: %s" % (item, pkg_pn.keys()))
412 for pn in pkg_pn.keys():
413 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
414 eligible.append(preferred_versions[pn][1])
417 if p in self.build_cache_fail:
418 bb.msg.debug(1, bb.msg.domain.Provider, "rejecting already-failed %s" % p)
421 if len(eligible) == 0:
422 bb.msg.error(bb.msg.domain.Provider, "no eligible providers for %s" % item)
425 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
427 # try the preferred provider first
430 if prefervar == self.status.pkg_fn[p]:
431 bb.msg.note(1, bb.msg.domain.Provider, "Selecting PREFERRED_PROVIDER %s" % prefervar)
433 eligible = [p] + eligible
440 # try to avoid adding the same rdepends over an over again
445 def add_depends(package_list):
447 Add all depends of all packages from this list
449 for package in package_list:
450 if package in seen_depends or package in ignore_deps:
453 seen_depends.append( package )
454 if not package in self.status.providers:
456 We have not seen this name -> error in
459 bb.note( "ERROR with provider: %(package)s" % vars() )
460 print >> depends_file, '"%(package)s" -> ERROR' % vars()
463 # get all providers for this package
464 providers = self.status.providers[package]
466 # now let us find the bestProvider for it
467 fn = myFilterProvider(providers, package)[0]
469 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
470 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
471 add_depends ( depends )
473 # now create the node
474 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
476 depends = filter( (lambda x: x not in ignore_deps), depends )
477 for depend in depends:
478 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
481 def add_all_depends( the_depends, the_rdepends ):
483 Add both DEPENDS and RDEPENDS. RDEPENDS will get dashed
486 package_list = the_depends + the_rdepends
487 for package in package_list:
488 if package in seen_rdepends or package in ignore_deps:
491 seen_rdepends.append( package )
493 # Let us find out if the package is a DEPENDS or RDEPENDS
494 # and we will set 'providers' with the avilable providers
496 if package in the_depends:
497 if not package in self.status.providers:
498 bb.note( "ERROR with provider: %(package)s" % vars() )
499 print >> alldepends_file, '"%(package)s" -> ERROR' % vars()
502 providers = self.status.providers[package]
503 elif package in the_rdepends:
504 if len(self.getProvidersRun(package)) == 0:
505 bb.note( "ERROR with rprovider: %(package)s" % vars() )
506 print >> alldepends_file, '"%(package)s" -> ERROR [style="dashed"]' % vars()
509 providers = self.getProvidersRun(package)
511 # something went wrong...
512 print "Complete ERROR! %s" % package
515 # now let us find the bestProvider for it
516 fn = myFilterProvider(providers, package)[0]
518 # Now we have a filename let us get the depends and RDEPENDS of it
519 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
520 if fn in self.status.rundeps and package in self.status.rundeps[fn]:
521 rdepends= self.status.rundeps[fn][package].keys()
524 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
526 # handle all the depends and rdepends of package
527 add_all_depends ( depends, rdepends )
529 # now create the node using package name
530 print >> alldepends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
532 # remove the stuff we want to ignore and add the edges
533 depends = filter( (lambda x: x not in ignore_deps), depends )
534 rdepends = filter( (lambda x: x not in ignore_deps), rdepends )
535 for depend in depends:
536 print >> alldepends_file, '"%(package)s" -> "%(depend)s"' % vars()
537 for depend in rdepends:
538 print >> alldepends_file, '"%(package)s" -> "%(depend)s" [style=dashed]' % vars()
542 depends_file = file('depends.dot', 'w' )
543 print >> depends_file, "digraph depends {"
544 add_depends( pkgs_to_build )
545 print >> depends_file, "}"
547 # Add all depends now
548 alldepends_file = file('alldepends.dot', 'w' )
549 print >> alldepends_file, "digraph alldepends {"
550 add_all_depends( pkgs_to_build, [] )
551 print >> alldepends_file, "}"
553 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
555 Build something to provide a named build requirement
556 (takes item names from DEPENDS namespace)
560 discriminated = False
562 if not item in self.status.providers:
563 bb.msg.error(bb.msg.domain.Depends, "Nothing provides dependency %s" % item)
564 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
567 all_p = self.status.providers[item]
570 if p in self.build_cache:
571 bb.msg.debug(1, bb.msg.domain.Provider, "already built %s in this run" % p)
574 eligible = bb.providers.filterProviders(all_p, item, self.configuration.data, self.status, self.build_cache_fail)
579 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
581 self.status.preferred[item] = prefervar
583 if item in self.status.preferred:
585 pn = self.status.pkg_fn[p]
586 if self.status.preferred[item] == pn:
587 bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
589 eligible = [p] + eligible
593 if len(eligible) > 1 and discriminated == False:
594 if item not in self.consider_msgs_cache:
597 providers_list.append(self.status.pkg_fn[fn])
598 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
599 bb.msg.note(1, bb.msg.domain.Provider, "consider defining PREFERRED_PROVIDER_%s" % item)
600 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
601 self.consider_msgs_cache.append(item)
604 # run through the list until we find one that we can build
606 bb.msg.debug(2, bb.msg.domain.Provider, "selecting %s to satisfy %s" % (fn, item))
607 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
610 bb.msg.note(1, bb.msg.domain.Provider, "no buildable providers for %s" % item)
611 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
614 def buildRProvider( self, item , buildAllDeps ):
616 Build something to provide a named runtime requirement
617 (takes item names from RDEPENDS/PACKAGES namespace)
622 discriminated = False
627 all_p = self.getProvidersRun(item)
630 bb.msg.error(bb.msg.domain.Provider, "Nothing provides runtime dependency %s" % (item))
631 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True))
635 if p in self.rbuild_cache:
636 bb.msg.debug(2, bb.msg.domain.Provider, "Already built %s providing runtime %s" % (p,item))
638 if p in self.build_cache:
639 bb.msg.debug(2, bb.msg.domain.Provider, "Already built %s but adding any further RDEPENDS for %s" % (p, item))
640 return self.addRunDeps(p, item , buildAllDeps)
642 eligible = bb.providers.filterProviders(all_p, item, self.configuration.data, self.status, self.build_cache_fail)
648 pn = self.status.pkg_fn[p]
649 provides = self.status.pn_provides[pn]
650 for provide in provides:
651 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
653 bb.msg.note(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
655 eligible = [p] + eligible
658 if len(eligible) > 1 and len(preferred) == 0:
659 if item not in self.consider_msgs_cache:
662 providers_list.append(self.status.pkg_fn[fn])
663 bb.msg.note(1, bb.msg.domain.Provider, "multiple providers are available (%s);" % ", ".join(providers_list))
664 bb.msg.note(1, bb.msg.domain.Provider, "consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
665 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
666 self.consider_msgs_cache.append(item)
668 if len(preferred) > 1:
669 if item not in self.consider_msgs_cache:
672 providers_list.append(self.status.pkg_fn[fn])
673 bb.msg.note(1, bb.msg.domain.Provider, "multiple preferred providers are available (%s);" % ", ".join(providers_list))
674 bb.msg.note(1, bb.msg.domain.Provider, "consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
675 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
676 self.consider_msgs_cache.append(item)
678 # run through the list until we find one that we can build
680 bb.msg.debug(2, bb.msg.domain.Provider, "selecting %s to satisfy runtime %s" % (fn, item))
681 if self.tryBuild(fn, item, buildAllDeps):
684 bb.msg.error(bb.msg.domain.Provider, "No buildable providers for runtime %s" % item)
685 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
688 def getProvidersRun(self, rdepend):
690 Return any potential providers of runtime rdepend
694 if rdepend in self.status.rproviders:
695 rproviders += self.status.rproviders[rdepend]
697 if rdepend in self.status.packages:
698 rproviders += self.status.packages[rdepend]
703 # Only search dynamic packages if we can't find anything in other variables
704 for pattern in self.status.packages_dynamic:
705 regexp = re.compile(pattern)
706 if regexp.match(rdepend):
707 rproviders += self.status.packages_dynamic[pattern]
711 def addRunDeps(self , fn, item , buildAllDeps):
713 Add any runtime dependencies of runtime item provided by fn
714 as long as item has't previously been processed by this function.
717 if item in self.rbuild_cache:
724 self.rbuild_cache.append(item)
726 if fn in self.status.rundeps and item in self.status.rundeps[fn]:
727 rdepends += self.status.rundeps[fn][item].keys()
728 if fn in self.status.runrecs and item in self.status.runrecs[fn]:
729 rdepends += self.status.runrecs[fn][item].keys()
731 bb.msg.debug(2, bb.msg.domain.Provider, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
733 for rdepend in rdepends:
734 if rdepend in self.status.ignored_dependencies:
736 if not self.buildRProvider(rdepend, buildAllDeps):
740 def buildDepgraph( self ):
741 all_depends = self.status.all_depends
742 pn_provides = self.status.pn_provides
744 localdata = data.createCopy(self.configuration.data)
745 bb.data.update_data(localdata)
747 def calc_bbfile_priority(filename):
748 for (regex, pri) in self.status.bbfile_config_priorities:
749 if regex.match(filename):
753 # Handle PREFERRED_PROVIDERS
754 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
755 (providee, provider) = p.split(':')
756 if providee in self.status.preferred and self.status.preferred[providee] != provider:
757 bb.msg.error(bb.msg.domain.Provider, "conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.status.preferred[providee]))
758 self.status.preferred[providee] = provider
760 # Calculate priorities for each file
761 for p in self.status.pkg_fn.keys():
762 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
764 def buildWorldTargetList(self):
766 Build package list for "bitbake world"
768 all_depends = self.status.all_depends
769 pn_provides = self.status.pn_provides
770 bb.msg.debug(1, bb.msg.domain.Parsing, "collating packages for \"world\"")
771 for f in self.status.possible_world:
773 pn = self.status.pkg_fn[f]
775 for p in pn_provides[pn]:
776 if p.startswith('virtual/'):
777 bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to %s provider starting with virtual/" % (f, p))
780 for pf in self.status.providers[p]:
781 if self.status.pkg_fn[pf] != pn:
782 bb.msg.debug(2, bb.msg.domain.Parsing, "World build skipping %s due to both us and %s providing %s" % (f, pf, p))
786 self.status.world_target.add(pn)
788 # drop reference count now
789 self.status.possible_world = None
790 self.status.all_depends = None
792 def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
793 # feed the status with new input
795 self.status.handle_bb_data(f, bb_cache, from_cache)
797 if os.isatty(sys.stdout.fileno()):
798 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
802 sys.stdout.write("Parsing .bb files, please wait...")
805 sys.stdout.write("done.")
808 def interactiveMode( self ):
809 """Drop off into a shell"""
812 except ImportError, details:
813 bb.msg.fatal(bb.msg.domain.Parsing, "Sorry, shell not available (%s)" % details )
815 bb.data.update_data( self.configuration.data )
819 def parseConfigurationFile( self, afile ):
821 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
823 # Add the handlers we inherited by INHERIT
824 # we need to do this manually as it is not guranteed
825 # we will pick up these classes... as we only INHERIT
826 # on .inc and .bb files but not on .conf
827 data = bb.data.createCopy( self.configuration.data )
828 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
829 for inherit in inherits:
830 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
832 # FIXME: This assumes that we included at least one .inc file
833 for var in bb.data.keys(data):
834 if bb.data.getVarFlag(var, 'handler', data):
835 bb.event.register(var,bb.data.getVar(var, data))
838 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % afile )
839 except bb.parse.ParseError, details:
840 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to parse %s (%s)" % (afile, details) )
842 def handleCollections( self, collections ):
843 """Handle collections"""
845 collection_list = collections.split()
846 for c in collection_list:
847 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
849 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s not defined" % c)
851 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
853 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PRIORITY_%s not defined" % c)
856 cre = re.compile(regex)
858 bb.msg.error(bb.msg.domain.Parsing, "BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
862 self.status.bbfile_config_priorities.append((cre, pri))
864 bb.msg.error(bb.msg.domain.Parsing, "invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
867 def cook( self, configuration, args ):
869 We are building stuff here. We do the building
870 from here. By default we try to execute task
874 self.configuration = configuration
876 if self.configuration.verbose:
877 bb.msg.set_verbose(True)
879 if not self.configuration.cmd:
880 self.configuration.cmd = "build"
882 if self.configuration.debug:
883 bb.msg.set_debug_level(self.configuration.debug)
885 self.configuration.data = bb.data.init()
887 for f in self.configuration.file:
888 self.parseConfigurationFile( f )
890 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
894 # Special updated configuration we use for firing events
896 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
897 bb.data.update_data(self.configuration.event_data)
899 if self.configuration.show_environment:
900 self.showEnvironment()
903 # inject custom variables
904 if not bb.data.getVar("BUILDNAME", self.configuration.data):
905 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
906 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
908 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
910 if self.configuration.interactive:
911 self.interactiveMode()
913 if self.configuration.buildfile is not None:
914 bf = os.path.abspath( self.configuration.buildfile )
916 bbfile_data = bb.parse.handle(bf, self.configuration.data)
918 bb.msg.fatal(bb.msg.domain.Parsing, "Unable to open %s" % bf)
920 item = bb.data.getVar('PN', bbfile_data, 1)
922 self.tryBuildPackage( bf, item, bbfile_data )
923 except bb.build.EventException:
924 bb.msg.error(bb.msg.domain.Build, "Build of '%s' failed" % item )
926 sys.exit( self.stats.show() )
928 # initialise the parsing status now we know we will need deps
929 self.status = BBParsingStatus()
931 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
932 self.status.ignored_dependencies = Set( ignore.split() )
934 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
938 if not pkgs_to_build:
940 pkgs_to_build.extend(args)
941 if not pkgs_to_build:
942 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
944 pkgs_to_build = bbpkgs.split()
945 if not pkgs_to_build and not self.configuration.show_versions \
946 and not self.configuration.interactive \
947 and not self.configuration.show_environment:
948 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
949 print "for usage information."
952 # Import Psyco if available and not disabled
953 if not self.configuration.disable_psyco:
957 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
959 psyco.bind( self.collect_bbfiles )
961 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
964 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
965 self.collect_bbfiles( self.myProgressCallback )
966 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
968 if self.configuration.parse_only:
969 bb.msg.note(1, bb.msg.domain.Collection, "Requested parsing .bb files only. Exiting.")
975 if self.configuration.show_versions:
978 if 'world' in pkgs_to_build:
979 self.buildWorldTargetList()
980 pkgs_to_build.remove('world')
981 for t in self.status.world_target:
982 pkgs_to_build.append(t)
984 if self.configuration.dot_graph:
985 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
989 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
992 for k in pkgs_to_build:
995 if self.buildProvider( k , False ) == 0:
998 except bb.build.EventException:
999 bb.msg.error(bb.msg.domain.Build, "Build of " + k + " failed")
1003 failures += failures
1004 if self.configuration.abort:
1007 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
1009 sys.exit( self.stats.show() )
1011 except KeyboardInterrupt:
1012 bb.msg.note(1, bb.msg.domain.Collection, "KeyboardInterrupt - Build not completed.")
1015 def get_bbfiles( self, path = os.getcwd() ):
1016 """Get list of default .bb files by reading out the current directory"""
1017 contents = os.listdir(path)
1020 (root, ext) = os.path.splitext(f)
1022 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
1025 def find_bbfiles( self, path ):
1026 """Find all the .bb files in a directory (uses find)"""
1027 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
1029 finddata = os.popen(findcmd)
1032 return finddata.readlines()
1034 def collect_bbfiles( self, progressCallback ):
1035 """Collect all available .bb build files"""
1036 self.cb = progressCallback
1037 parsed, cached, skipped, masked = 0, 0, 0, 0
1038 self.bb_cache = bb.cache.init(self)
1040 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
1041 data.setVar("BBFILES", " ".join(files), self.configuration.data)
1044 files = self.get_bbfiles()
1047 bb.msg.error(bb.msg.domain.Collection, "no files to build.")
1051 if os.path.isdir(f):
1052 dirfiles = self.find_bbfiles(f)
1054 newfiles += dirfiles
1056 newfiles += glob.glob(f) or [ f ]
1058 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
1060 bbmask_compiled = re.compile(bbmask)
1061 except sre_constants.error:
1062 bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
1064 for i in xrange( len( newfiles ) ):
1066 if bbmask and bbmask_compiled.search(f):
1067 bb.msg.debug(1, bb.msg.domain.Collection, "bbmake: skipping %s" % f, f)
1070 bb.msg.debug(1, bb.msg.domain.Collection, "bbmake: parsing %s" % f, f)
1072 # read a file's metadata
1074 fromCache, skip = self.bb_cache.loadData(f, self)
1077 bb.msg.debug(2, bb.msg.domain.Collection, "Skipping %s" % f, f)
1078 self.bb_cache.skip(f)
1080 elif fromCache: cached += 1
1084 # allow metadata files to add items to BBFILES
1085 #data.update_data(self.pkgdata[f])
1086 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1088 for aof in addbbfiles.split():
1089 if not files.count(aof):
1090 if not os.path.isabs(aof):
1091 aof = os.path.join(os.path.dirname(f),aof)
1094 # now inform the caller
1095 if self.cb is not None:
1096 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
1099 self.bb_cache.remove(f)
1100 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e), f)
1102 except KeyboardInterrupt:
1103 self.bb_cache.sync()
1105 except Exception, e:
1106 self.bb_cache.remove(f)
1107 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f), f)
1109 self.bb_cache.remove(f)
1112 if self.cb is not None:
1113 print "\r" # need newline after Handling Bitbake files message
1114 bb.msg.note(1, bb.msg.domain.Collection, "Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ))
1116 self.bb_cache.sync()
1118 #============================================================================#
1120 #============================================================================#
1123 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
1124 usage = """%prog [options] [package ...]
1126 Executes the specified task (default is 'build') for a given set of BitBake files.
1127 It expects that BBFILES is defined, which is a space seperated list of files to
1128 be executed. BBFILES does support wildcards.
1129 Default BBFILES are the .bb files in the current directory.""" )
1131 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
1132 action = "store", dest = "buildfile", default = None )
1134 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
1135 action = "store_false", dest = "abort", default = True )
1137 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1138 action = "store_true", dest = "force", default = False )
1140 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1141 action = "store_true", dest = "interactive", default = False )
1143 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1144 action = "store", dest = "cmd", default = "build" )
1146 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1147 action = "append", dest = "file", default = [] )
1149 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1150 action = "store_true", dest = "verbose", default = False )
1152 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1153 action = "count", dest="debug", default = 0)
1155 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
1156 action = "store_true", dest = "dry_run", default = False )
1158 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
1159 action = "store_true", dest = "parse_only", default = False )
1161 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
1162 action = "store_true", dest = "disable_psyco", default = False )
1164 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
1165 action = "store_true", dest = "show_versions", default = False )
1167 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1168 action = "store_true", dest = "show_environment", default = False )
1170 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
1171 action = "store_true", dest = "dot_graph", default = False )
1172 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
1173 action = "append", dest = "ignored_dot_deps", default = [] )
1176 options, args = parser.parse_args( sys.argv )
1179 cooker.cook( BBConfiguration( options ), args[1:] )
1183 if __name__ == "__main__":
1184 print """WARNING, WARNING, WARNING
1185 This is a Bitbake from the Unstable/Development Branch.
1186 You might want to use the bitbake-1.4 stable branch (if you are not a BitBake developer or tester). I'm going to sleep 5 seconds now to make sure you see that."""