2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re, time
25 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
27 from bb import utils, data, parse, debug, event, fatal, cache
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
50 self.packages_dynamic = {}
51 self.bbfile_priority = {}
52 self.bbfile_config_priorities = []
53 self.ignored_dependencies = None
54 self.possible_world = []
55 self.world_target = Set()
61 self.all_depends = Set()
67 def handle_bb_data(self, file_name, bb_cache, cached):
69 We will fill the dictionaries with the stuff we
70 need for building the tree more fast
73 pn = bb_cache.getVar('PN', file_name, True)
74 pv = bb_cache.getVar('PV', file_name, True)
75 pr = bb_cache.getVar('PR', file_name, True)
76 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
77 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
78 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
79 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
80 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
81 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
83 # build PackageName to FileName lookup table
84 if pn not in self.pkg_pn:
86 self.pkg_pn[pn].append(file_name)
88 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
89 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
91 # build FileName to PackageName lookup table
92 self.pkg_fn[file_name] = pn
93 self.pkg_pvpr[file_name] = (pv,pr)
94 self.pkg_dp[file_name] = dp
96 # Build forward and reverse provider hashes
97 # Forward: virtual -> [filenames]
98 # Reverse: PN -> [virtuals]
99 if pn not in self.pn_provides:
100 self.pn_provides[pn] = Set()
101 self.pn_provides[pn] |= provides
103 for provide in provides:
104 if provide not in self.providers:
105 self.providers[provide] = []
106 self.providers[provide].append(file_name)
109 self.all_depends.add(dep)
111 # Build reverse hash for PACKAGES, so runtime dependencies
112 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
113 for package in packages:
114 if not package in self.packages:
115 self.packages[package] = []
116 self.packages[package].append(file_name)
117 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
119 for package in packages_dynamic:
120 if not package in self.packages_dynamic:
121 self.packages_dynamic[package] = []
122 self.packages_dynamic[package].append(file_name)
124 for rprovide in rprovides:
125 if not rprovide in self.rproviders:
126 self.rproviders[rprovide] = []
127 self.rproviders[rprovide].append(file_name)
129 # Build hash of runtime depeneds and rececommends
131 def add_dep(deplist, deps):
133 if not dep in deplist:
136 if not file_name in self.rundeps:
137 self.rundeps[file_name] = {}
138 if not file_name in self.runrecs:
139 self.runrecs[file_name] = {}
141 for package in packages + [pn]:
142 if not package in self.rundeps[file_name]:
143 self.rundeps[file_name][package] = {}
144 if not package in self.runrecs[file_name]:
145 self.runrecs[file_name][package] = {}
147 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
148 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
149 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
150 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
152 # Collect files we may need for possible world-dep
154 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
155 self.possible_world.append(file_name)
158 #============================================================================#
160 #============================================================================#
163 Manage build statistics for one run
172 print "Build statistics:"
173 print " Attempted builds: %d" % self.attempt
175 print " Failed builds: %d" % self.fail
177 print " Dependencies not satisfied: %d" % self.deps
178 if self.fail or self.deps: return 1
182 #============================================================================#
184 #============================================================================#
185 class BBConfiguration( object ):
187 Manages build options and configurations for one run
189 def __init__( self, options ):
190 for key, val in options.__dict__.items():
191 setattr( self, key, val )
193 #============================================================================#
195 #============================================================================#
198 Manages one bitbake build run
201 ParsingStatus = BBParsingStatus # make it visible from the shell
202 Statistics = BBStatistics # make it visible from the shell
204 def __init__( self ):
205 self.build_cache_fail = []
206 self.build_cache = []
207 self.rbuild_cache = []
208 self.building_list = []
210 self.consider_msgs_cache = []
212 self.stats = BBStatistics()
218 def tryBuildPackage( self, fn, item, the_data ):
219 """Build one package"""
220 bb.event.fire(bb.event.PkgStarted(item, the_data))
222 self.stats.attempt += 1
223 if self.configuration.force:
224 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
225 if not self.configuration.dry_run:
226 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
227 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
228 self.build_cache.append(fn)
230 except bb.build.FuncFailed:
232 bb.error("task stack execution failed")
233 bb.event.fire(bb.event.PkgFailed(item, the_data))
234 self.build_cache_fail.append(fn)
236 except bb.build.EventException, e:
239 bb.error("%s event exception, aborting" % bb.event.getName(event))
240 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn)
244 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
246 Build a provider and its dependencies.
247 build_depends is a list of previous build dependencies (not runtime)
248 If build_depends is empty, we're dealing with a runtime depends
251 the_data = self.bb_cache.loadDataFull(fn, self)
253 # Only follow all (runtime) dependencies if doing a build
254 if not buildAllDeps and self.configuration.cmd is "build":
255 buildAllDeps = self.status.build_all[fn]
257 # Error on build time dependency loops
258 if build_depends and build_depends.count(fn) > 1:
259 bb.error("%s depends on itself (eventually)" % fn)
260 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
263 # See if this is a runtime dependency we've already built
264 # Or a build dependency being handled in a different build chain
265 if fn in self.building_list:
266 return self.addRunDeps(fn, virtual , buildAllDeps)
268 item = self.status.pkg_fn[fn]
270 self.building_list.append(fn)
272 pathstr = "%s (%s)" % (item, virtual)
273 self.build_path.append(pathstr)
275 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
277 if self.configuration.verbose:
278 bb.note("current path: %s" % (" -> ".join(self.build_path)))
279 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
284 depcmd = self.configuration.cmd
285 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
286 if bbdepcmd is not None:
293 oldcmd = self.configuration.cmd
294 self.configuration.cmd = depcmd
296 for dependency in depends_list:
297 if dependency in self.status.ignored_dependencies:
301 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
302 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
304 if self.configuration.abort:
308 self.configuration.cmd = oldcmd
314 if not self.addRunDeps(fn, virtual , buildAllDeps):
317 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
318 self.build_cache.append(fn)
321 return self.tryBuildPackage( fn, item, the_data )
324 self.building_list.remove(fn)
325 self.build_path.remove(pathstr)
327 def findBestProvider( self, pn, pkg_pn = None):
329 If there is a PREFERRED_VERSION, find the highest-priority bbfile
330 providing that version. If not, find the latest version provided by
331 an bbfile in the highest-priority set.
334 pkg_pn = self.status.pkg_pn
339 priority = self.status.bbfile_priority[f]
340 if priority not in priorities:
341 priorities[priority] = []
342 priorities[priority].append(f)
343 p_list = priorities.keys()
344 p_list.sort(lambda a, b: a - b)
347 tmp_pn = [priorities[p]] + tmp_pn
349 preferred_file = None
351 localdata = data.createCopy(self.configuration.data)
352 bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
353 bb.data.update_data(localdata)
355 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
357 m = re.match('(.*)_(.*)', preferred_v)
359 preferred_v = m.group(1)
360 preferred_r = m.group(2)
364 for file_set in tmp_pn:
366 pv,pr = self.status.pkg_pvpr[f]
367 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
369 preferred_ver = (pv, pr)
374 pv_str = '%s-%s' % (preferred_v, preferred_r)
377 if preferred_file is None:
378 bb.note("preferred version %s of %s not available" % (pv_str, pn))
380 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
384 # get highest priority file set
389 for file_name in files:
390 pv,pr = self.status.pkg_pvpr[file_name]
391 dp = self.status.pkg_dp[file_name]
393 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
397 if preferred_file is None:
398 preferred_file = latest_f
399 preferred_ver = latest
401 return (latest,latest_f,preferred_ver, preferred_file)
403 def showVersions( self ):
404 pkg_pn = self.status.pkg_pn
405 preferred_versions = {}
409 for pn in pkg_pn.keys():
410 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
411 preferred_versions[pn] = (pref_ver, pref_file)
412 latest_versions[pn] = (last_ver, last_file)
414 pkg_list = pkg_pn.keys()
418 pref = preferred_versions[p]
419 latest = latest_versions[p]
422 prefstr = pref[0][0] + "-" + pref[0][1]
426 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
430 def showEnvironment( self ):
431 """Show the outer or per-package environment"""
432 if self.configuration.buildfile:
434 self.bb_cache = bb.cache.init(self)
436 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
438 fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
441 # emit variables and shell functions
443 data.update_data( self.configuration.data )
444 data.emit_env(sys.__stdout__, self.configuration.data, True)
447 # emit the metadata which isnt valid shell
448 for e in self.configuration.data.keys():
449 if data.getVarFlag( e, 'python', self.configuration.data ):
450 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
452 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
454 Generate two graphs one for the DEPENDS and RDEPENDS. The current
455 implementation creates crappy graphs ;)
457 pkgs_to_build A list of packages that needs to be built
458 ignore_deps A list of names where processing of dependencies
459 should be stopped. e.g. dependencies that get
462 def myFilterProvider( providers, item):
464 Take a list of providers and filter according to environment
465 variables. In contrast to filterProviders we do not discriminate
466 and take PREFERRED_PROVIDER into account.
469 preferred_versions = {}
471 # Collate providers by PN
474 pn = self.status.pkg_fn[p]
479 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
481 for pn in pkg_pn.keys():
482 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
483 eligible.append(preferred_versions[pn][1])
486 if p in self.build_cache_fail:
487 bb.debug(1, "rejecting already-failed %s" % p)
490 if len(eligible) == 0:
491 bb.error("no eligible providers for %s" % item)
494 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % package, self.configuration.data, 1)
496 # try the preferred provider first
499 if prefervar == self.status.pkg_fn[p]:
500 bb.note("Selecting PREFERRED_PROVIDER %s" % prefervar)
502 elligible = [p] + elligible
509 depends_file = file('depends.dot', 'w' )
510 alldepends_file = file('alldepends.dot', 'w' )
514 print >> depends_file, "digraph depends {"
515 print >> alldepends_file, "digraph alldepends {"
517 # try to avoid adding the same rdepends over an over again
522 def add_depends(package_list):
524 Add all depends of all packages from this list
526 package_list = filter( lambda x: x not in seen_depends and not x in ignore_deps, package_list )
528 for package in package_list:
529 seen_depends.append( package )
530 if not package in self.status.providers:
532 We have not seen this name -> error in
535 bb.note( "ERROR with provider: %(package)s" % vars() )
536 print >> depends_file, '"%(package)s" -> ERROR' % vars()
539 # get all providers for this package
540 providers = self.status.providers[package]
542 # now let us find the bestProvider for it
543 fn = self.filterProviders(providers, package)[0]
545 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
546 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
547 add_depends ( depends )
549 # now create the node
550 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
552 depends = filter( (lambda x: x not in ignore_deps), depends )
553 for depend in depends:
554 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
557 def add_all_depends( package_list ):
559 Add both DEPENDS and RDEPENDS. RDEPENDS will get dashed
562 package_list = filter( lambda x: x not in seen_rdepends and not x in ignore_deps, package_list )
564 for package in package_list:
565 seen_rdepends.append( package )
566 if not package in self.status.providers:
568 We have not seen this name -> error in
571 bb.note( "ERROR with provider: %(package)s" % vars() )
572 print >> alldepends_file, '"%(package)s" -> ERROR' % vars()
575 # get all providers for this package
576 providers = self.status.providers[package]
578 # now let us find the bestProvider for it
579 fn = self.filterProviders(providers, package)[0]
581 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
582 rdepends = bb.utils.explode_deps(self.bb_cache.getVar('RDEPENDS', fn, True) or "")
583 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
584 add_all_depends ( depends+rdepends )
586 # now create the node
587 print >> alldepends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
589 depends = filter( (lambda x: x not in ignore_deps), depends )
590 rdepends = filter( (lambda x: x not in ignore_deps), rdepends )
591 for depend in depends:
592 print >> alldepends_file, '"%(package)s" -> "%(depend)s"' % vars()
593 for depend in rdepends:
594 print >> alldepends_file, '"%(package)s" -> "%(depend)s" [style=dashed]' % vars()
599 # start with the initial list
600 add_depends( pkgs_to_build )
601 add_all_depends( pkgs_to_build )
604 print >> depends_file, "}"
605 print >> alldepends_file, "}"
607 def filterProviders(self, providers, item):
609 Take a list of providers and filter/reorder according to the
610 environment variables and previous build results
613 preferred_versions = {}
615 # Collate providers by PN
618 pn = self.status.pkg_fn[p]
623 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
625 for pn in pkg_pn.keys():
626 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
627 eligible.append(preferred_versions[pn][1])
630 if p in self.build_cache_fail:
631 bb.debug(1, "rejecting already-failed %s" % p)
634 if len(eligible) == 0:
635 bb.error("no eligible providers for %s" % item)
638 # look to see if one of them is already staged, or marked as preferred.
639 # if so, bump it to the head of the queue
641 pn = self.status.pkg_fn[p]
642 pv, pr = self.status.pkg_pvpr[p]
644 stamp = '%s.do_populate_staging' % self.status.stamp[p]
645 if os.path.exists(stamp):
646 (newvers, fn) = preferred_versions[pn]
647 if not fn in eligible:
648 # package was made ineligible by already-failed check
650 oldver = "%s-%s" % (pv, pr)
651 newver = '-'.join(newvers)
652 if (newver != oldver):
653 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
655 extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
656 if self.configuration.verbose:
657 bb.note("%s" % extra_chat)
659 eligible = [fn] + eligible
665 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
667 Build something to provide a named build requirement
668 (takes item names from DEPENDS namespace)
672 discriminated = False
674 if not item in self.status.providers:
675 bb.error("Nothing provides dependency %s" % item)
676 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
679 all_p = self.status.providers[item]
682 if p in self.build_cache:
683 bb.debug(1, "already built %s in this run\n" % p)
686 eligible = self.filterProviders(all_p, item)
691 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
693 self.preferred[item] = prefervar
695 if item in self.preferred:
697 pn = self.status.pkg_fn[p]
698 if self.preferred[item] == pn:
699 if self.configuration.verbose:
700 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
702 eligible = [p] + eligible
706 if len(eligible) > 1 and discriminated == False:
707 if item not in self.consider_msgs_cache:
710 providers_list.append(self.status.pkg_fn[fn])
711 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
712 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
713 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
714 self.consider_msgs_cache.append(item)
717 # run through the list until we find one that we can build
719 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
720 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
723 bb.note("no buildable providers for %s" % item)
724 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
727 def buildRProvider( self, item , buildAllDeps ):
729 Build something to provide a named runtime requirement
730 (takes item names from RDEPENDS/PACKAGES namespace)
735 discriminated = False
740 all_p = self.getProvidersRun(item)
743 bb.error("Nothing provides runtime dependency %s" % (item))
744 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True))
748 if p in self.rbuild_cache:
749 bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
751 if p in self.build_cache:
752 bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
753 return self.addRunDeps(p, item , buildAllDeps)
755 eligible = self.filterProviders(all_p, item)
761 pn = self.status.pkg_fn[p]
762 provides = self.status.pn_provides[pn]
763 for provide in provides:
764 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
766 if self.configuration.verbose:
767 bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
769 eligible = [p] + eligible
772 if len(eligible) > 1 and len(preferred) == 0:
773 if item not in self.consider_msgs_cache:
776 providers_list.append(self.status.pkg_fn[fn])
777 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
778 bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
779 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
780 self.consider_msgs_cache.append(item)
782 if len(preferred) > 1:
783 if item not in self.consider_msgs_cache:
786 providers_list.append(self.status.pkg_fn[fn])
787 bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
788 bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
789 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
790 self.consider_msgs_cache.append(item)
792 # run through the list until we find one that we can build
794 bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
795 if self.tryBuild(fn, item, buildAllDeps):
798 bb.error("No buildable providers for runtime %s" % item)
799 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
802 def getProvidersRun(self, rdepend):
804 Return any potential providers of runtime rdepend
808 if rdepend in self.status.rproviders:
809 rproviders += self.status.rproviders[rdepend]
811 if rdepend in self.status.packages:
812 rproviders += self.status.packages[rdepend]
817 # Only search dynamic packages if we can't find anything in other variables
818 for pattern in self.status.packages_dynamic:
819 regexp = re.compile(pattern)
820 if regexp.match(rdepend):
821 rproviders += self.status.packages_dynamic[pattern]
825 def addRunDeps(self , fn, item , buildAllDeps):
827 Add any runtime dependencies of runtime item provided by fn
828 as long as item has't previously been processed by this function.
831 if item in self.rbuild_cache:
838 self.rbuild_cache.append(item)
840 if fn in self.status.rundeps and item in self.status.rundeps[fn]:
841 rdepends += self.status.rundeps[fn][item].keys()
842 if fn in self.status.runrecs and item in self.status.runrecs[fn]:
843 rdepends += self.status.runrecs[fn][item].keys()
845 bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
847 for rdepend in rdepends:
848 if rdepend in self.status.ignored_dependencies:
850 if not self.buildRProvider(rdepend, buildAllDeps):
854 def buildDepgraph( self ):
855 all_depends = self.status.all_depends
856 pn_provides = self.status.pn_provides
858 localdata = data.createCopy(self.configuration.data)
859 bb.data.update_data(localdata)
861 def calc_bbfile_priority(filename):
862 for (regex, pri) in self.status.bbfile_config_priorities:
863 if regex.match(filename):
867 # Handle PREFERRED_PROVIDERS
868 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
869 (providee, provider) = p.split(':')
870 if providee in self.preferred and self.preferred[providee] != provider:
871 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
872 self.preferred[providee] = provider
874 # Calculate priorities for each file
875 for p in self.status.pkg_fn.keys():
876 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
878 def buildWorldTargetList(self):
880 Build package list for "bitbake world"
882 all_depends = self.status.all_depends
883 pn_provides = self.status.pn_provides
884 bb.debug(1, "collating packages for \"world\"")
885 for f in self.status.possible_world:
887 pn = self.status.pkg_fn[f]
889 for p in pn_provides[pn]:
890 if p.startswith('virtual/'):
891 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
894 for pf in self.status.providers[p]:
895 if self.status.pkg_fn[pf] != pn:
896 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
900 self.status.world_target.add(pn)
902 # drop reference count now
903 self.status.possible_world = None
904 self.status.all_depends = None
906 def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
907 # feed the status with new input
909 self.status.handle_bb_data(f, bb_cache, from_cache)
913 if os.isatty(sys.stdout.fileno()):
914 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
918 sys.stdout.write("Parsing .bb files, please wait...")
921 sys.stdout.write("done.")
924 def interactiveMode( self ):
925 """Drop off into a shell"""
928 except ImportError, details:
929 bb.fatal("Sorry, shell not available (%s)" % details )
931 bb.data.update_data( self.configuration.data )
935 def parseConfigurationFile( self, afile ):
937 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
939 # Add the handlers we inherited by INHERIT
940 # we need to do this manually as it is not guranteed
941 # we will pick up these classes... as we only INHERIT
942 # on .inc and .bb files but not on .conf
943 data = bb.data.createCopy( self.configuration.data )
944 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
945 for inherit in inherits:
946 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
948 # FIXME: This assumes that we included at least one .inc file
949 for var in bb.data.keys(data):
950 if bb.data.getVarFlag(var, 'handler', data):
951 bb.event.register(var,bb.data.getVar(var, data))
954 bb.fatal( "Unable to open %s" % afile )
955 except bb.parse.ParseError, details:
956 bb.fatal( "Unable to parse %s (%s)" % (afile, details) )
958 def handleCollections( self, collections ):
959 """Handle collections"""
961 collection_list = collections.split()
962 for c in collection_list:
963 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
965 bb.error("BBFILE_PATTERN_%s not defined" % c)
967 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
969 bb.error("BBFILE_PRIORITY_%s not defined" % c)
972 cre = re.compile(regex)
974 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
978 self.status.bbfile_config_priorities.append((cre, pri))
980 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
983 def cook( self, configuration, args ):
985 We are building stuff here. We do the building
986 from here. By default we try to execute task
990 self.configuration = configuration
992 if not self.configuration.cmd:
993 self.configuration.cmd = "build"
995 if self.configuration.debug:
996 bb.debug_level = self.configuration.debug
998 self.configuration.data = bb.data.init()
1000 for f in self.configuration.file:
1001 self.parseConfigurationFile( f )
1003 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
1007 # Special updated configuration we use for firing events
1009 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
1010 bb.data.update_data(self.configuration.event_data)
1012 if self.configuration.show_environment:
1013 self.showEnvironment()
1016 # inject custom variables
1017 if not bb.data.getVar("BUILDNAME", self.configuration.data):
1018 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
1019 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
1021 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
1023 if self.configuration.interactive:
1024 self.interactiveMode()
1026 if self.configuration.buildfile is not None:
1027 bf = os.path.abspath( self.configuration.buildfile )
1029 bbfile_data = bb.parse.handle(bf, self.configuration.data)
1031 bb.fatal("Unable to open %s" % bf)
1033 item = bb.data.getVar('PN', bbfile_data, 1)
1035 self.tryBuildPackage( bf, item, bbfile_data )
1036 except bb.build.EventException:
1037 bb.error( "Build of '%s' failed" % item )
1039 sys.exit( self.stats.show() )
1041 # initialise the parsing status now we know we will need deps
1042 self.status = BBParsingStatus()
1044 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
1045 self.status.ignored_dependencies = Set( ignore.split() )
1047 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
1049 pkgs_to_build = None
1051 if not pkgs_to_build:
1053 pkgs_to_build.extend(args)
1054 if not pkgs_to_build:
1055 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
1057 pkgs_to_build = bbpkgs.split()
1058 if not pkgs_to_build and not self.configuration.show_versions \
1059 and not self.configuration.interactive \
1060 and not self.configuration.show_environment:
1061 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
1062 print "for usage information."
1065 # Import Psyco if available and not disabled
1066 if not self.configuration.disable_psyco:
1071 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
1073 psyco.bind( self.collect_bbfiles )
1075 bb.note("You have disabled Psyco. This decreases performance.")
1078 bb.debug(1, "collecting .bb files")
1079 self.collect_bbfiles( self.myProgressCallback )
1080 bb.debug(1, "parsing complete")
1083 if self.configuration.parse_only:
1084 print "Requested parsing .bb files only. Exiting."
1088 self.buildDepgraph()
1090 if self.configuration.show_versions:
1093 if 'world' in pkgs_to_build:
1094 self.buildWorldTargetList()
1095 pkgs_to_build.remove('world')
1096 for t in self.status.world_target:
1097 pkgs_to_build.append(t)
1099 if self.configuration.dot_graph:
1100 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
1104 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
1107 for k in pkgs_to_build:
1110 if self.buildProvider( k , False ) == 0:
1113 except bb.build.EventException:
1114 bb.error("Build of " + k + " failed")
1118 failures += failures
1119 if self.configuration.abort:
1122 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
1124 sys.exit( self.stats.show() )
1126 except KeyboardInterrupt:
1127 print "\nNOTE: KeyboardInterrupt - Build not completed."
1130 def get_bbfiles( self, path = os.getcwd() ):
1131 """Get list of default .bb files by reading out the current directory"""
1132 contents = os.listdir(path)
1135 (root, ext) = os.path.splitext(f)
1137 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
1140 def find_bbfiles( self, path ):
1141 """Find all the .bb files in a directory (uses find)"""
1142 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
1144 finddata = os.popen(findcmd)
1147 return finddata.readlines()
1149 def collect_bbfiles( self, progressCallback ):
1150 """Collect all available .bb build files"""
1151 self.cb = progressCallback
1152 parsed, cached, skipped, masked = 0, 0, 0, 0
1153 self.bb_cache = bb.cache.init(self)
1155 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
1156 data.setVar("BBFILES", " ".join(files), self.configuration.data)
1159 files = self.get_bbfiles()
1162 bb.error("no files to build.")
1166 if os.path.isdir(f):
1167 dirfiles = self.find_bbfiles(f)
1169 newfiles += dirfiles
1171 newfiles += glob.glob(f) or [ f ]
1173 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
1175 bbmask_compiled = re.compile(bbmask)
1176 except sre_constants.error:
1177 bb.fatal("BBMASK is not a valid regular expression.")
1179 for i in xrange( len( newfiles ) ):
1181 if bbmask and bbmask_compiled.search(f):
1182 bb.debug(1, "bbmake: skipping %s" % f)
1185 debug(1, "bbmake: parsing %s" % f)
1187 # read a file's metadata
1189 fromCache, skip = self.bb_cache.loadData(f, self)
1192 #bb.note("Skipping %s" % f)
1193 self.bb_cache.skip(f)
1195 elif fromCache: cached += 1
1199 # allow metadata files to add items to BBFILES
1200 #data.update_data(self.pkgdata[f])
1201 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1203 for aof in addbbfiles.split():
1204 if not files.count(aof):
1205 if not os.path.isabs(aof):
1206 aof = os.path.join(os.path.dirname(f),aof)
1209 # now inform the caller
1210 if self.cb is not None:
1211 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
1214 self.bb_cache.remove(f)
1215 bb.error("opening %s: %s" % (f, e))
1217 except KeyboardInterrupt:
1218 self.bb_cache.sync()
1220 except Exception, e:
1221 self.bb_cache.remove(f)
1222 bb.error("%s while parsing %s" % (e, f))
1224 self.bb_cache.remove(f)
1227 if self.cb is not None:
1228 print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ),
1230 self.bb_cache.sync()
1232 #============================================================================#
1234 #============================================================================#
1237 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
1238 usage = """%prog [options] [package ...]
1240 Executes the specified task (default is 'build') for a given set of BitBake files.
1241 It expects that BBFILES is defined, which is a space seperated list of files to
1242 be executed. BBFILES does support wildcards.
1243 Default BBFILES are the .bb files in the current directory.""" )
1245 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
1246 action = "store", dest = "buildfile", default = None )
1248 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
1249 action = "store_false", dest = "abort", default = True )
1251 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1252 action = "store_true", dest = "force", default = False )
1254 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1255 action = "store_true", dest = "interactive", default = False )
1257 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1258 action = "store", dest = "cmd", default = "build" )
1260 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1261 action = "append", dest = "file", default = [] )
1263 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1264 action = "store_true", dest = "verbose", default = False )
1266 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1267 action = "count", dest="debug", default = 0)
1269 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
1270 action = "store_true", dest = "dry_run", default = False )
1272 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
1273 action = "store_true", dest = "parse_only", default = False )
1275 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
1276 action = "store_true", dest = "disable_psyco", default = False )
1278 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
1279 action = "store_true", dest = "show_versions", default = False )
1281 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1282 action = "store_true", dest = "show_environment", default = False )
1284 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
1285 action = "store_true", dest = "dot_graph", default = False )
1286 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
1287 action = "append", dest = "ignored_dot_deps", default = [] )
1290 options, args = parser.parse_args( sys.argv )
1293 cooker.cook( BBConfiguration( options ), args[1:] )
1297 if __name__ == "__main__":
1298 print """WARNING, WARNING, WARNING
1299 This is a Bitbake from the Unstable/Development Branch.
1300 You might want to use the bitbake-1.4 stable branch (if you are not a BitBake developer or tester). I'm going to sleep 5 seconds now to make sure you see that."""