2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re, time
25 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
27 from bb import utils, data, parse, debug, event, fatal, cache
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
50 self.packages_dynamic = {}
51 self.bbfile_priority = {}
52 self.bbfile_config_priorities = []
53 self.ignored_dependencies = None
54 self.possible_world = []
55 self.world_target = Set()
61 self.all_depends = Set()
67 def handle_bb_data(self, file_name, bb_cache, cached):
69 We will fill the dictionaries with the stuff we
70 need for building the tree more fast
73 pn = bb_cache.getVar('PN', file_name, True)
74 pv = bb_cache.getVar('PV', file_name, True)
75 pr = bb_cache.getVar('PR', file_name, True)
76 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
77 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
78 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
79 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
80 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
81 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
83 # build PackageName to FileName lookup table
84 if pn not in self.pkg_pn:
86 self.pkg_pn[pn].append(file_name)
88 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
89 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
91 # build FileName to PackageName lookup table
92 self.pkg_fn[file_name] = pn
93 self.pkg_pvpr[file_name] = (pv,pr)
94 self.pkg_dp[file_name] = dp
96 # Build forward and reverse provider hashes
97 # Forward: virtual -> [filenames]
98 # Reverse: PN -> [virtuals]
99 if pn not in self.pn_provides:
100 self.pn_provides[pn] = Set()
101 self.pn_provides[pn] |= provides
103 for provide in provides:
104 if provide not in self.providers:
105 self.providers[provide] = []
106 self.providers[provide].append(file_name)
109 self.all_depends.add(dep)
111 # Build reverse hash for PACKAGES, so runtime dependencies
112 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
113 for package in packages:
114 if not package in self.packages:
115 self.packages[package] = []
116 self.packages[package].append(file_name)
117 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
119 for package in packages_dynamic:
120 if not package in self.packages_dynamic:
121 self.packages_dynamic[package] = []
122 self.packages_dynamic[package].append(file_name)
124 for rprovide in rprovides:
125 if not rprovide in self.rproviders:
126 self.rproviders[rprovide] = []
127 self.rproviders[rprovide].append(file_name)
129 # Build hash of runtime depeneds and rececommends
131 def add_dep(deplist, deps):
133 if not dep in deplist:
136 if not file_name in self.rundeps:
137 self.rundeps[file_name] = {}
138 if not file_name in self.runrecs:
139 self.runrecs[file_name] = {}
141 for package in packages + [pn]:
142 if not package in self.rundeps[file_name]:
143 self.rundeps[file_name][package] = {}
144 if not package in self.runrecs[file_name]:
145 self.runrecs[file_name][package] = {}
147 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
148 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
149 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
150 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
152 # Collect files we may need for possible world-dep
154 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
155 self.possible_world.append(file_name)
158 #============================================================================#
160 #============================================================================#
163 Manage build statistics for one run
172 print "Build statistics:"
173 print " Attempted builds: %d" % self.attempt
175 print " Failed builds: %d" % self.fail
177 print " Dependencies not satisfied: %d" % self.deps
178 if self.fail or self.deps: return 1
182 #============================================================================#
184 #============================================================================#
185 class BBConfiguration( object ):
187 Manages build options and configurations for one run
189 def __init__( self, options ):
190 for key, val in options.__dict__.items():
191 setattr( self, key, val )
193 #============================================================================#
195 #============================================================================#
198 Manages one bitbake build run
201 ParsingStatus = BBParsingStatus # make it visible from the shell
202 Statistics = BBStatistics # make it visible from the shell
204 def __init__( self ):
205 self.build_cache_fail = []
206 self.build_cache = []
207 self.rbuild_cache = []
208 self.building_list = []
210 self.consider_msgs_cache = []
212 self.stats = BBStatistics()
218 def tryBuildPackage( self, fn, item, the_data ):
219 """Build one package"""
220 bb.event.fire(bb.event.PkgStarted(item, the_data))
222 self.stats.attempt += 1
223 if self.configuration.force:
224 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
225 if not self.configuration.dry_run:
226 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
227 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
228 self.build_cache.append(fn)
230 except bb.build.FuncFailed:
232 bb.error("task stack execution failed")
233 bb.event.fire(bb.event.PkgFailed(item, the_data))
234 self.build_cache_fail.append(fn)
236 except bb.build.EventException, e:
239 bb.error("%s event exception, aborting" % bb.event.getName(event))
240 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn)
244 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
246 Build a provider and its dependencies.
247 build_depends is a list of previous build dependencies (not runtime)
248 If build_depends is empty, we're dealing with a runtime depends
251 the_data = self.bb_cache.loadDataFull(fn, self)
253 # Only follow all (runtime) dependencies if doing a build
254 if not buildAllDeps and self.configuration.cmd is "build":
255 buildAllDeps = self.status.build_all[fn]
257 # Error on build time dependency loops
258 if build_depends and build_depends.count(fn) > 1:
259 bb.error("%s depends on itself (eventually)" % fn)
260 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
263 # See if this is a runtime dependency we've already built
264 # Or a build dependency being handled in a different build chain
265 if fn in self.building_list:
266 return self.addRunDeps(fn, virtual , buildAllDeps)
268 item = self.status.pkg_fn[fn]
270 self.building_list.append(fn)
272 pathstr = "%s (%s)" % (item, virtual)
273 self.build_path.append(pathstr)
275 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
277 if self.configuration.verbose:
278 bb.note("current path: %s" % (" -> ".join(self.build_path)))
279 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
284 depcmd = self.configuration.cmd
285 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
286 if bbdepcmd is not None:
293 oldcmd = self.configuration.cmd
294 self.configuration.cmd = depcmd
296 for dependency in depends_list:
297 if dependency in self.status.ignored_dependencies:
301 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
302 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
304 if self.configuration.abort:
308 self.configuration.cmd = oldcmd
314 if not self.addRunDeps(fn, virtual , buildAllDeps):
317 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
318 self.build_cache.append(fn)
321 return self.tryBuildPackage( fn, item, the_data )
324 self.building_list.remove(fn)
325 self.build_path.remove(pathstr)
327 def findBestProvider( self, pn, pkg_pn = None):
329 If there is a PREFERRED_VERSION, find the highest-priority bbfile
330 providing that version. If not, find the latest version provided by
331 an bbfile in the highest-priority set.
334 pkg_pn = self.status.pkg_pn
339 priority = self.status.bbfile_priority[f]
340 if priority not in priorities:
341 priorities[priority] = []
342 priorities[priority].append(f)
343 p_list = priorities.keys()
344 p_list.sort(lambda a, b: a - b)
347 tmp_pn = [priorities[p]] + tmp_pn
349 preferred_file = None
351 localdata = data.createCopy(self.configuration.data)
352 bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
353 bb.data.update_data(localdata)
355 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
357 m = re.match('(.*)_(.*)', preferred_v)
359 preferred_v = m.group(1)
360 preferred_r = m.group(2)
364 for file_set in tmp_pn:
366 pv,pr = self.status.pkg_pvpr[f]
367 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
369 preferred_ver = (pv, pr)
374 pv_str = '%s-%s' % (preferred_v, preferred_r)
377 if preferred_file is None:
378 bb.note("preferred version %s of %s not available" % (pv_str, pn))
380 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
384 # get highest priority file set
389 for file_name in files:
390 pv,pr = self.status.pkg_pvpr[file_name]
391 dp = self.status.pkg_dp[file_name]
393 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
397 if preferred_file is None:
398 preferred_file = latest_f
399 preferred_ver = latest
401 return (latest,latest_f,preferred_ver, preferred_file)
403 def showVersions( self ):
404 pkg_pn = self.status.pkg_pn
405 preferred_versions = {}
409 for pn in pkg_pn.keys():
410 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
411 preferred_versions[pn] = (pref_ver, pref_file)
412 latest_versions[pn] = (last_ver, last_file)
414 pkg_list = pkg_pn.keys()
418 pref = preferred_versions[p]
419 latest = latest_versions[p]
422 prefstr = pref[0][0] + "-" + pref[0][1]
426 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
430 def showEnvironment( self ):
431 """Show the outer or per-package environment"""
432 if self.configuration.buildfile:
434 self.bb_cache = bb.cache.init(self)
436 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
438 fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
441 # emit variables and shell functions
443 data.update_data( self.configuration.data )
444 data.emit_env(sys.__stdout__, self.configuration.data, True)
447 # emit the metadata which isnt valid shell
448 for e in self.configuration.data.keys():
449 if data.getVarFlag( e, 'python', self.configuration.data ):
450 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
452 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
454 Generate two graphs one for the DEPENDS and RDEPENDS. The current
455 implementation creates crappy graphs ;)
457 pkgs_to_build A list of packages that needs to be built
458 ignore_deps A list of names where processing of dependencies
459 should be stopped. e.g. dependencies that get
462 def myFilterProvider( providers, item):
464 Take a list of providers and filter according to environment
465 variables. In contrast to filterProviders we do not discriminate
466 and take PREFERRED_PROVIDER into account.
469 preferred_versions = {}
471 # Collate providers by PN
474 pn = self.status.pkg_fn[p]
479 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
481 for pn in pkg_pn.keys():
482 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
483 eligible.append(preferred_versions[pn][1])
486 if p in self.build_cache_fail:
487 bb.debug(1, "rejecting already-failed %s" % p)
490 if len(eligible) == 0:
491 bb.error("no eligible providers for %s" % item)
494 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
496 # try the preferred provider first
499 if prefervar == self.status.pkg_fn[p]:
500 bb.note("Selecting PREFERRED_PROVIDER %s" % prefervar)
502 eligible = [p] + eligible
509 # try to avoid adding the same rdepends over an over again
514 def add_depends(package_list):
516 Add all depends of all packages from this list
518 for package in package_list:
519 if package in seen_depends or package in ignore_deps:
522 seen_depends.append( package )
523 if not package in self.status.providers:
525 We have not seen this name -> error in
528 bb.note( "ERROR with provider: %(package)s" % vars() )
529 print >> depends_file, '"%(package)s" -> ERROR' % vars()
532 # get all providers for this package
533 providers = self.status.providers[package]
535 # now let us find the bestProvider for it
536 fn = myFilterProvider(providers, package)[0]
538 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
539 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
540 add_depends ( depends )
542 # now create the node
543 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
545 depends = filter( (lambda x: x not in ignore_deps), depends )
546 for depend in depends:
547 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
550 def add_all_depends( the_depends, the_rdepends ):
552 Add both DEPENDS and RDEPENDS. RDEPENDS will get dashed
555 package_list = the_depends + the_rdepends
556 for package in package_list:
557 if package in seen_rdepends or package in ignore_deps:
560 seen_rdepends.append( package )
562 # Let us find out if the package is a DEPENDS or RDEPENDS
563 # and we will set 'providers' with the avilable providers
565 if package in the_depends:
566 if not package in self.status.providers:
567 bb.note( "ERROR with provider: %(package)s" % vars() )
568 print >> alldepends_file, '"%(package)s" -> ERROR' % vars()
571 providers = self.status.providers[package]
572 elif package in the_rdepends:
573 if len(self.getProvidersRun(package)) == 0:
574 bb.note( "ERROR with rprovider: %(package)s" % vars() )
575 print >> alldepends_file, '"%(package)s" -> ERROR [style="dashed"]' % vars()
578 providers = self.getProvidersRun(package)
580 # something went wrong...
581 print "Complete ERROR! %s" % package
584 # now let us find the bestProvider for it
585 fn = myFilterProvider(providers, package)[0]
587 # Now we have a filename let us get the depends and RDEPENDS of it
588 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
589 if fn in self.status.rundeps and package in self.status.rundeps[fn]:
590 rdepends= self.status.rundeps[fn][package].keys()
593 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
595 # handle all the depends and rdepends of package
596 add_all_depends ( depends, rdepends )
598 # now create the node using package name
599 print >> alldepends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
601 # remove the stuff we want to ignore and add the edges
602 depends = filter( (lambda x: x not in ignore_deps), depends )
603 rdepends = filter( (lambda x: x not in ignore_deps), rdepends )
604 for depend in depends:
605 print >> alldepends_file, '"%(package)s" -> "%(depend)s"' % vars()
606 for depend in rdepends:
607 print >> alldepends_file, '"%(package)s" -> "%(depend)s" [style=dashed]' % vars()
611 depends_file = file('depends.dot', 'w' )
612 print >> depends_file, "digraph depends {"
613 add_depends( pkgs_to_build )
614 print >> depends_file, "}"
616 # Add all depends now
617 alldepends_file = file('alldepends.dot', 'w' )
618 print >> alldepends_file, "digraph alldepends {"
619 add_all_depends( pkgs_to_build, [] )
620 print >> alldepends_file, "}"
622 def filterProviders(self, providers, item):
624 Take a list of providers and filter/reorder according to the
625 environment variables and previous build results
628 preferred_versions = {}
630 # Collate providers by PN
633 pn = self.status.pkg_fn[p]
638 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
640 for pn in pkg_pn.keys():
641 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
642 eligible.append(preferred_versions[pn][1])
645 if p in self.build_cache_fail:
646 bb.debug(1, "rejecting already-failed %s" % p)
649 if len(eligible) == 0:
650 bb.error("no eligible providers for %s" % item)
653 # look to see if one of them is already staged, or marked as preferred.
654 # if so, bump it to the head of the queue
656 pn = self.status.pkg_fn[p]
657 pv, pr = self.status.pkg_pvpr[p]
659 stamp = '%s.do_populate_staging' % self.status.stamp[p]
660 if os.path.exists(stamp):
661 (newvers, fn) = preferred_versions[pn]
662 if not fn in eligible:
663 # package was made ineligible by already-failed check
665 oldver = "%s-%s" % (pv, pr)
666 newver = '-'.join(newvers)
667 if (newver != oldver):
668 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
670 extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
671 if self.configuration.verbose:
672 bb.note("%s" % extra_chat)
674 eligible = [fn] + eligible
680 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
682 Build something to provide a named build requirement
683 (takes item names from DEPENDS namespace)
687 discriminated = False
689 if not item in self.status.providers:
690 bb.error("Nothing provides dependency %s" % item)
691 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
694 all_p = self.status.providers[item]
697 if p in self.build_cache:
698 bb.debug(1, "already built %s in this run\n" % p)
701 eligible = self.filterProviders(all_p, item)
706 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
708 self.preferred[item] = prefervar
710 if item in self.preferred:
712 pn = self.status.pkg_fn[p]
713 if self.preferred[item] == pn:
714 if self.configuration.verbose:
715 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
717 eligible = [p] + eligible
721 if len(eligible) > 1 and discriminated == False:
722 if item not in self.consider_msgs_cache:
725 providers_list.append(self.status.pkg_fn[fn])
726 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
727 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
728 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
729 self.consider_msgs_cache.append(item)
732 # run through the list until we find one that we can build
734 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
735 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
738 bb.note("no buildable providers for %s" % item)
739 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
742 def buildRProvider( self, item , buildAllDeps ):
744 Build something to provide a named runtime requirement
745 (takes item names from RDEPENDS/PACKAGES namespace)
750 discriminated = False
755 all_p = self.getProvidersRun(item)
758 bb.error("Nothing provides runtime dependency %s" % (item))
759 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True))
763 if p in self.rbuild_cache:
764 bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
766 if p in self.build_cache:
767 bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
768 return self.addRunDeps(p, item , buildAllDeps)
770 eligible = self.filterProviders(all_p, item)
776 pn = self.status.pkg_fn[p]
777 provides = self.status.pn_provides[pn]
778 for provide in provides:
779 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
781 if self.configuration.verbose:
782 bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
784 eligible = [p] + eligible
787 if len(eligible) > 1 and len(preferred) == 0:
788 if item not in self.consider_msgs_cache:
791 providers_list.append(self.status.pkg_fn[fn])
792 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
793 bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
794 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
795 self.consider_msgs_cache.append(item)
797 if len(preferred) > 1:
798 if item not in self.consider_msgs_cache:
801 providers_list.append(self.status.pkg_fn[fn])
802 bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
803 bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
804 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
805 self.consider_msgs_cache.append(item)
807 # run through the list until we find one that we can build
809 bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
810 if self.tryBuild(fn, item, buildAllDeps):
813 bb.error("No buildable providers for runtime %s" % item)
814 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
817 def getProvidersRun(self, rdepend):
819 Return any potential providers of runtime rdepend
823 if rdepend in self.status.rproviders:
824 rproviders += self.status.rproviders[rdepend]
826 if rdepend in self.status.packages:
827 rproviders += self.status.packages[rdepend]
832 # Only search dynamic packages if we can't find anything in other variables
833 for pattern in self.status.packages_dynamic:
834 regexp = re.compile(pattern)
835 if regexp.match(rdepend):
836 rproviders += self.status.packages_dynamic[pattern]
840 def addRunDeps(self , fn, item , buildAllDeps):
842 Add any runtime dependencies of runtime item provided by fn
843 as long as item has't previously been processed by this function.
846 if item in self.rbuild_cache:
853 self.rbuild_cache.append(item)
855 if fn in self.status.rundeps and item in self.status.rundeps[fn]:
856 rdepends += self.status.rundeps[fn][item].keys()
857 if fn in self.status.runrecs and item in self.status.runrecs[fn]:
858 rdepends += self.status.runrecs[fn][item].keys()
860 bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
862 for rdepend in rdepends:
863 if rdepend in self.status.ignored_dependencies:
865 if not self.buildRProvider(rdepend, buildAllDeps):
869 def buildDepgraph( self ):
870 all_depends = self.status.all_depends
871 pn_provides = self.status.pn_provides
873 localdata = data.createCopy(self.configuration.data)
874 bb.data.update_data(localdata)
876 def calc_bbfile_priority(filename):
877 for (regex, pri) in self.status.bbfile_config_priorities:
878 if regex.match(filename):
882 # Handle PREFERRED_PROVIDERS
883 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
884 (providee, provider) = p.split(':')
885 if providee in self.preferred and self.preferred[providee] != provider:
886 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
887 self.preferred[providee] = provider
889 # Calculate priorities for each file
890 for p in self.status.pkg_fn.keys():
891 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
893 def buildWorldTargetList(self):
895 Build package list for "bitbake world"
897 all_depends = self.status.all_depends
898 pn_provides = self.status.pn_provides
899 bb.debug(1, "collating packages for \"world\"")
900 for f in self.status.possible_world:
902 pn = self.status.pkg_fn[f]
904 for p in pn_provides[pn]:
905 if p.startswith('virtual/'):
906 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
909 for pf in self.status.providers[p]:
910 if self.status.pkg_fn[pf] != pn:
911 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
915 self.status.world_target.add(pn)
917 # drop reference count now
918 self.status.possible_world = None
919 self.status.all_depends = None
921 def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
922 # feed the status with new input
924 self.status.handle_bb_data(f, bb_cache, from_cache)
928 if os.isatty(sys.stdout.fileno()):
929 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
933 sys.stdout.write("Parsing .bb files, please wait...")
936 sys.stdout.write("done.")
939 def interactiveMode( self ):
940 """Drop off into a shell"""
943 except ImportError, details:
944 bb.fatal("Sorry, shell not available (%s)" % details )
946 bb.data.update_data( self.configuration.data )
950 def parseConfigurationFile( self, afile ):
952 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
954 # Add the handlers we inherited by INHERIT
955 # we need to do this manually as it is not guranteed
956 # we will pick up these classes... as we only INHERIT
957 # on .inc and .bb files but not on .conf
958 data = bb.data.createCopy( self.configuration.data )
959 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
960 for inherit in inherits:
961 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
963 # FIXME: This assumes that we included at least one .inc file
964 for var in bb.data.keys(data):
965 if bb.data.getVarFlag(var, 'handler', data):
966 bb.event.register(var,bb.data.getVar(var, data))
969 bb.fatal( "Unable to open %s" % afile )
970 except bb.parse.ParseError, details:
971 bb.fatal( "Unable to parse %s (%s)" % (afile, details) )
973 def handleCollections( self, collections ):
974 """Handle collections"""
976 collection_list = collections.split()
977 for c in collection_list:
978 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
980 bb.error("BBFILE_PATTERN_%s not defined" % c)
982 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
984 bb.error("BBFILE_PRIORITY_%s not defined" % c)
987 cre = re.compile(regex)
989 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
993 self.status.bbfile_config_priorities.append((cre, pri))
995 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
998 def cook( self, configuration, args ):
1000 We are building stuff here. We do the building
1001 from here. By default we try to execute task
1005 self.configuration = configuration
1007 if not self.configuration.cmd:
1008 self.configuration.cmd = "build"
1010 if self.configuration.debug:
1011 bb.debug_level = self.configuration.debug
1013 self.configuration.data = bb.data.init()
1015 for f in self.configuration.file:
1016 self.parseConfigurationFile( f )
1018 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
1022 # Special updated configuration we use for firing events
1024 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
1025 bb.data.update_data(self.configuration.event_data)
1027 if self.configuration.show_environment:
1028 self.showEnvironment()
1031 # inject custom variables
1032 if not bb.data.getVar("BUILDNAME", self.configuration.data):
1033 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
1034 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
1036 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
1038 if self.configuration.interactive:
1039 self.interactiveMode()
1041 if self.configuration.buildfile is not None:
1042 bf = os.path.abspath( self.configuration.buildfile )
1044 bbfile_data = bb.parse.handle(bf, self.configuration.data)
1046 bb.fatal("Unable to open %s" % bf)
1048 item = bb.data.getVar('PN', bbfile_data, 1)
1050 self.tryBuildPackage( bf, item, bbfile_data )
1051 except bb.build.EventException:
1052 bb.error( "Build of '%s' failed" % item )
1054 sys.exit( self.stats.show() )
1056 # initialise the parsing status now we know we will need deps
1057 self.status = BBParsingStatus()
1059 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
1060 self.status.ignored_dependencies = Set( ignore.split() )
1062 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
1064 pkgs_to_build = None
1066 if not pkgs_to_build:
1068 pkgs_to_build.extend(args)
1069 if not pkgs_to_build:
1070 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
1072 pkgs_to_build = bbpkgs.split()
1073 if not pkgs_to_build and not self.configuration.show_versions \
1074 and not self.configuration.interactive \
1075 and not self.configuration.show_environment:
1076 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
1077 print "for usage information."
1080 # Import Psyco if available and not disabled
1081 if not self.configuration.disable_psyco:
1086 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
1088 psyco.bind( self.collect_bbfiles )
1090 bb.note("You have disabled Psyco. This decreases performance.")
1093 bb.debug(1, "collecting .bb files")
1094 self.collect_bbfiles( self.myProgressCallback )
1095 bb.debug(1, "parsing complete")
1098 if self.configuration.parse_only:
1099 print "Requested parsing .bb files only. Exiting."
1103 self.buildDepgraph()
1105 if self.configuration.show_versions:
1108 if 'world' in pkgs_to_build:
1109 self.buildWorldTargetList()
1110 pkgs_to_build.remove('world')
1111 for t in self.status.world_target:
1112 pkgs_to_build.append(t)
1114 if self.configuration.dot_graph:
1115 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
1119 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
1122 for k in pkgs_to_build:
1125 if self.buildProvider( k , False ) == 0:
1128 except bb.build.EventException:
1129 bb.error("Build of " + k + " failed")
1133 failures += failures
1134 if self.configuration.abort:
1137 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
1139 sys.exit( self.stats.show() )
1141 except KeyboardInterrupt:
1142 print "\nNOTE: KeyboardInterrupt - Build not completed."
1145 def get_bbfiles( self, path = os.getcwd() ):
1146 """Get list of default .bb files by reading out the current directory"""
1147 contents = os.listdir(path)
1150 (root, ext) = os.path.splitext(f)
1152 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
1155 def find_bbfiles( self, path ):
1156 """Find all the .bb files in a directory (uses find)"""
1157 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
1159 finddata = os.popen(findcmd)
1162 return finddata.readlines()
1164 def collect_bbfiles( self, progressCallback ):
1165 """Collect all available .bb build files"""
1166 self.cb = progressCallback
1167 parsed, cached, skipped, masked = 0, 0, 0, 0
1168 self.bb_cache = bb.cache.init(self)
1170 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
1171 data.setVar("BBFILES", " ".join(files), self.configuration.data)
1174 files = self.get_bbfiles()
1177 bb.error("no files to build.")
1181 if os.path.isdir(f):
1182 dirfiles = self.find_bbfiles(f)
1184 newfiles += dirfiles
1186 newfiles += glob.glob(f) or [ f ]
1188 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
1190 bbmask_compiled = re.compile(bbmask)
1191 except sre_constants.error:
1192 bb.fatal("BBMASK is not a valid regular expression.")
1194 for i in xrange( len( newfiles ) ):
1196 if bbmask and bbmask_compiled.search(f):
1197 bb.debug(1, "bbmake: skipping %s" % f)
1200 debug(1, "bbmake: parsing %s" % f)
1202 # read a file's metadata
1204 fromCache, skip = self.bb_cache.loadData(f, self)
1207 #bb.note("Skipping %s" % f)
1208 self.bb_cache.skip(f)
1210 elif fromCache: cached += 1
1214 # allow metadata files to add items to BBFILES
1215 #data.update_data(self.pkgdata[f])
1216 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1218 for aof in addbbfiles.split():
1219 if not files.count(aof):
1220 if not os.path.isabs(aof):
1221 aof = os.path.join(os.path.dirname(f),aof)
1224 # now inform the caller
1225 if self.cb is not None:
1226 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
1229 self.bb_cache.remove(f)
1230 bb.error("opening %s: %s" % (f, e))
1232 except KeyboardInterrupt:
1233 self.bb_cache.sync()
1235 except Exception, e:
1236 self.bb_cache.remove(f)
1237 bb.error("%s while parsing %s" % (e, f))
1239 self.bb_cache.remove(f)
1242 if self.cb is not None:
1243 print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ),
1245 self.bb_cache.sync()
1247 #============================================================================#
1249 #============================================================================#
1252 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
1253 usage = """%prog [options] [package ...]
1255 Executes the specified task (default is 'build') for a given set of BitBake files.
1256 It expects that BBFILES is defined, which is a space seperated list of files to
1257 be executed. BBFILES does support wildcards.
1258 Default BBFILES are the .bb files in the current directory.""" )
1260 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
1261 action = "store", dest = "buildfile", default = None )
1263 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
1264 action = "store_false", dest = "abort", default = True )
1266 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1267 action = "store_true", dest = "force", default = False )
1269 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1270 action = "store_true", dest = "interactive", default = False )
1272 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1273 action = "store", dest = "cmd", default = "build" )
1275 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1276 action = "append", dest = "file", default = [] )
1278 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1279 action = "store_true", dest = "verbose", default = False )
1281 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1282 action = "count", dest="debug", default = 0)
1284 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
1285 action = "store_true", dest = "dry_run", default = False )
1287 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
1288 action = "store_true", dest = "parse_only", default = False )
1290 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
1291 action = "store_true", dest = "disable_psyco", default = False )
1293 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
1294 action = "store_true", dest = "show_versions", default = False )
1296 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1297 action = "store_true", dest = "show_environment", default = False )
1299 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
1300 action = "store_true", dest = "dot_graph", default = False )
1301 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
1302 action = "append", dest = "ignored_dot_deps", default = [] )
1305 options, args = parser.parse_args( sys.argv )
1308 cooker.cook( BBConfiguration( options ), args[1:] )
1312 if __name__ == "__main__":
1313 print """WARNING, WARNING, WARNING
1314 This is a Bitbake from the Unstable/Development Branch.
1315 You might want to use the bitbake-1.4 stable branch (if you are not a BitBake developer or tester). I'm going to sleep 5 seconds now to make sure you see that."""