2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re, time
25 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
27 from bb import utils, data, parse, debug, event, fatal, cache
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
50 self.packages_dynamic = {}
51 self.bbfile_priority = {}
52 self.bbfile_config_priorities = []
53 self.ignored_dependencies = None
54 self.possible_world = []
55 self.world_target = Set()
61 self.all_depends = Set()
67 def handle_bb_data(self, file_name, bb_cache, cached):
69 We will fill the dictionaries with the stuff we
70 need for building the tree more fast
73 pn = bb_cache.getVar('PN', file_name, True)
74 pv = bb_cache.getVar('PV', file_name, True)
75 pr = bb_cache.getVar('PR', file_name, True)
76 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
77 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
78 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
79 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
80 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
81 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
83 # build PackageName to FileName lookup table
84 if pn not in self.pkg_pn:
86 self.pkg_pn[pn].append(file_name)
88 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
89 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
91 # build FileName to PackageName lookup table
92 self.pkg_fn[file_name] = pn
93 self.pkg_pvpr[file_name] = (pv,pr)
94 self.pkg_dp[file_name] = dp
96 # Build forward and reverse provider hashes
97 # Forward: virtual -> [filenames]
98 # Reverse: PN -> [virtuals]
99 if pn not in self.pn_provides:
100 self.pn_provides[pn] = Set()
101 self.pn_provides[pn] |= provides
103 for provide in provides:
104 if provide not in self.providers:
105 self.providers[provide] = []
106 self.providers[provide].append(file_name)
109 self.all_depends.add(dep)
111 # Build reverse hash for PACKAGES, so runtime dependencies
112 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
113 for package in packages:
114 if not package in self.packages:
115 self.packages[package] = []
116 self.packages[package].append(file_name)
117 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
119 for package in packages_dynamic:
120 if not package in self.packages_dynamic:
121 self.packages_dynamic[package] = []
122 self.packages_dynamic[package].append(file_name)
124 for rprovide in rprovides:
125 if not rprovide in self.rproviders:
126 self.rproviders[rprovide] = []
127 self.rproviders[rprovide].append(file_name)
129 # Build hash of runtime depeneds and rececommends
131 def add_dep(deplist, deps):
133 if not dep in deplist:
136 if not file_name in self.rundeps:
137 self.rundeps[file_name] = {}
138 if not file_name in self.runrecs:
139 self.runrecs[file_name] = {}
141 for package in packages + [pn]:
142 if not package in self.rundeps[file_name]:
143 self.rundeps[file_name][package] = {}
144 if not package in self.runrecs[file_name]:
145 self.runrecs[file_name][package] = {}
147 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
148 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
149 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
150 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
152 # Collect files we may need for possible world-dep
154 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
155 self.possible_world.append(file_name)
158 #============================================================================#
160 #============================================================================#
163 Manage build statistics for one run
172 print "Build statistics:"
173 print " Attempted builds: %d" % self.attempt
175 print " Failed builds: %d" % self.fail
177 print " Dependencies not satisfied: %d" % self.deps
178 if self.fail or self.deps: return 1
182 #============================================================================#
184 #============================================================================#
185 class BBConfiguration( object ):
187 Manages build options and configurations for one run
189 def __init__( self, options ):
190 for key, val in options.__dict__.items():
191 setattr( self, key, val )
193 #============================================================================#
195 #============================================================================#
198 Manages one bitbake build run
201 ParsingStatus = BBParsingStatus # make it visible from the shell
202 Statistics = BBStatistics # make it visible from the shell
204 def __init__( self ):
205 self.build_cache_fail = []
206 self.build_cache = []
207 self.rbuild_cache = []
208 self.building_list = []
210 self.consider_msgs_cache = []
212 self.stats = BBStatistics()
218 def tryBuildPackage( self, fn, item, the_data ):
219 """Build one package"""
220 bb.event.fire(bb.event.PkgStarted(item, the_data))
222 self.stats.attempt += 1
223 if self.configuration.force:
224 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
225 if not self.configuration.dry_run:
226 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
227 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
228 self.build_cache.append(fn)
230 except bb.build.FuncFailed:
232 bb.error("task stack execution failed")
233 bb.event.fire(bb.event.PkgFailed(item, the_data))
234 self.build_cache_fail.append(fn)
236 except bb.build.EventException, e:
239 bb.error("%s event exception, aborting" % bb.event.getName(event))
240 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn)
244 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
246 Build a provider and its dependencies.
247 build_depends is a list of previous build dependencies (not runtime)
248 If build_depends is empty, we're dealing with a runtime depends
251 the_data = self.bb_cache.loadDataFull(fn, self)
253 # Only follow all (runtime) dependencies if doing a build
254 if not buildAllDeps and self.configuration.cmd is "build":
255 buildAllDeps = self.status.build_all[fn]
257 # Error on build time dependency loops
258 if build_depends and build_depends.count(fn) > 1:
259 bb.error("%s depends on itself (eventually)" % fn)
260 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
263 # See if this is a runtime dependency we've already built
264 # Or a build dependency being handled in a different build chain
265 if fn in self.building_list:
266 return self.addRunDeps(fn, virtual , buildAllDeps)
268 item = self.status.pkg_fn[fn]
270 self.building_list.append(fn)
272 pathstr = "%s (%s)" % (item, virtual)
273 self.build_path.append(pathstr)
275 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
277 if self.configuration.verbose:
278 bb.note("current path: %s" % (" -> ".join(self.build_path)))
279 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
284 depcmd = self.configuration.cmd
285 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
286 if bbdepcmd is not None:
293 oldcmd = self.configuration.cmd
294 self.configuration.cmd = depcmd
296 for dependency in depends_list:
297 if dependency in self.status.ignored_dependencies:
301 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
302 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
304 if self.configuration.abort:
308 self.configuration.cmd = oldcmd
314 if not self.addRunDeps(fn, virtual , buildAllDeps):
317 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
318 self.build_cache.append(fn)
321 return self.tryBuildPackage( fn, item, the_data )
324 self.building_list.remove(fn)
325 self.build_path.remove(pathstr)
327 def findBestProvider( self, pn, pkg_pn = None):
329 If there is a PREFERRED_VERSION, find the highest-priority bbfile
330 providing that version. If not, find the latest version provided by
331 an bbfile in the highest-priority set.
334 pkg_pn = self.status.pkg_pn
339 priority = self.status.bbfile_priority[f]
340 if priority not in priorities:
341 priorities[priority] = []
342 priorities[priority].append(f)
343 p_list = priorities.keys()
344 p_list.sort(lambda a, b: a - b)
347 tmp_pn = [priorities[p]] + tmp_pn
349 preferred_file = None
351 localdata = data.createCopy(self.configuration.data)
352 bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
353 bb.data.update_data(localdata)
355 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
357 m = re.match('(.*)_(.*)', preferred_v)
359 preferred_v = m.group(1)
360 preferred_r = m.group(2)
364 for file_set in tmp_pn:
366 pv,pr = self.status.pkg_pvpr[f]
367 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
369 preferred_ver = (pv, pr)
374 pv_str = '%s-%s' % (preferred_v, preferred_r)
377 if preferred_file is None:
378 bb.note("preferred version %s of %s not available" % (pv_str, pn))
380 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
384 # get highest priority file set
389 for file_name in files:
390 pv,pr = self.status.pkg_pvpr[file_name]
391 dp = self.status.pkg_dp[file_name]
393 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
397 if preferred_file is None:
398 preferred_file = latest_f
399 preferred_ver = latest
401 return (latest,latest_f,preferred_ver, preferred_file)
403 def showVersions( self ):
404 pkg_pn = self.status.pkg_pn
405 preferred_versions = {}
409 for pn in pkg_pn.keys():
410 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
411 preferred_versions[pn] = (pref_ver, pref_file)
412 latest_versions[pn] = (last_ver, last_file)
414 pkg_list = pkg_pn.keys()
418 pref = preferred_versions[p]
419 latest = latest_versions[p]
422 prefstr = pref[0][0] + "-" + pref[0][1]
426 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
430 def showEnvironment( self ):
431 """Show the outer or per-package environment"""
432 if self.configuration.buildfile:
434 self.bb_cache = bb.cache.init(self)
436 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
438 fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
441 # emit variables and shell functions
443 data.update_data( self.configuration.data )
444 data.emit_env(sys.__stdout__, self.configuration.data, True)
447 # emit the metadata which isnt valid shell
448 for e in self.configuration.data.keys():
449 if data.getVarFlag( e, 'python', self.configuration.data ):
450 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
452 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
454 Generate two graphs one for the DEPENDS and RDEPENDS. The current
455 implementation creates crappy graphs ;)
457 pkgs_to_build A list of packages that needs to be built
458 ignore_deps A list of names where processing of dependencies
459 should be stopped. e.g. dependencies that get
462 def myFilterProvider( providers, item):
464 Take a list of providers and filter according to environment
465 variables. In contrast to filterProviders we do not discriminate
466 and take PREFERRED_PROVIDER into account.
469 preferred_versions = {}
471 # Collate providers by PN
474 pn = self.status.pkg_fn[p]
479 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
481 for pn in pkg_pn.keys():
482 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
483 eligible.append(preferred_versions[pn][1])
486 if p in self.build_cache_fail:
487 bb.debug(1, "rejecting already-failed %s" % p)
490 if len(eligible) == 0:
491 bb.error("no eligible providers for %s" % item)
494 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % package, self.configuration.data, 1)
496 # try the preferred provider first
499 if prefervar == self.status.pkg_fn[p]:
500 bb.note("Selecting PREFERRED_PROVIDER %s" % prefervar)
502 elligible = [p] + elligible
509 # try to avoid adding the same rdepends over an over again
514 def add_depends(package_list):
516 Add all depends of all packages from this list
518 for package in package_list:
519 if package in seen_depends or package in ignore_deps:
522 seen_depends.append( package )
523 if not package in self.status.providers:
525 We have not seen this name -> error in
528 bb.note( "ERROR with provider: %(package)s" % vars() )
529 print >> depends_file, '"%(package)s" -> ERROR' % vars()
532 # get all providers for this package
533 providers = self.status.providers[package]
535 # now let us find the bestProvider for it
536 fn = self.filterProviders(providers, package)[0]
538 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
539 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
540 add_depends ( depends )
542 # now create the node
543 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
545 depends = filter( (lambda x: x not in ignore_deps), depends )
546 for depend in depends:
547 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
550 def add_all_depends( the_depends, the_rdepends ):
552 Add both DEPENDS and RDEPENDS. RDEPENDS will get dashed
555 package_list = the_depends + the_rdepends
556 for package in package_list:
557 if package in seen_rdepends or package in ignore_deps:
560 seen_rdepends.append( package )
562 # let us see if this is a runtime or
563 if package in the_depends:
564 if not package in self.status.providers:
565 bb.note( "ERROR with provider: %(package)s" % vars() )
566 print >> alldepends_file, '"%(package)s" -> ERROR' % vars()
569 providers = self.status.providers[package]
570 elif package in the_rdepends:
571 if len(self.getProvidersRun(package)) == 0:
572 bb.note( "ERROR with rprovider: %(package)s" % vars() )
573 print >> alldepends_file, '"%(package)s" -> ERROR [style="dashed"]' % vars()
576 providers = self.getProvidersRun(package)
578 print "Complete ERROR! %s" % package
581 # now let us find the bestProvider for it
582 fn = self.filterProviders(providers, package)[0]
584 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
585 if fn in self.status.rundeps and package in self.status.rundeps[fn]:
586 rdepends= self.status.rundeps[fn][package].keys()
589 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
590 if package == "task-opie-applets":
596 add_all_depends ( depends, rdepends )
598 # now create the node
599 print >> alldepends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
601 depends = filter( (lambda x: x not in ignore_deps), depends )
602 rdepends = filter( (lambda x: x not in ignore_deps), rdepends )
603 for depend in depends:
604 print >> alldepends_file, '"%(package)s" -> "%(depend)s"' % vars()
605 for depend in rdepends:
606 print >> alldepends_file, '"%(package)s" -> "%(depend)s" [style=dashed]' % vars()
610 depends_file = file('depends.dot', 'w' )
611 print >> depends_file, "digraph depends {"
612 add_depends( pkgs_to_build )
613 print >> depends_file, "}"
615 # Add all depends now
616 alldepends_file = file('alldepends.dot', 'w' )
617 print >> alldepends_file, "digraph alldepends {"
618 add_all_depends( pkgs_to_build, [] )
619 print >> alldepends_file, "}"
621 def filterProviders(self, providers, item):
623 Take a list of providers and filter/reorder according to the
624 environment variables and previous build results
627 preferred_versions = {}
629 # Collate providers by PN
632 pn = self.status.pkg_fn[p]
637 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
639 for pn in pkg_pn.keys():
640 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
641 eligible.append(preferred_versions[pn][1])
644 if p in self.build_cache_fail:
645 bb.debug(1, "rejecting already-failed %s" % p)
648 if len(eligible) == 0:
649 bb.error("no eligible providers for %s" % item)
652 # look to see if one of them is already staged, or marked as preferred.
653 # if so, bump it to the head of the queue
655 pn = self.status.pkg_fn[p]
656 pv, pr = self.status.pkg_pvpr[p]
658 stamp = '%s.do_populate_staging' % self.status.stamp[p]
659 if os.path.exists(stamp):
660 (newvers, fn) = preferred_versions[pn]
661 if not fn in eligible:
662 # package was made ineligible by already-failed check
664 oldver = "%s-%s" % (pv, pr)
665 newver = '-'.join(newvers)
666 if (newver != oldver):
667 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
669 extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
670 if self.configuration.verbose:
671 bb.note("%s" % extra_chat)
673 eligible = [fn] + eligible
679 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
681 Build something to provide a named build requirement
682 (takes item names from DEPENDS namespace)
686 discriminated = False
688 if not item in self.status.providers:
689 bb.error("Nothing provides dependency %s" % item)
690 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
693 all_p = self.status.providers[item]
696 if p in self.build_cache:
697 bb.debug(1, "already built %s in this run\n" % p)
700 eligible = self.filterProviders(all_p, item)
705 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
707 self.preferred[item] = prefervar
709 if item in self.preferred:
711 pn = self.status.pkg_fn[p]
712 if self.preferred[item] == pn:
713 if self.configuration.verbose:
714 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
716 eligible = [p] + eligible
720 if len(eligible) > 1 and discriminated == False:
721 if item not in self.consider_msgs_cache:
724 providers_list.append(self.status.pkg_fn[fn])
725 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
726 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
727 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
728 self.consider_msgs_cache.append(item)
731 # run through the list until we find one that we can build
733 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
734 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
737 bb.note("no buildable providers for %s" % item)
738 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
741 def buildRProvider( self, item , buildAllDeps ):
743 Build something to provide a named runtime requirement
744 (takes item names from RDEPENDS/PACKAGES namespace)
749 discriminated = False
754 all_p = self.getProvidersRun(item)
757 bb.error("Nothing provides runtime dependency %s" % (item))
758 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True))
762 if p in self.rbuild_cache:
763 bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
765 if p in self.build_cache:
766 bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
767 return self.addRunDeps(p, item , buildAllDeps)
769 eligible = self.filterProviders(all_p, item)
775 pn = self.status.pkg_fn[p]
776 provides = self.status.pn_provides[pn]
777 for provide in provides:
778 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
780 if self.configuration.verbose:
781 bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
783 eligible = [p] + eligible
786 if len(eligible) > 1 and len(preferred) == 0:
787 if item not in self.consider_msgs_cache:
790 providers_list.append(self.status.pkg_fn[fn])
791 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
792 bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
793 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
794 self.consider_msgs_cache.append(item)
796 if len(preferred) > 1:
797 if item not in self.consider_msgs_cache:
800 providers_list.append(self.status.pkg_fn[fn])
801 bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
802 bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
803 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
804 self.consider_msgs_cache.append(item)
806 # run through the list until we find one that we can build
808 bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
809 if self.tryBuild(fn, item, buildAllDeps):
812 bb.error("No buildable providers for runtime %s" % item)
813 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
816 def getProvidersRun(self, rdepend):
818 Return any potential providers of runtime rdepend
822 if rdepend in self.status.rproviders:
823 rproviders += self.status.rproviders[rdepend]
825 if rdepend in self.status.packages:
826 rproviders += self.status.packages[rdepend]
831 # Only search dynamic packages if we can't find anything in other variables
832 for pattern in self.status.packages_dynamic:
833 regexp = re.compile(pattern)
834 if regexp.match(rdepend):
835 rproviders += self.status.packages_dynamic[pattern]
839 def addRunDeps(self , fn, item , buildAllDeps):
841 Add any runtime dependencies of runtime item provided by fn
842 as long as item has't previously been processed by this function.
845 if item in self.rbuild_cache:
852 self.rbuild_cache.append(item)
854 if fn in self.status.rundeps and item in self.status.rundeps[fn]:
855 rdepends += self.status.rundeps[fn][item].keys()
856 if fn in self.status.runrecs and item in self.status.runrecs[fn]:
857 rdepends += self.status.runrecs[fn][item].keys()
859 bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
861 for rdepend in rdepends:
862 if rdepend in self.status.ignored_dependencies:
864 if not self.buildRProvider(rdepend, buildAllDeps):
868 def buildDepgraph( self ):
869 all_depends = self.status.all_depends
870 pn_provides = self.status.pn_provides
872 localdata = data.createCopy(self.configuration.data)
873 bb.data.update_data(localdata)
875 def calc_bbfile_priority(filename):
876 for (regex, pri) in self.status.bbfile_config_priorities:
877 if regex.match(filename):
881 # Handle PREFERRED_PROVIDERS
882 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
883 (providee, provider) = p.split(':')
884 if providee in self.preferred and self.preferred[providee] != provider:
885 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
886 self.preferred[providee] = provider
888 # Calculate priorities for each file
889 for p in self.status.pkg_fn.keys():
890 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
892 def buildWorldTargetList(self):
894 Build package list for "bitbake world"
896 all_depends = self.status.all_depends
897 pn_provides = self.status.pn_provides
898 bb.debug(1, "collating packages for \"world\"")
899 for f in self.status.possible_world:
901 pn = self.status.pkg_fn[f]
903 for p in pn_provides[pn]:
904 if p.startswith('virtual/'):
905 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
908 for pf in self.status.providers[p]:
909 if self.status.pkg_fn[pf] != pn:
910 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
914 self.status.world_target.add(pn)
916 # drop reference count now
917 self.status.possible_world = None
918 self.status.all_depends = None
920 def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
921 # feed the status with new input
923 self.status.handle_bb_data(f, bb_cache, from_cache)
927 if os.isatty(sys.stdout.fileno()):
928 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
932 sys.stdout.write("Parsing .bb files, please wait...")
935 sys.stdout.write("done.")
938 def interactiveMode( self ):
939 """Drop off into a shell"""
942 except ImportError, details:
943 bb.fatal("Sorry, shell not available (%s)" % details )
945 bb.data.update_data( self.configuration.data )
949 def parseConfigurationFile( self, afile ):
951 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
953 # Add the handlers we inherited by INHERIT
954 # we need to do this manually as it is not guranteed
955 # we will pick up these classes... as we only INHERIT
956 # on .inc and .bb files but not on .conf
957 data = bb.data.createCopy( self.configuration.data )
958 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
959 for inherit in inherits:
960 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
962 # FIXME: This assumes that we included at least one .inc file
963 for var in bb.data.keys(data):
964 if bb.data.getVarFlag(var, 'handler', data):
965 bb.event.register(var,bb.data.getVar(var, data))
968 bb.fatal( "Unable to open %s" % afile )
969 except bb.parse.ParseError, details:
970 bb.fatal( "Unable to parse %s (%s)" % (afile, details) )
972 def handleCollections( self, collections ):
973 """Handle collections"""
975 collection_list = collections.split()
976 for c in collection_list:
977 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
979 bb.error("BBFILE_PATTERN_%s not defined" % c)
981 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
983 bb.error("BBFILE_PRIORITY_%s not defined" % c)
986 cre = re.compile(regex)
988 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
992 self.status.bbfile_config_priorities.append((cre, pri))
994 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
997 def cook( self, configuration, args ):
999 We are building stuff here. We do the building
1000 from here. By default we try to execute task
1004 self.configuration = configuration
1006 if not self.configuration.cmd:
1007 self.configuration.cmd = "build"
1009 if self.configuration.debug:
1010 bb.debug_level = self.configuration.debug
1012 self.configuration.data = bb.data.init()
1014 for f in self.configuration.file:
1015 self.parseConfigurationFile( f )
1017 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
1021 # Special updated configuration we use for firing events
1023 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
1024 bb.data.update_data(self.configuration.event_data)
1026 if self.configuration.show_environment:
1027 self.showEnvironment()
1030 # inject custom variables
1031 if not bb.data.getVar("BUILDNAME", self.configuration.data):
1032 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
1033 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
1035 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
1037 if self.configuration.interactive:
1038 self.interactiveMode()
1040 if self.configuration.buildfile is not None:
1041 bf = os.path.abspath( self.configuration.buildfile )
1043 bbfile_data = bb.parse.handle(bf, self.configuration.data)
1045 bb.fatal("Unable to open %s" % bf)
1047 item = bb.data.getVar('PN', bbfile_data, 1)
1049 self.tryBuildPackage( bf, item, bbfile_data )
1050 except bb.build.EventException:
1051 bb.error( "Build of '%s' failed" % item )
1053 sys.exit( self.stats.show() )
1055 # initialise the parsing status now we know we will need deps
1056 self.status = BBParsingStatus()
1058 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
1059 self.status.ignored_dependencies = Set( ignore.split() )
1061 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
1063 pkgs_to_build = None
1065 if not pkgs_to_build:
1067 pkgs_to_build.extend(args)
1068 if not pkgs_to_build:
1069 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
1071 pkgs_to_build = bbpkgs.split()
1072 if not pkgs_to_build and not self.configuration.show_versions \
1073 and not self.configuration.interactive \
1074 and not self.configuration.show_environment:
1075 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
1076 print "for usage information."
1079 # Import Psyco if available and not disabled
1080 if not self.configuration.disable_psyco:
1085 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
1087 psyco.bind( self.collect_bbfiles )
1089 bb.note("You have disabled Psyco. This decreases performance.")
1092 bb.debug(1, "collecting .bb files")
1093 self.collect_bbfiles( self.myProgressCallback )
1094 bb.debug(1, "parsing complete")
1097 if self.configuration.parse_only:
1098 print "Requested parsing .bb files only. Exiting."
1102 self.buildDepgraph()
1104 if self.configuration.show_versions:
1107 if 'world' in pkgs_to_build:
1108 self.buildWorldTargetList()
1109 pkgs_to_build.remove('world')
1110 for t in self.status.world_target:
1111 pkgs_to_build.append(t)
1113 if self.configuration.dot_graph:
1114 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
1118 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
1121 for k in pkgs_to_build:
1124 if self.buildProvider( k , False ) == 0:
1127 except bb.build.EventException:
1128 bb.error("Build of " + k + " failed")
1132 failures += failures
1133 if self.configuration.abort:
1136 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
1138 sys.exit( self.stats.show() )
1140 except KeyboardInterrupt:
1141 print "\nNOTE: KeyboardInterrupt - Build not completed."
1144 def get_bbfiles( self, path = os.getcwd() ):
1145 """Get list of default .bb files by reading out the current directory"""
1146 contents = os.listdir(path)
1149 (root, ext) = os.path.splitext(f)
1151 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
1154 def find_bbfiles( self, path ):
1155 """Find all the .bb files in a directory (uses find)"""
1156 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
1158 finddata = os.popen(findcmd)
1161 return finddata.readlines()
1163 def collect_bbfiles( self, progressCallback ):
1164 """Collect all available .bb build files"""
1165 self.cb = progressCallback
1166 parsed, cached, skipped, masked = 0, 0, 0, 0
1167 self.bb_cache = bb.cache.init(self)
1169 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
1170 data.setVar("BBFILES", " ".join(files), self.configuration.data)
1173 files = self.get_bbfiles()
1176 bb.error("no files to build.")
1180 if os.path.isdir(f):
1181 dirfiles = self.find_bbfiles(f)
1183 newfiles += dirfiles
1185 newfiles += glob.glob(f) or [ f ]
1187 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
1189 bbmask_compiled = re.compile(bbmask)
1190 except sre_constants.error:
1191 bb.fatal("BBMASK is not a valid regular expression.")
1193 for i in xrange( len( newfiles ) ):
1195 if bbmask and bbmask_compiled.search(f):
1196 bb.debug(1, "bbmake: skipping %s" % f)
1199 debug(1, "bbmake: parsing %s" % f)
1201 # read a file's metadata
1203 fromCache, skip = self.bb_cache.loadData(f, self)
1206 #bb.note("Skipping %s" % f)
1207 self.bb_cache.skip(f)
1209 elif fromCache: cached += 1
1213 # allow metadata files to add items to BBFILES
1214 #data.update_data(self.pkgdata[f])
1215 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1217 for aof in addbbfiles.split():
1218 if not files.count(aof):
1219 if not os.path.isabs(aof):
1220 aof = os.path.join(os.path.dirname(f),aof)
1223 # now inform the caller
1224 if self.cb is not None:
1225 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
1228 self.bb_cache.remove(f)
1229 bb.error("opening %s: %s" % (f, e))
1231 except KeyboardInterrupt:
1232 self.bb_cache.sync()
1234 except Exception, e:
1235 self.bb_cache.remove(f)
1236 bb.error("%s while parsing %s" % (e, f))
1238 self.bb_cache.remove(f)
1241 if self.cb is not None:
1242 print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ),
1244 self.bb_cache.sync()
1246 #============================================================================#
1248 #============================================================================#
1251 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
1252 usage = """%prog [options] [package ...]
1254 Executes the specified task (default is 'build') for a given set of BitBake files.
1255 It expects that BBFILES is defined, which is a space seperated list of files to
1256 be executed. BBFILES does support wildcards.
1257 Default BBFILES are the .bb files in the current directory.""" )
1259 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
1260 action = "store", dest = "buildfile", default = None )
1262 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
1263 action = "store_false", dest = "abort", default = True )
1265 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1266 action = "store_true", dest = "force", default = False )
1268 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1269 action = "store_true", dest = "interactive", default = False )
1271 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1272 action = "store", dest = "cmd", default = "build" )
1274 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1275 action = "append", dest = "file", default = [] )
1277 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1278 action = "store_true", dest = "verbose", default = False )
1280 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1281 action = "count", dest="debug", default = 0)
1283 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
1284 action = "store_true", dest = "dry_run", default = False )
1286 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
1287 action = "store_true", dest = "parse_only", default = False )
1289 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
1290 action = "store_true", dest = "disable_psyco", default = False )
1292 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
1293 action = "store_true", dest = "show_versions", default = False )
1295 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1296 action = "store_true", dest = "show_environment", default = False )
1298 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
1299 action = "store_true", dest = "dot_graph", default = False )
1300 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
1301 action = "append", dest = "ignored_dot_deps", default = [] )
1304 options, args = parser.parse_args( sys.argv )
1307 cooker.cook( BBConfiguration( options ), args[1:] )
1311 if __name__ == "__main__":
1312 print """WARNING, WARNING, WARNING
1313 This is a Bitbake from the Unstable/Development Branch.
1314 You might want to use the bitbake-1.4 stable branch (if you are not a BitBake developer or tester). I'm going to sleep 5 seconds now to make sure you see that."""