2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re, time
25 sys.path.insert(0,os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
27 from bb import utils, data, parse, debug, event, fatal, cache
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
50 self.packages_dynamic = {}
51 self.bbfile_priority = {}
52 self.bbfile_config_priorities = []
53 self.ignored_dependencies = None
54 self.possible_world = []
55 self.world_target = Set()
61 self.all_depends = Set()
67 def handle_bb_data(self, file_name, bb_cache, cached):
69 We will fill the dictionaries with the stuff we
70 need for building the tree more fast
73 pn = bb_cache.getVar('PN', file_name, True)
74 pv = bb_cache.getVar('PV', file_name, True)
75 pr = bb_cache.getVar('PR', file_name, True)
76 dp = int(bb_cache.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
77 provides = Set([pn] + (bb_cache.getVar("PROVIDES", file_name, True) or "").split())
78 depends = (bb_cache.getVar("DEPENDS", file_name, True) or "").split()
79 packages = (bb_cache.getVar('PACKAGES', file_name, True) or "").split()
80 packages_dynamic = (bb_cache.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
81 rprovides = (bb_cache.getVar("RPROVIDES", file_name, True) or "").split()
83 # build PackageName to FileName lookup table
84 if pn not in self.pkg_pn:
86 self.pkg_pn[pn].append(file_name)
88 self.build_all[file_name] = int(bb_cache.getVar('BUILD_ALL_DEPS', file_name, True) or "0")
89 self.stamp[file_name] = bb_cache.getVar('STAMP', file_name, True)
91 # build FileName to PackageName lookup table
92 self.pkg_fn[file_name] = pn
93 self.pkg_pvpr[file_name] = (pv,pr)
94 self.pkg_dp[file_name] = dp
96 # Build forward and reverse provider hashes
97 # Forward: virtual -> [filenames]
98 # Reverse: PN -> [virtuals]
99 if pn not in self.pn_provides:
100 self.pn_provides[pn] = Set()
101 self.pn_provides[pn] |= provides
103 for provide in provides:
104 if provide not in self.providers:
105 self.providers[provide] = []
106 self.providers[provide].append(file_name)
109 self.all_depends.add(dep)
111 # Build reverse hash for PACKAGES, so runtime dependencies
112 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
113 for package in packages:
114 if not package in self.packages:
115 self.packages[package] = []
116 self.packages[package].append(file_name)
117 rprovides += (bb_cache.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
119 for package in packages_dynamic:
120 if not package in self.packages_dynamic:
121 self.packages_dynamic[package] = []
122 self.packages_dynamic[package].append(file_name)
124 for rprovide in rprovides:
125 if not rprovide in self.rproviders:
126 self.rproviders[rprovide] = []
127 self.rproviders[rprovide].append(file_name)
129 # Build hash of runtime depeneds and rececommends
131 def add_dep(deplist, deps):
133 if not dep in deplist:
136 if not file_name in self.rundeps:
137 self.rundeps[file_name] = {}
138 if not file_name in self.runrecs:
139 self.runrecs[file_name] = {}
141 for package in packages + [pn]:
142 if not package in self.rundeps[file_name]:
143 self.rundeps[file_name][package] = {}
144 if not package in self.runrecs[file_name]:
145 self.runrecs[file_name][package] = {}
147 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RDEPENDS', file_name, True) or ""))
148 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar('RRECOMMENDS', file_name, True) or ""))
149 add_dep(self.rundeps[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
150 add_dep(self.runrecs[file_name][package], bb.utils.explode_deps(bb_cache.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
152 # Collect files we may need for possible world-dep
154 if not bb_cache.getVar('BROKEN', file_name, True) and not bb_cache.getVar('EXCLUDE_FROM_WORLD', file_name, True):
155 self.possible_world.append(file_name)
158 #============================================================================#
160 #============================================================================#
163 Manage build statistics for one run
172 print "Build statistics:"
173 print " Attempted builds: %d" % self.attempt
175 print " Failed builds: %d" % self.fail
177 print " Dependencies not satisfied: %d" % self.deps
178 if self.fail or self.deps: return 1
182 #============================================================================#
184 #============================================================================#
185 class BBConfiguration( object ):
187 Manages build options and configurations for one run
189 def __init__( self, options ):
190 for key, val in options.__dict__.items():
191 setattr( self, key, val )
193 #============================================================================#
195 #============================================================================#
198 Manages one bitbake build run
201 ParsingStatus = BBParsingStatus # make it visible from the shell
202 Statistics = BBStatistics # make it visible from the shell
204 def __init__( self ):
205 self.build_cache_fail = []
206 self.build_cache = []
207 self.rbuild_cache = []
208 self.building_list = []
210 self.consider_msgs_cache = []
212 self.stats = BBStatistics()
218 def tryBuildPackage( self, fn, item, the_data ):
219 """Build one package"""
220 bb.event.fire(bb.event.PkgStarted(item, the_data))
222 self.stats.attempt += 1
223 if self.configuration.force:
224 bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
225 if not self.configuration.dry_run:
226 bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
227 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
228 self.build_cache.append(fn)
230 except bb.build.FuncFailed:
232 bb.error("task stack execution failed")
233 bb.event.fire(bb.event.PkgFailed(item, the_data))
234 self.build_cache_fail.append(fn)
236 except bb.build.EventException, e:
239 bb.error("%s event exception, aborting" % bb.event.getName(event))
240 bb.event.fire(bb.event.PkgFailed(item, the_data))
241 self.build_cache_fail.append(fn)
244 def tryBuild( self, fn, virtual , buildAllDeps , build_depends = []):
246 Build a provider and its dependencies.
247 build_depends is a list of previous build dependencies (not runtime)
248 If build_depends is empty, we're dealing with a runtime depends
251 the_data = self.bb_cache.loadDataFull(fn, self)
253 # Only follow all (runtime) dependencies if doing a build
254 if not buildAllDeps and self.configuration.cmd is "build":
255 buildAllDeps = self.status.build_all[fn]
257 # Error on build time dependency loops
258 if build_depends and build_depends.count(fn) > 1:
259 bb.error("%s depends on itself (eventually)" % fn)
260 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
263 # See if this is a runtime dependency we've already built
264 # Or a build dependency being handled in a different build chain
265 if fn in self.building_list:
266 return self.addRunDeps(fn, virtual , buildAllDeps)
268 item = self.status.pkg_fn[fn]
270 self.building_list.append(fn)
272 pathstr = "%s (%s)" % (item, virtual)
273 self.build_path.append(pathstr)
275 depends_list = (bb.data.getVar('DEPENDS', the_data, True) or "").split()
277 if self.configuration.verbose:
278 bb.note("current path: %s" % (" -> ".join(self.build_path)))
279 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
284 depcmd = self.configuration.cmd
285 bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
286 if bbdepcmd is not None:
293 oldcmd = self.configuration.cmd
294 self.configuration.cmd = depcmd
296 for dependency in depends_list:
297 if dependency in self.status.ignored_dependencies:
301 if self.buildProvider( dependency , buildAllDeps , build_depends ) == 0:
302 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
304 if self.configuration.abort:
308 self.configuration.cmd = oldcmd
314 if not self.addRunDeps(fn, virtual , buildAllDeps):
317 if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
318 self.build_cache.append(fn)
321 return self.tryBuildPackage( fn, item, the_data )
324 self.building_list.remove(fn)
325 self.build_path.remove(pathstr)
327 def findBestProvider( self, pn, pkg_pn = None):
329 If there is a PREFERRED_VERSION, find the highest-priority bbfile
330 providing that version. If not, find the latest version provided by
331 an bbfile in the highest-priority set.
334 pkg_pn = self.status.pkg_pn
339 priority = self.status.bbfile_priority[f]
340 if priority not in priorities:
341 priorities[priority] = []
342 priorities[priority].append(f)
343 p_list = priorities.keys()
344 p_list.sort(lambda a, b: a - b)
347 tmp_pn = [priorities[p]] + tmp_pn
349 preferred_file = None
351 localdata = data.createCopy(self.configuration.data)
352 bb.data.setVar('OVERRIDES', "%s:%s" % (pn, data.getVar('OVERRIDES', localdata)), localdata)
353 bb.data.update_data(localdata)
355 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, localdata, True)
357 m = re.match('(.*)_(.*)', preferred_v)
359 preferred_v = m.group(1)
360 preferred_r = m.group(2)
364 for file_set in tmp_pn:
366 pv,pr = self.status.pkg_pvpr[f]
367 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
369 preferred_ver = (pv, pr)
374 pv_str = '%s-%s' % (preferred_v, preferred_r)
377 if preferred_file is None:
378 bb.note("preferred version %s of %s not available" % (pv_str, pn))
380 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
384 # get highest priority file set
389 for file_name in files:
390 pv,pr = self.status.pkg_pvpr[file_name]
391 dp = self.status.pkg_dp[file_name]
393 if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
397 if preferred_file is None:
398 preferred_file = latest_f
399 preferred_ver = latest
401 return (latest,latest_f,preferred_ver, preferred_file)
403 def showVersions( self ):
404 pkg_pn = self.status.pkg_pn
405 preferred_versions = {}
409 for pn in pkg_pn.keys():
410 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
411 preferred_versions[pn] = (pref_ver, pref_file)
412 latest_versions[pn] = (last_ver, last_file)
414 pkg_list = pkg_pn.keys()
418 pref = preferred_versions[p]
419 latest = latest_versions[p]
422 prefstr = pref[0][0] + "-" + pref[0][1]
426 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
430 def showEnvironment( self ):
431 """Show the outer or per-package environment"""
432 if self.configuration.buildfile:
434 self.bb_cache = bb.cache.init(self)
436 self.configuration.data = self.bb_cache.loadDataFull(self.configuration.buildfile, self)
438 fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
441 # emit variables and shell functions
443 data.update_data( self.configuration.data )
444 data.emit_env(sys.__stdout__, self.configuration.data, True)
447 # emit the metadata which isnt valid shell
448 for e in self.configuration.data.keys():
449 if data.getVarFlag( e, 'python', self.configuration.data ):
450 sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
452 def generateDotGraph( self, pkgs_to_build, ignore_deps ):
454 Generate two graphs one for the DEPENDS and RDEPENDS. The current
455 implementation creates crappy graphs ;)
457 pkgs_to_build A list of packages that needs to be built
458 ignore_deps A list of names where processing of dependencies
459 should be stopped. e.g. dependencies that get
462 def myFilterProvider( providers, item):
464 Take a list of providers and filter according to environment
465 variables. In contrast to filterProviders we do not discriminate
466 and take PREFERRED_PROVIDER into account.
469 preferred_versions = {}
471 # Collate providers by PN
474 pn = self.status.pkg_fn[p]
479 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
481 for pn in pkg_pn.keys():
482 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
483 eligible.append(preferred_versions[pn][1])
486 if p in self.build_cache_fail:
487 bb.debug(1, "rejecting already-failed %s" % p)
490 if len(eligible) == 0:
491 bb.error("no eligible providers for %s" % item)
494 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % package, self.configuration.data, 1)
496 # try the preferred provider first
499 if prefervar == self.status.pkg_fn[p]:
500 bb.note("Selecting PREFERRED_PROVIDER %s" % prefervar)
502 elligible = [p] + elligible
509 # try to avoid adding the same rdepends over an over again
514 def add_depends(package_list):
516 Add all depends of all packages from this list
518 for package in package_list:
519 if package in seen_depends or package in ignore_deps:
522 seen_depends.append( package )
523 if not package in self.status.providers:
525 We have not seen this name -> error in
528 bb.note( "ERROR with provider: %(package)s" % vars() )
529 print >> depends_file, '"%(package)s" -> ERROR' % vars()
532 # get all providers for this package
533 providers = self.status.providers[package]
535 # now let us find the bestProvider for it
536 fn = self.filterProviders(providers, package)[0]
538 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
539 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
540 add_depends ( depends )
542 # now create the node
543 print >> depends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
545 depends = filter( (lambda x: x not in ignore_deps), depends )
546 for depend in depends:
547 print >> depends_file, '"%(package)s" -> "%(depend)s"' % vars()
550 def add_all_depends( package_list ):
552 Add both DEPENDS and RDEPENDS. RDEPENDS will get dashed
555 for package in package_list:
556 if package in seen_rdepends or package in ignore_deps:
559 seen_rdepends.append( package )
560 if not package in self.status.providers:
562 We have not seen this name -> error in
565 bb.note( "ERROR with provider: %(package)s" % vars() )
566 print >> alldepends_file, '"%(package)s" -> ERROR' % vars()
569 if package == "task-bootstrap":
570 print "PackageList: %s" % package_list
572 # get all providers for this package
573 providers = self.status.providers[package]
575 # now let us find the bestProvider for it
576 fn = self.filterProviders(providers, package)[0]
578 depends = bb.utils.explode_deps(self.bb_cache.getVar('DEPENDS', fn, True) or "")
579 rdepends = bb.utils.explode_deps(self.bb_cache.getVar('RDEPENDS', fn, True) or "")
580 version = self.bb_cache.getVar('PV', fn, True ) + '-' + self.bb_cache.getVar('PR', fn, True)
581 add_all_depends ( depends+rdepends )
583 # now create the node
584 print >> alldepends_file, '"%(package)s" [label="%(package)s\\n%(version)s"]' % vars()
586 depends = filter( (lambda x: x not in ignore_deps), depends )
587 rdepends = filter( (lambda x: x not in ignore_deps), rdepends )
588 for depend in depends:
589 print >> alldepends_file, '"%(package)s" -> "%(depend)s"' % vars()
590 for depend in rdepends:
591 print >> alldepends_file, '"%(package)s" -> "%(depend)s" [style=dashed]' % vars()
595 depends_file = file('depends.dot', 'w' )
596 print >> depends_file, "digraph depends {"
597 add_depends( pkgs_to_build )
598 print >> depends_file, "}"
600 # Add all depends now
601 alldepends_file = file('alldepends.dot', 'w' )
602 print >> alldepends_file, "digraph alldepends {"
603 add_all_depends( pkgs_to_build )
604 print >> alldepends_file, "}"
606 def filterProviders(self, providers, item):
608 Take a list of providers and filter/reorder according to the
609 environment variables and previous build results
612 preferred_versions = {}
614 # Collate providers by PN
617 pn = self.status.pkg_fn[p]
622 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
624 for pn in pkg_pn.keys():
625 preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
626 eligible.append(preferred_versions[pn][1])
629 if p in self.build_cache_fail:
630 bb.debug(1, "rejecting already-failed %s" % p)
633 if len(eligible) == 0:
634 bb.error("no eligible providers for %s" % item)
637 # look to see if one of them is already staged, or marked as preferred.
638 # if so, bump it to the head of the queue
640 pn = self.status.pkg_fn[p]
641 pv, pr = self.status.pkg_pvpr[p]
643 stamp = '%s.do_populate_staging' % self.status.stamp[p]
644 if os.path.exists(stamp):
645 (newvers, fn) = preferred_versions[pn]
646 if not fn in eligible:
647 # package was made ineligible by already-failed check
649 oldver = "%s-%s" % (pv, pr)
650 newver = '-'.join(newvers)
651 if (newver != oldver):
652 extra_chat = "%s (%s) already staged but upgrading to %s to satisfy %s" % (pn, oldver, newver, item)
654 extra_chat = "Selecting already-staged %s (%s) to satisfy %s" % (pn, oldver, item)
655 if self.configuration.verbose:
656 bb.note("%s" % extra_chat)
658 eligible = [fn] + eligible
664 def buildProvider( self, item , buildAllDeps , build_depends = [] ):
666 Build something to provide a named build requirement
667 (takes item names from DEPENDS namespace)
671 discriminated = False
673 if not item in self.status.providers:
674 bb.error("Nothing provides dependency %s" % item)
675 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
678 all_p = self.status.providers[item]
681 if p in self.build_cache:
682 bb.debug(1, "already built %s in this run\n" % p)
685 eligible = self.filterProviders(all_p, item)
690 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
692 self.preferred[item] = prefervar
694 if item in self.preferred:
696 pn = self.status.pkg_fn[p]
697 if self.preferred[item] == pn:
698 if self.configuration.verbose:
699 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
701 eligible = [p] + eligible
705 if len(eligible) > 1 and discriminated == False:
706 if item not in self.consider_msgs_cache:
709 providers_list.append(self.status.pkg_fn[fn])
710 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
711 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
712 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data))
713 self.consider_msgs_cache.append(item)
716 # run through the list until we find one that we can build
718 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
719 if self.tryBuild(fn, item, buildAllDeps, build_depends + [fn]):
722 bb.note("no buildable providers for %s" % item)
723 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
726 def buildRProvider( self, item , buildAllDeps ):
728 Build something to provide a named runtime requirement
729 (takes item names from RDEPENDS/PACKAGES namespace)
734 discriminated = False
739 all_p = self.getProvidersRun(item)
742 bb.error("Nothing provides runtime dependency %s" % (item))
743 bb.event.fire(bb.event.NoProvider(item,self.configuration.data,runtime=True))
747 if p in self.rbuild_cache:
748 bb.debug(2, "Already built %s providing runtime %s\n" % (p,item))
750 if p in self.build_cache:
751 bb.debug(2, "Already built %s but adding any further RDEPENDS for %s\n" % (p, item))
752 return self.addRunDeps(p, item , buildAllDeps)
754 eligible = self.filterProviders(all_p, item)
760 pn = self.status.pkg_fn[p]
761 provides = self.status.pn_provides[pn]
762 for provide in provides:
763 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % provide, self.configuration.data, 1)
765 if self.configuration.verbose:
766 bb.note("selecting %s to satisfy runtime %s due to PREFERRED_PROVIDERS" % (pn, item))
768 eligible = [p] + eligible
771 if len(eligible) > 1 and len(preferred) == 0:
772 if item not in self.consider_msgs_cache:
775 providers_list.append(self.status.pkg_fn[fn])
776 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
777 bb.note("consider defining a PREFERRED_PROVIDER to match runtime %s" % item)
778 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
779 self.consider_msgs_cache.append(item)
781 if len(preferred) > 1:
782 if item not in self.consider_msgs_cache:
785 providers_list.append(self.status.pkg_fn[fn])
786 bb.note("multiple preferred providers are available (%s);" % ", ".join(providers_list))
787 bb.note("consider defining only one PREFERRED_PROVIDER to match runtime %s" % item)
788 bb.event.fire(bb.event.MultipleProviders(item,providers_list,self.configuration.data,runtime=True))
789 self.consider_msgs_cache.append(item)
791 # run through the list until we find one that we can build
793 bb.debug(2, "selecting %s to satisfy runtime %s" % (fn, item))
794 if self.tryBuild(fn, item, buildAllDeps):
797 bb.error("No buildable providers for runtime %s" % item)
798 bb.event.fire(bb.event.NoProvider(item,self.configuration.data))
801 def getProvidersRun(self, rdepend):
803 Return any potential providers of runtime rdepend
807 if rdepend in self.status.rproviders:
808 rproviders += self.status.rproviders[rdepend]
810 if rdepend in self.status.packages:
811 rproviders += self.status.packages[rdepend]
816 # Only search dynamic packages if we can't find anything in other variables
817 for pattern in self.status.packages_dynamic:
818 regexp = re.compile(pattern)
819 if regexp.match(rdepend):
820 rproviders += self.status.packages_dynamic[pattern]
824 def addRunDeps(self , fn, item , buildAllDeps):
826 Add any runtime dependencies of runtime item provided by fn
827 as long as item has't previously been processed by this function.
830 if item in self.rbuild_cache:
837 self.rbuild_cache.append(item)
839 if fn in self.status.rundeps and item in self.status.rundeps[fn]:
840 rdepends += self.status.rundeps[fn][item].keys()
841 if fn in self.status.runrecs and item in self.status.runrecs[fn]:
842 rdepends += self.status.runrecs[fn][item].keys()
844 bb.debug(2, "Additional runtime dependencies for %s are: %s" % (item, " ".join(rdepends)))
846 for rdepend in rdepends:
847 if rdepend in self.status.ignored_dependencies:
849 if not self.buildRProvider(rdepend, buildAllDeps):
853 def buildDepgraph( self ):
854 all_depends = self.status.all_depends
855 pn_provides = self.status.pn_provides
857 localdata = data.createCopy(self.configuration.data)
858 bb.data.update_data(localdata)
860 def calc_bbfile_priority(filename):
861 for (regex, pri) in self.status.bbfile_config_priorities:
862 if regex.match(filename):
866 # Handle PREFERRED_PROVIDERS
867 for p in (bb.data.getVar('PREFERRED_PROVIDERS', localdata, 1) or "").split():
868 (providee, provider) = p.split(':')
869 if providee in self.preferred and self.preferred[providee] != provider:
870 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
871 self.preferred[providee] = provider
873 # Calculate priorities for each file
874 for p in self.status.pkg_fn.keys():
875 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
877 def buildWorldTargetList(self):
879 Build package list for "bitbake world"
881 all_depends = self.status.all_depends
882 pn_provides = self.status.pn_provides
883 bb.debug(1, "collating packages for \"world\"")
884 for f in self.status.possible_world:
886 pn = self.status.pkg_fn[f]
888 for p in pn_provides[pn]:
889 if p.startswith('virtual/'):
890 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
893 for pf in self.status.providers[p]:
894 if self.status.pkg_fn[pf] != pn:
895 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
899 self.status.world_target.add(pn)
901 # drop reference count now
902 self.status.possible_world = None
903 self.status.all_depends = None
905 def myProgressCallback( self, x, y, f, bb_cache, from_cache ):
906 # feed the status with new input
908 self.status.handle_bb_data(f, bb_cache, from_cache)
912 if os.isatty(sys.stdout.fileno()):
913 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
917 sys.stdout.write("Parsing .bb files, please wait...")
920 sys.stdout.write("done.")
923 def interactiveMode( self ):
924 """Drop off into a shell"""
927 except ImportError, details:
928 bb.fatal("Sorry, shell not available (%s)" % details )
930 bb.data.update_data( self.configuration.data )
934 def parseConfigurationFile( self, afile ):
936 self.configuration.data = bb.parse.handle( afile, self.configuration.data )
938 # Add the handlers we inherited by INHERIT
939 # we need to do this manually as it is not guranteed
940 # we will pick up these classes... as we only INHERIT
941 # on .inc and .bb files but not on .conf
942 data = bb.data.createCopy( self.configuration.data )
943 inherits = ["base"] + (bb.data.getVar('INHERIT', data, True ) or "").split()
944 for inherit in inherits:
945 data = bb.parse.handle( os.path.join('classes', '%s.bbclass' % inherit ), data, True )
947 # FIXME: This assumes that we included at least one .inc file
948 for var in bb.data.keys(data):
949 if bb.data.getVarFlag(var, 'handler', data):
950 bb.event.register(var,bb.data.getVar(var, data))
953 bb.fatal( "Unable to open %s" % afile )
954 except bb.parse.ParseError, details:
955 bb.fatal( "Unable to parse %s (%s)" % (afile, details) )
957 def handleCollections( self, collections ):
958 """Handle collections"""
960 collection_list = collections.split()
961 for c in collection_list:
962 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
964 bb.error("BBFILE_PATTERN_%s not defined" % c)
966 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
968 bb.error("BBFILE_PRIORITY_%s not defined" % c)
971 cre = re.compile(regex)
973 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
977 self.status.bbfile_config_priorities.append((cre, pri))
979 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
982 def cook( self, configuration, args ):
984 We are building stuff here. We do the building
985 from here. By default we try to execute task
989 self.configuration = configuration
991 if not self.configuration.cmd:
992 self.configuration.cmd = "build"
994 if self.configuration.debug:
995 bb.debug_level = self.configuration.debug
997 self.configuration.data = bb.data.init()
999 for f in self.configuration.file:
1000 self.parseConfigurationFile( f )
1002 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
1006 # Special updated configuration we use for firing events
1008 self.configuration.event_data = bb.data.createCopy(self.configuration.data)
1009 bb.data.update_data(self.configuration.event_data)
1011 if self.configuration.show_environment:
1012 self.showEnvironment()
1015 # inject custom variables
1016 if not bb.data.getVar("BUILDNAME", self.configuration.data):
1017 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
1018 bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
1020 buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
1022 if self.configuration.interactive:
1023 self.interactiveMode()
1025 if self.configuration.buildfile is not None:
1026 bf = os.path.abspath( self.configuration.buildfile )
1028 bbfile_data = bb.parse.handle(bf, self.configuration.data)
1030 bb.fatal("Unable to open %s" % bf)
1032 item = bb.data.getVar('PN', bbfile_data, 1)
1034 self.tryBuildPackage( bf, item, bbfile_data )
1035 except bb.build.EventException:
1036 bb.error( "Build of '%s' failed" % item )
1038 sys.exit( self.stats.show() )
1040 # initialise the parsing status now we know we will need deps
1041 self.status = BBParsingStatus()
1043 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
1044 self.status.ignored_dependencies = Set( ignore.split() )
1046 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
1048 pkgs_to_build = None
1050 if not pkgs_to_build:
1052 pkgs_to_build.extend(args)
1053 if not pkgs_to_build:
1054 bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
1056 pkgs_to_build = bbpkgs.split()
1057 if not pkgs_to_build and not self.configuration.show_versions \
1058 and not self.configuration.interactive \
1059 and not self.configuration.show_environment:
1060 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
1061 print "for usage information."
1064 # Import Psyco if available and not disabled
1065 if not self.configuration.disable_psyco:
1070 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
1072 psyco.bind( self.collect_bbfiles )
1074 bb.note("You have disabled Psyco. This decreases performance.")
1077 bb.debug(1, "collecting .bb files")
1078 self.collect_bbfiles( self.myProgressCallback )
1079 bb.debug(1, "parsing complete")
1082 if self.configuration.parse_only:
1083 print "Requested parsing .bb files only. Exiting."
1087 self.buildDepgraph()
1089 if self.configuration.show_versions:
1092 if 'world' in pkgs_to_build:
1093 self.buildWorldTargetList()
1094 pkgs_to_build.remove('world')
1095 for t in self.status.world_target:
1096 pkgs_to_build.append(t)
1098 if self.configuration.dot_graph:
1099 self.generateDotGraph( pkgs_to_build, self.configuration.ignored_dot_deps )
1103 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.event_data))
1106 for k in pkgs_to_build:
1109 if self.buildProvider( k , False ) == 0:
1112 except bb.build.EventException:
1113 bb.error("Build of " + k + " failed")
1117 failures += failures
1118 if self.configuration.abort:
1121 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.event_data, failures))
1123 sys.exit( self.stats.show() )
1125 except KeyboardInterrupt:
1126 print "\nNOTE: KeyboardInterrupt - Build not completed."
1129 def get_bbfiles( self, path = os.getcwd() ):
1130 """Get list of default .bb files by reading out the current directory"""
1131 contents = os.listdir(path)
1134 (root, ext) = os.path.splitext(f)
1136 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
1139 def find_bbfiles( self, path ):
1140 """Find all the .bb files in a directory (uses find)"""
1141 findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
1143 finddata = os.popen(findcmd)
1146 return finddata.readlines()
1148 def collect_bbfiles( self, progressCallback ):
1149 """Collect all available .bb build files"""
1150 self.cb = progressCallback
1151 parsed, cached, skipped, masked = 0, 0, 0, 0
1152 self.bb_cache = bb.cache.init(self)
1154 files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
1155 data.setVar("BBFILES", " ".join(files), self.configuration.data)
1158 files = self.get_bbfiles()
1161 bb.error("no files to build.")
1165 if os.path.isdir(f):
1166 dirfiles = self.find_bbfiles(f)
1168 newfiles += dirfiles
1170 newfiles += glob.glob(f) or [ f ]
1172 bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
1174 bbmask_compiled = re.compile(bbmask)
1175 except sre_constants.error:
1176 bb.fatal("BBMASK is not a valid regular expression.")
1178 for i in xrange( len( newfiles ) ):
1180 if bbmask and bbmask_compiled.search(f):
1181 bb.debug(1, "bbmake: skipping %s" % f)
1184 debug(1, "bbmake: parsing %s" % f)
1186 # read a file's metadata
1188 fromCache, skip = self.bb_cache.loadData(f, self)
1191 #bb.note("Skipping %s" % f)
1192 self.bb_cache.skip(f)
1194 elif fromCache: cached += 1
1198 # allow metadata files to add items to BBFILES
1199 #data.update_data(self.pkgdata[f])
1200 addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
1202 for aof in addbbfiles.split():
1203 if not files.count(aof):
1204 if not os.path.isabs(aof):
1205 aof = os.path.join(os.path.dirname(f),aof)
1208 # now inform the caller
1209 if self.cb is not None:
1210 self.cb( i + 1, len( newfiles ), f, self.bb_cache, fromCache )
1213 self.bb_cache.remove(f)
1214 bb.error("opening %s: %s" % (f, e))
1216 except KeyboardInterrupt:
1217 self.bb_cache.sync()
1219 except Exception, e:
1220 self.bb_cache.remove(f)
1221 bb.error("%s while parsing %s" % (e, f))
1223 self.bb_cache.remove(f)
1226 if self.cb is not None:
1227 print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ),
1229 self.bb_cache.sync()
1231 #============================================================================#
1233 #============================================================================#
1236 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
1237 usage = """%prog [options] [package ...]
1239 Executes the specified task (default is 'build') for a given set of BitBake files.
1240 It expects that BBFILES is defined, which is a space seperated list of files to
1241 be executed. BBFILES does support wildcards.
1242 Default BBFILES are the .bb files in the current directory.""" )
1244 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
1245 action = "store", dest = "buildfile", default = None )
1247 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
1248 action = "store_false", dest = "abort", default = True )
1250 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
1251 action = "store_true", dest = "force", default = False )
1253 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode also called the BitBake shell.",
1254 action = "store_true", dest = "interactive", default = False )
1256 parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtaks tasks is defined and will show available tasks",
1257 action = "store", dest = "cmd", default = "build" )
1259 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
1260 action = "append", dest = "file", default = [] )
1262 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
1263 action = "store_true", dest = "verbose", default = False )
1265 parser.add_option( "-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
1266 action = "count", dest="debug", default = 0)
1268 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
1269 action = "store_true", dest = "dry_run", default = False )
1271 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
1272 action = "store_true", dest = "parse_only", default = False )
1274 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
1275 action = "store_true", dest = "disable_psyco", default = False )
1277 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
1278 action = "store_true", dest = "show_versions", default = False )
1280 parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
1281 action = "store_true", dest = "show_environment", default = False )
1283 parser.add_option( "-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax",
1284 action = "store_true", dest = "dot_graph", default = False )
1285 parser.add_option( "-I", "--ignore-deps", help = """Stop processing at the given list of dependencies when generating dependency graphs. This can help to make the graph more appealing""",
1286 action = "append", dest = "ignored_dot_deps", default = [] )
1289 options, args = parser.parse_args( sys.argv )
1292 cooker.cook( BBConfiguration( options ), args[1:] )
1296 if __name__ == "__main__":
1297 print """WARNING, WARNING, WARNING
1298 This is a Bitbake from the Unstable/Development Branch.
1299 You might want to use the bitbake-1.4 stable branch (if you are not a BitBake developer or tester). I'm going to sleep 5 seconds now to make sure you see that."""