2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if pn not in self.pkg_pn:
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if pn not in self.pn_provides:
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if provide not in self.providers:
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
143 ParsingStatus = BBParsingStatus # make it visible from the shell
144 Statistics = BBStatistics # make it visible from the shell
146 def __init__( self ):
147 self.build_cache_fail = []
148 self.build_cache = []
149 self.building_list = []
151 self.consider_msgs_cache = []
153 self.stats = BBStatistics()
156 def tryBuild( self, fn, virtual ):
157 if fn in self.building_list:
158 bb.error("%s depends on itself (eventually)" % fn)
159 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
162 the_data = make.pkgdata[fn]
163 item = self.status.pkg_fn[fn]
165 self.building_list.append(fn)
167 pathstr = "%s (%s)" % (item, virtual)
168 self.build_path.append(pathstr)
170 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
171 if make.options.verbose:
172 bb.note("current path: %s" % (" -> ".join(self.build_path)))
173 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
178 depcmd = make.options.cmd
179 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
180 if bbdepcmd is not None:
187 oldcmd = make.options.cmd
188 make.options.cmd = depcmd
190 for d in depends_list:
191 if d in self.status.ignored_dependencies:
195 if self.buildPackage(d) == 0:
196 bb.error("dependency %s (for %s) not satisfied" % (d,item))
198 if make.options.abort:
202 make.options.cmd = oldcmd
208 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
209 self.build_cache.append(fn)
212 bb.event.fire(bb.event.PkgStarted(item, the_data))
214 self.stats.attempt += 1
215 if not make.options.dry_run:
216 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
217 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
218 self.build_cache.append(fn)
220 except bb.build.FuncFailed:
222 bb.error("task stack execution failed")
223 bb.event.fire(bb.event.PkgFailed(item, the_data))
224 self.build_cache_fail.append(fn)
226 except bb.build.EventException:
228 (type, value, traceback) = sys.exc_info()
230 bb.error("%s event exception, aborting" % bb.event.getName(e))
231 bb.event.fire(bb.event.PkgFailed(item, the_data))
232 self.build_cache_fail.append(fn)
235 self.building_list.remove(fn)
236 self.build_path.remove(pathstr)
238 def showVersions( self ):
239 pkg_pn = self.status.pkg_pn
240 preferred_versions = {}
244 for pn in pkg_pn.keys():
248 priority = self.status.bbfile_priority[f]
249 if priority not in priorities:
250 priorities[priority] = []
251 priorities[priority].append(f)
252 p_list = priorities.keys()
253 p_list.sort(lambda a, b: a - b)
256 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
258 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
259 # version. If not, find the latest version provided by an bbfile in the
260 # highest-priority set.
261 for pn in pkg_pn.keys():
262 preferred_file = None
264 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
267 m = re.match('(.*)_(.*)', preferred_v)
269 preferred_v = m.group(1)
270 preferred_r = m.group(2)
272 for file_set in pkg_pn[pn]:
274 pv,pr = self.status.pkg_pvpr[f]
275 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
277 preferred_ver = (pv, pr)
282 pv_str = '%s-%s' % (preferred_v, preferred_r)
285 if preferred_file is None:
286 bb.note("preferred version %s of %s not available" % (pv_str, pn))
288 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
290 # get highest priority file set
291 files = pkg_pn[pn][0]
296 pv,pr = self.status.pkg_pvpr[f]
297 dp = self.status.pkg_dp[f]
299 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
303 if preferred_file is None:
304 preferred_file = latest_f
305 preferred_ver = latest
307 preferred_versions[pn] = (preferred_ver, preferred_file)
308 latest_versions[pn] = (latest, latest_f)
310 pkg_list = pkg_pn.keys()
314 pref = preferred_versions[p]
315 latest = latest_versions[p]
318 prefstr = pref[0][0] + "-" + pref[0][1]
322 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
325 def buildPackage( self, item ):
328 discriminated = False
330 if item not in self.status.providers:
331 bb.error("Nothing provides %s" % item)
334 all_p = self.status.providers[item]
337 if p in self.build_cache:
338 bb.debug(1, "already built %s in this run\n" % p)
342 preferred_versions = {}
344 # Collate providers by PN
347 pn = self.status.pkg_fn[p]
352 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
355 for pn in pkg_pn.keys():
359 priority = self.status.bbfile_priority[f]
360 if priority not in priorities:
361 priorities[priority] = []
362 priorities[priority].append(f)
363 p_list = priorities.keys()
364 p_list.sort(lambda a, b: a - b)
367 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
369 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
370 # version. If not, find the latest version provided by an bbfile in the
371 # highest-priority set.
372 for pn in pkg_pn.keys():
373 preferred_file = None
375 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
378 m = re.match('(.*)_(.*)', preferred_v)
380 preferred_v = m.group(1)
381 preferred_r = m.group(2)
383 for file_set in pkg_pn[pn]:
385 pv,pr = self.status.pkg_pvpr[f]
386 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
388 preferred_ver = (pv, pr)
393 pv_str = '%s-%s' % (preferred_v, preferred_r)
396 if preferred_file is None:
397 bb.note("preferred version %s of %s not available" % (pv_str, pn))
399 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
401 if preferred_file is None:
402 # get highest priority file set
403 files = pkg_pn[pn][0]
408 pv,pr = self.status.pkg_pvpr[f]
409 dp = self.status.pkg_dp[f]
411 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
415 preferred_file = latest_f
416 preferred_ver = latest
418 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
420 preferred_versions[pn] = (preferred_ver, preferred_file)
421 eligible.append(preferred_file)
424 if p in self.build_cache_fail:
425 bb.debug(1, "rejecting already-failed %s" % p)
428 if len(eligible) == 0:
429 bb.error("no eligible providers for %s" % item)
432 # look to see if one of them is already staged, or marked as preferred.
433 # if so, bump it to the head of the queue
435 the_data = make.pkgdata[p]
436 pn = bb.data.getVar('PN', the_data, 1)
437 pv = bb.data.getVar('PV', the_data, 1)
438 pr = bb.data.getVar('PR', the_data, 1)
439 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
440 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
441 if os.path.exists(stamp):
442 (newvers, fn) = preferred_versions[pn]
443 if not fn in eligible:
444 # package was made ineligible by already-failed check
446 oldver = "%s-%s" % (pv, pr)
447 newver = '-'.join(newvers)
448 if (newver != oldver):
449 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
452 if make.options.verbose:
453 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
455 eligible = [fn] + eligible
459 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
461 self.preferred[item] = prefervar
463 if item in self.preferred:
465 pn = self.status.pkg_fn[p]
466 if self.preferred[item] == pn:
467 if make.options.verbose:
468 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
470 eligible = [p] + eligible
474 if len(eligible) > 1 and discriminated == False:
475 if item not in self.consider_msgs_cache:
478 providers_list.append(self.status.pkg_fn[fn])
479 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
480 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
481 self.consider_msgs_cache.append(item)
484 # run through the list until we find one that we can build
486 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
487 if self.tryBuild(fn, item):
490 bb.note("no buildable providers for %s" % item)
493 def buildDepgraph( self ):
494 all_depends = self.status.all_depends
495 pn_provides = self.status.pn_provides
497 def calc_bbfile_priority(filename):
498 for (regex, pri) in self.status.bbfile_config_priorities:
499 if regex.match(filename):
503 # Handle PREFERRED_PROVIDERS
504 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
505 (providee, provider) = p.split(':')
506 if providee in self.preferred and self.preferred[providee] != provider:
507 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
508 self.preferred[providee] = provider
510 # Calculate priorities for each file
511 for p in make.pkgdata.keys():
512 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
514 # Build package list for "bitbake world"
515 bb.debug(1, "collating packages for \"world\"")
516 for f in self.status.possible_world:
518 pn = self.status.pkg_fn[f]
520 for p in pn_provides[pn]:
521 if p.startswith('virtual/'):
522 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
525 for pf in self.status.providers[p]:
526 if self.status.pkg_fn[pf] != pn:
527 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
531 self.status.world_target.add(pn)
533 # drop reference count now
534 self.status.possible_world = None
535 self.status.all_depends = None
537 def myProgressCallback( self, x, y, f, file_data, from_cache ):
538 # feed the status with new input
539 self.status.handle_bb_data(f, file_data, from_cache)
543 if os.isatty(sys.stdout.fileno()):
544 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
548 sys.stdout.write("Parsing .bb files, please wait...")
551 sys.stdout.write("done.")
554 def executeOneBB( self, fn ):
556 d = bb.parse.handle(fn, make.cfg)
558 bb.fatal("Unable to open %s" % fn)
560 if make.options.parse_only:
561 print "Requested parsing .bb files only. Exiting."
564 name = bb.data.getVar('PN', d, 1)
565 bb.event.fire(bb.event.PkgStarted(name, d))
567 self.stats.attempt += 1
568 if make.options.force:
569 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
570 if not make.options.dry_run:
571 bb.build.exec_task('do_%s' % make.options.cmd, d)
572 bb.event.fire(bb.event.PkgSucceeded(name, d))
573 self.build_cache.append(fn)
574 except bb.build.FuncFailed:
576 bb.error("task stack execution failed")
577 bb.event.fire(bb.event.PkgFailed(name, d))
578 self.build_cache_fail.append(fn)
579 except bb.build.EventException:
581 (type, value, traceback) = sys.exc_info()
583 bb.error("%s event exception, aborting" % bb.event.getName(e))
584 bb.event.fire(bb.event.PkgFailed(name, d))
585 self.build_cache_fail.append(fn)
587 def interactiveMode( self ):
588 """Drop off into a shell"""
591 except ImportError, details:
592 bb.fatal("Sorry, shell not available (%s)" % details )
597 def parseConfigurationFile( self, afile ):
599 make.cfg = bb.parse.handle( afile, make.cfg )
601 bb.fatal( "Unable to open %s" % afile )
603 def handleCollections( self, collections ):
604 """Handle collections"""
606 collection_list = collections.split()
607 for c in collection_list:
608 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
610 bb.error("BBFILE_PATTERN_%s not defined" % c)
612 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
614 bb.error("BBFILE_PRIORITY_%s not defined" % c)
617 cre = re.compile(regex)
619 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
623 self.status.bbfile_config_priorities.append((cre, pri))
625 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
628 def cook( self, args ):
629 if not make.options.cmd:
630 make.options.cmd = "build"
632 if make.options.debug:
633 bb.debug_level = make.options.debug
635 make.cfg = bb.data.init()
637 for f in make.options.file:
638 self.parseConfigurationFile( f )
640 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
642 if not bb.data.getVar("BUILDNAME", make.cfg):
643 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
645 buildname = bb.data.getVar("BUILDNAME", make.cfg)
647 if make.options.interactive:
648 self.interactiveMode()
650 bf = make.options.buildfile
652 self.executeOneBB( os.path.abspath(bf) )
653 sys.exit( self.stats.show() )
655 # initialise the parsing status now we know we will need deps
656 self.status = BBParsingStatus()
658 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
659 self.status.ignored_dependencies = Set( ignore.split() )
661 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1) )
665 if not pkgs_to_build:
667 pkgs_to_build.extend(args)
668 if not pkgs_to_build:
669 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
671 pkgs_to_build = bbpkgs.split()
672 if not pkgs_to_build and not make.options.show_versions and not make.options.interactive:
673 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
674 print "for usage information."
677 # Import Psyco if available and not disabled
678 if not make.options.disable_psyco:
683 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
685 psyco.bind( make.collect_bbfiles )
687 bb.note("You have disabled Psyco. This decreases performance.")
690 bb.debug(1, "collecting .bb files")
691 make.collect_bbfiles( self.myProgressCallback )
692 bb.debug(1, "parsing complete")
695 if make.options.parse_only:
696 print "Requested parsing .bb files only. Exiting."
701 if make.options.show_versions:
704 if 'world' in pkgs_to_build:
705 pkgs_to_build.remove('world')
706 for t in self.status.world_target:
707 pkgs_to_build.append(t)
709 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
711 for k in pkgs_to_build:
714 if self.buildPackage(k) == 0:
717 except bb.build.EventException:
718 bb.error("Build of " + k + " failed")
722 if make.options.abort:
725 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
727 sys.exit( self.stats.show() )
729 except KeyboardInterrupt:
730 print "\nNOTE: KeyboardInterrupt - Build not completed."
733 #============================================================================#
735 #============================================================================#
737 if __name__ == "__main__":
739 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
740 usage = """%prog [options] [package ...]
742 Executes the specified task (default is 'build') for a given set of BitBake files.
743 It expects that BBFILES is defined, which is a space seperated list of files to
744 be executed. BBFILES does support wildcards.
745 Default BBFILES are the .bb files in the current directory.""" )
747 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
748 action = "store", dest = "buildfile", default = None )
750 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
751 action = "store_false", dest = "abort", default = True )
753 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
754 action = "store_true", dest = "force", default = False )
756 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
757 action = "store_true", dest = "interactive", default = False )
759 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
760 action = "store", dest = "cmd", default = "build" )
762 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
763 action = "append", dest = "file", default = [] )
765 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
766 action = "store_true", dest = "verbose", default = False )
768 parser.add_option( "-D", "--debug", help = "Increase the debug level",
769 action = "count", dest="debug", default = 0)
771 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
772 action = "store_true", dest = "dry_run", default = False )
774 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
775 action = "store_true", dest = "parse_only", default = False )
777 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
778 action = "store_true", dest = "disable_psyco", default = False )
780 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
781 action = "store_true", dest = "show_versions", default = False )
783 options, args = parser.parse_args( sys.argv )
785 make.options = options
787 cooker.cook( args[1:] )