2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if pn not in self.pkg_pn:
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if pn not in self.pn_provides:
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if provide not in self.providers:
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
143 ParsingStatus = BBParsingStatus # make it visible from the shell
144 Statistics = BBStatistics # make it visible from the shell
146 def __init__( self ):
147 self.build_cache_fail = []
148 self.build_cache = []
149 self.building_list = []
151 self.consider_msgs_cache = []
153 self.stats = BBStatistics()
156 def tryBuildPackage( self, fn, item, the_data ):
157 print ">>>tryBuildPackage. fn = '%s', item = '%s', the_data = '%s'" % (fn, item, the_data )
158 """Build one package"""
159 bb.event.fire(bb.event.PkgStarted(item, the_data))
161 self.stats.attempt += 1
162 if make.options.force:
163 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
164 if not make.options.dry_run:
165 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
166 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
167 self.build_cache.append(fn)
169 except bb.build.FuncFailed:
171 bb.error("task stack execution failed")
172 bb.event.fire(bb.event.PkgFailed(item, the_data))
173 self.build_cache_fail.append(fn)
175 except bb.build.EventException:
177 (type, value, traceback) = sys.exc_info()
179 bb.error("%s event exception, aborting" % bb.event.getName(e))
180 bb.event.fire(bb.event.PkgFailed(item, the_data))
181 self.build_cache_fail.append(fn)
184 def tryBuild( self, fn, virtual ):
185 """Build a provider and its dependencies"""
186 if fn in self.building_list:
187 bb.error("%s depends on itself (eventually)" % fn)
188 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
191 the_data = make.pkgdata[fn]
192 item = self.status.pkg_fn[fn]
194 self.building_list.append(fn)
196 pathstr = "%s (%s)" % (item, virtual)
197 self.build_path.append(pathstr)
199 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
200 if make.options.verbose:
201 bb.note("current path: %s" % (" -> ".join(self.build_path)))
202 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
207 depcmd = make.options.cmd
208 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
209 if bbdepcmd is not None:
216 oldcmd = make.options.cmd
217 make.options.cmd = depcmd
219 for dependency in depends_list:
220 if dependency in self.status.ignored_dependencies:
224 if self.buildProvider( dependency ) == 0:
225 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
227 if make.options.abort:
231 make.options.cmd = oldcmd
237 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
238 self.build_cache.append(fn)
241 return self.tryBuildPackage( fn, item, the_data )
244 self.building_list.remove(fn)
245 self.build_path.remove(pathstr)
247 def showVersions( self ):
248 pkg_pn = self.status.pkg_pn
249 preferred_versions = {}
253 for pn in pkg_pn.keys():
257 priority = self.status.bbfile_priority[f]
258 if priority not in priorities:
259 priorities[priority] = []
260 priorities[priority].append(f)
261 p_list = priorities.keys()
262 p_list.sort(lambda a, b: a - b)
265 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
267 for pn in pkg_pn.keys():
268 preferred_file = None
269 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
272 m = re.match('(.*)_(.*)', preferred_v)
274 preferred_v = m.group(1)
275 preferred_r = m.group(2)
277 for file_set in pkg_pn[pn]:
279 pv,pr = self.status.pkg_pvpr[f]
280 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
282 preferred_ver = (pv, pr)
287 pv_str = '%s-%s' % (preferred_v, preferred_r)
290 if preferred_file is None:
291 bb.note("preferred version %s of %s not available" % (pv_str, pn))
293 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
295 # get highest priority file set
296 files = pkg_pn[pn][0]
301 pv,pr = self.status.pkg_pvpr[f]
302 dp = self.status.pkg_dp[f]
304 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
308 if preferred_file is None:
309 preferred_file = latest_f
310 preferred_ver = latest
312 preferred_versions[pn] = (preferred_ver, preferred_file)
313 latest_versions[pn] = (latest, latest_f)
315 pkg_list = pkg_pn.keys()
319 pref = preferred_versions[p]
320 latest = latest_versions[p]
323 prefstr = pref[0][0] + "-" + pref[0][1]
327 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
330 def buildProvider( self, item ):
333 discriminated = False
335 if item not in self.status.providers:
336 bb.error("Nothing provides %s" % item)
339 all_p = self.status.providers[item]
342 if p in self.build_cache:
343 bb.debug(1, "already built %s in this run\n" % p)
347 preferred_versions = {}
349 # Collate providers by PN
352 pn = self.status.pkg_fn[p]
357 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
360 for pn in pkg_pn.keys():
364 priority = self.status.bbfile_priority[f]
365 if priority not in priorities:
366 priorities[priority] = []
367 priorities[priority].append(f)
368 p_list = priorities.keys()
369 p_list.sort(lambda a, b: a - b)
372 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
374 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
375 # version. If not, find the latest version provided by an bbfile in the
376 # highest-priority set.
377 for pn in pkg_pn.keys():
378 preferred_file = None
380 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
383 m = re.match('(.*)_(.*)', preferred_v)
385 preferred_v = m.group(1)
386 preferred_r = m.group(2)
388 for file_set in pkg_pn[pn]:
390 pv,pr = self.status.pkg_pvpr[f]
391 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
393 preferred_ver = (pv, pr)
398 pv_str = '%s-%s' % (preferred_v, preferred_r)
401 if preferred_file is None:
402 bb.note("preferred version %s of %s not available" % (pv_str, pn))
404 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
406 if preferred_file is None:
407 # get highest priority file set
408 files = pkg_pn[pn][0]
413 pv,pr = self.status.pkg_pvpr[f]
414 dp = self.status.pkg_dp[f]
416 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
420 preferred_file = latest_f
421 preferred_ver = latest
423 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
425 preferred_versions[pn] = (preferred_ver, preferred_file)
426 eligible.append(preferred_file)
429 if p in self.build_cache_fail:
430 bb.debug(1, "rejecting already-failed %s" % p)
433 if len(eligible) == 0:
434 bb.error("no eligible providers for %s" % item)
437 # look to see if one of them is already staged, or marked as preferred.
438 # if so, bump it to the head of the queue
440 the_data = make.pkgdata[p]
441 pn = bb.data.getVar('PN', the_data, 1)
442 pv = bb.data.getVar('PV', the_data, 1)
443 pr = bb.data.getVar('PR', the_data, 1)
444 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
445 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
446 if os.path.exists(stamp):
447 (newvers, fn) = preferred_versions[pn]
448 if not fn in eligible:
449 # package was made ineligible by already-failed check
451 oldver = "%s-%s" % (pv, pr)
452 newver = '-'.join(newvers)
453 if (newver != oldver):
454 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
457 if make.options.verbose:
458 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
460 eligible = [fn] + eligible
464 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
466 self.preferred[item] = prefervar
468 if item in self.preferred:
470 pn = self.status.pkg_fn[p]
471 if self.preferred[item] == pn:
472 if make.options.verbose:
473 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
475 eligible = [p] + eligible
479 if len(eligible) > 1 and discriminated == False:
480 if item not in self.consider_msgs_cache:
483 providers_list.append(self.status.pkg_fn[fn])
484 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
485 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
486 self.consider_msgs_cache.append(item)
489 # run through the list until we find one that we can build
491 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
492 if self.tryBuild(fn, item):
495 bb.note("no buildable providers for %s" % item)
498 def buildDepgraph( self ):
499 all_depends = self.status.all_depends
500 pn_provides = self.status.pn_provides
502 def calc_bbfile_priority(filename):
503 for (regex, pri) in self.status.bbfile_config_priorities:
504 if regex.match(filename):
508 # Handle PREFERRED_PROVIDERS
509 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
510 (providee, provider) = p.split(':')
511 if providee in self.preferred and self.preferred[providee] != provider:
512 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
513 self.preferred[providee] = provider
515 # Calculate priorities for each file
516 for p in make.pkgdata.keys():
517 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
519 # Build package list for "bitbake world"
520 bb.debug(1, "collating packages for \"world\"")
521 for f in self.status.possible_world:
523 pn = self.status.pkg_fn[f]
525 for p in pn_provides[pn]:
526 if p.startswith('virtual/'):
527 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
530 for pf in self.status.providers[p]:
531 if self.status.pkg_fn[pf] != pn:
532 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
536 self.status.world_target.add(pn)
538 # drop reference count now
539 self.status.possible_world = None
540 self.status.all_depends = None
542 def myProgressCallback( self, x, y, f, file_data, from_cache ):
543 # feed the status with new input
544 self.status.handle_bb_data(f, file_data, from_cache)
548 if os.isatty(sys.stdout.fileno()):
549 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
553 sys.stdout.write("Parsing .bb files, please wait...")
556 sys.stdout.write("done.")
559 def interactiveMode( self ):
560 """Drop off into a shell"""
563 except ImportError, details:
564 bb.fatal("Sorry, shell not available (%s)" % details )
569 def parseConfigurationFile( self, afile ):
571 make.cfg = bb.parse.handle( afile, make.cfg )
573 bb.fatal( "Unable to open %s" % afile )
574 except bb.parse.ParseError:
575 bb.fatal( "Unable to parse %s" % afile )
577 def handleCollections( self, collections ):
578 """Handle collections"""
580 collection_list = collections.split()
581 for c in collection_list:
582 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
584 bb.error("BBFILE_PATTERN_%s not defined" % c)
586 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
588 bb.error("BBFILE_PRIORITY_%s not defined" % c)
591 cre = re.compile(regex)
593 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
597 self.status.bbfile_config_priorities.append((cre, pri))
599 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
602 def cook( self, args ):
603 if not make.options.cmd:
604 make.options.cmd = "build"
606 if make.options.debug:
607 bb.debug_level = make.options.debug
609 make.cfg = bb.data.init()
611 for f in make.options.file:
612 self.parseConfigurationFile( f )
614 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
616 if not bb.data.getVar("BUILDNAME", make.cfg):
617 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
619 buildname = bb.data.getVar("BUILDNAME", make.cfg)
621 if make.options.interactive:
622 self.interactiveMode()
624 bf = make.options.buildfile
627 bbfile_data = bb.parse.handle(bf, make.cfg)
629 bb.fatal("Unable to open %s" % bf)
631 item = bb.data.getVar('PN', bbfile_data, 1)
632 self.tryBuildPackage( os.path.abspath( bf ), item, bbfile_data )
633 sys.exit( self.stats.show() )
635 # initialise the parsing status now we know we will need deps
636 self.status = BBParsingStatus()
638 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
639 self.status.ignored_dependencies = Set( ignore.split() )
641 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1) )
645 if not pkgs_to_build:
647 pkgs_to_build.extend(args)
648 if not pkgs_to_build:
649 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
651 pkgs_to_build = bbpkgs.split()
652 if not pkgs_to_build and not make.options.show_versions and not make.options.interactive:
653 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
654 print "for usage information."
657 # Import Psyco if available and not disabled
658 if not make.options.disable_psyco:
663 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
665 psyco.bind( make.collect_bbfiles )
667 bb.note("You have disabled Psyco. This decreases performance.")
670 bb.debug(1, "collecting .bb files")
671 make.collect_bbfiles( self.myProgressCallback )
672 bb.debug(1, "parsing complete")
675 if make.options.parse_only:
676 print "Requested parsing .bb files only. Exiting."
679 bb.data.update_data( make.cfg )
682 if make.options.show_versions:
685 if 'world' in pkgs_to_build:
686 pkgs_to_build.remove('world')
687 for t in self.status.world_target:
688 pkgs_to_build.append(t)
690 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
692 for k in pkgs_to_build:
695 if self.buildProvider( k ) == 0:
698 except bb.build.EventException:
699 bb.error("Build of " + k + " failed")
703 if make.options.abort:
706 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
708 sys.exit( self.stats.show() )
710 except KeyboardInterrupt:
711 print "\nNOTE: KeyboardInterrupt - Build not completed."
714 #============================================================================#
716 #============================================================================#
718 if __name__ == "__main__":
720 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
721 usage = """%prog [options] [package ...]
723 Executes the specified task (default is 'build') for a given set of BitBake files.
724 It expects that BBFILES is defined, which is a space seperated list of files to
725 be executed. BBFILES does support wildcards.
726 Default BBFILES are the .bb files in the current directory.""" )
728 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
729 action = "store", dest = "buildfile", default = None )
731 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
732 action = "store_false", dest = "abort", default = True )
734 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
735 action = "store_true", dest = "force", default = False )
737 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
738 action = "store_true", dest = "interactive", default = False )
740 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
741 action = "store", dest = "cmd", default = "build" )
743 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
744 action = "append", dest = "file", default = [] )
746 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
747 action = "store_true", dest = "verbose", default = False )
749 parser.add_option( "-D", "--debug", help = "Increase the debug level",
750 action = "count", dest="debug", default = 0)
752 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
753 action = "store_true", dest = "dry_run", default = False )
755 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
756 action = "store_true", dest = "parse_only", default = False )
758 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
759 action = "store_true", dest = "disable_psyco", default = False )
761 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
762 action = "store_true", dest = "show_versions", default = False )
764 options, args = parser.parse_args( sys.argv )
766 make.options = options
768 cooker.cook( args[1:] )