2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if pn not in self.pkg_pn:
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if pn not in self.pn_provides:
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if provide not in self.providers:
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
143 ParsingStatus = BBParsingStatus # make it visible from the shell
144 Statistics = BBStatistics # make it visible from the shell
146 def __init__( self ):
147 self.build_cache_fail = []
148 self.build_cache = []
149 self.building_list = []
151 self.consider_msgs_cache = []
153 self.stats = BBStatistics()
156 def tryBuildPackage( self, fn, item, the_data ):
157 """Build one package"""
158 bb.event.fire(bb.event.PkgStarted(item, the_data))
160 self.stats.attempt += 1
161 if make.options.force:
162 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
163 if not make.options.dry_run:
164 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
165 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
166 self.build_cache.append(fn)
168 except bb.build.FuncFailed:
170 bb.error("task stack execution failed")
171 bb.event.fire(bb.event.PkgFailed(item, the_data))
172 self.build_cache_fail.append(fn)
174 except bb.build.EventException:
176 (type, value, traceback) = sys.exc_info()
178 bb.error("%s event exception, aborting" % bb.event.getName(e))
179 bb.event.fire(bb.event.PkgFailed(item, the_data))
180 self.build_cache_fail.append(fn)
183 def tryBuild( self, fn, virtual ):
184 """Build a provider and its dependencies"""
185 if fn in self.building_list:
186 bb.error("%s depends on itself (eventually)" % fn)
187 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
190 the_data = make.pkgdata[fn]
191 item = self.status.pkg_fn[fn]
193 self.building_list.append(fn)
195 pathstr = "%s (%s)" % (item, virtual)
196 self.build_path.append(pathstr)
198 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
199 if make.options.verbose:
200 bb.note("current path: %s" % (" -> ".join(self.build_path)))
201 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
206 depcmd = make.options.cmd
207 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
208 if bbdepcmd is not None:
215 oldcmd = make.options.cmd
216 make.options.cmd = depcmd
218 for dependency in depends_list:
219 if dependency in self.status.ignored_dependencies:
223 if self.buildProvider( dependency ) == 0:
224 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
226 if make.options.abort:
230 make.options.cmd = oldcmd
236 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
237 self.build_cache.append(fn)
240 return self.tryBuildPackage( fn, item, the_data )
243 self.building_list.remove(fn)
244 self.build_path.remove(pathstr)
246 def showVersions( self ):
247 pkg_pn = self.status.pkg_pn
248 preferred_versions = {}
252 for pn in pkg_pn.keys():
256 priority = self.status.bbfile_priority[f]
257 if priority not in priorities:
258 priorities[priority] = []
259 priorities[priority].append(f)
260 p_list = priorities.keys()
261 p_list.sort(lambda a, b: a - b)
264 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
266 for pn in pkg_pn.keys():
267 preferred_file = None
268 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
271 m = re.match('(.*)_(.*)', preferred_v)
273 preferred_v = m.group(1)
274 preferred_r = m.group(2)
276 for file_set in pkg_pn[pn]:
278 pv,pr = self.status.pkg_pvpr[f]
279 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
281 preferred_ver = (pv, pr)
286 pv_str = '%s-%s' % (preferred_v, preferred_r)
289 if preferred_file is None:
290 bb.note("preferred version %s of %s not available" % (pv_str, pn))
292 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
294 # get highest priority file set
295 files = pkg_pn[pn][0]
300 pv,pr = self.status.pkg_pvpr[f]
301 dp = self.status.pkg_dp[f]
303 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
307 if preferred_file is None:
308 preferred_file = latest_f
309 preferred_ver = latest
311 preferred_versions[pn] = (preferred_ver, preferred_file)
312 latest_versions[pn] = (latest, latest_f)
314 pkg_list = pkg_pn.keys()
318 pref = preferred_versions[p]
319 latest = latest_versions[p]
322 prefstr = pref[0][0] + "-" + pref[0][1]
326 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
329 def buildProvider( self, item ):
332 discriminated = False
334 if item not in self.status.providers:
335 bb.error("Nothing provides %s" % item)
338 all_p = self.status.providers[item]
341 if p in self.build_cache:
342 bb.debug(1, "already built %s in this run\n" % p)
346 preferred_versions = {}
348 # Collate providers by PN
351 pn = self.status.pkg_fn[p]
356 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
359 for pn in pkg_pn.keys():
363 priority = self.status.bbfile_priority[f]
364 if priority not in priorities:
365 priorities[priority] = []
366 priorities[priority].append(f)
367 p_list = priorities.keys()
368 p_list.sort(lambda a, b: a - b)
371 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
373 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
374 # version. If not, find the latest version provided by an bbfile in the
375 # highest-priority set.
376 for pn in pkg_pn.keys():
377 preferred_file = None
379 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
382 m = re.match('(.*)_(.*)', preferred_v)
384 preferred_v = m.group(1)
385 preferred_r = m.group(2)
387 for file_set in pkg_pn[pn]:
389 pv,pr = self.status.pkg_pvpr[f]
390 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
392 preferred_ver = (pv, pr)
397 pv_str = '%s-%s' % (preferred_v, preferred_r)
400 if preferred_file is None:
401 bb.note("preferred version %s of %s not available" % (pv_str, pn))
403 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
405 if preferred_file is None:
406 # get highest priority file set
407 files = pkg_pn[pn][0]
412 pv,pr = self.status.pkg_pvpr[f]
413 dp = self.status.pkg_dp[f]
415 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
419 preferred_file = latest_f
420 preferred_ver = latest
422 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
424 preferred_versions[pn] = (preferred_ver, preferred_file)
425 eligible.append(preferred_file)
428 if p in self.build_cache_fail:
429 bb.debug(1, "rejecting already-failed %s" % p)
432 if len(eligible) == 0:
433 bb.error("no eligible providers for %s" % item)
436 # look to see if one of them is already staged, or marked as preferred.
437 # if so, bump it to the head of the queue
439 the_data = make.pkgdata[p]
440 pn = bb.data.getVar('PN', the_data, 1)
441 pv = bb.data.getVar('PV', the_data, 1)
442 pr = bb.data.getVar('PR', the_data, 1)
443 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
444 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
445 if os.path.exists(stamp):
446 (newvers, fn) = preferred_versions[pn]
447 if not fn in eligible:
448 # package was made ineligible by already-failed check
450 oldver = "%s-%s" % (pv, pr)
451 newver = '-'.join(newvers)
452 if (newver != oldver):
453 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
456 if make.options.verbose:
457 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
459 eligible = [fn] + eligible
463 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
465 self.preferred[item] = prefervar
467 if item in self.preferred:
469 pn = self.status.pkg_fn[p]
470 if self.preferred[item] == pn:
471 if make.options.verbose:
472 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
474 eligible = [p] + eligible
478 if len(eligible) > 1 and discriminated == False:
479 if item not in self.consider_msgs_cache:
482 providers_list.append(self.status.pkg_fn[fn])
483 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
484 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
485 self.consider_msgs_cache.append(item)
488 # run through the list until we find one that we can build
490 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
491 if self.tryBuild(fn, item):
494 bb.note("no buildable providers for %s" % item)
497 def buildDepgraph( self ):
498 all_depends = self.status.all_depends
499 pn_provides = self.status.pn_provides
501 def calc_bbfile_priority(filename):
502 for (regex, pri) in self.status.bbfile_config_priorities:
503 if regex.match(filename):
507 # Handle PREFERRED_PROVIDERS
508 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
509 (providee, provider) = p.split(':')
510 if providee in self.preferred and self.preferred[providee] != provider:
511 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
512 self.preferred[providee] = provider
514 # Calculate priorities for each file
515 for p in make.pkgdata.keys():
516 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
518 # Build package list for "bitbake world"
519 bb.debug(1, "collating packages for \"world\"")
520 for f in self.status.possible_world:
522 pn = self.status.pkg_fn[f]
524 for p in pn_provides[pn]:
525 if p.startswith('virtual/'):
526 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
529 for pf in self.status.providers[p]:
530 if self.status.pkg_fn[pf] != pn:
531 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
535 self.status.world_target.add(pn)
537 # drop reference count now
538 self.status.possible_world = None
539 self.status.all_depends = None
541 def myProgressCallback( self, x, y, f, file_data, from_cache ):
542 # feed the status with new input
543 self.status.handle_bb_data(f, file_data, from_cache)
547 if os.isatty(sys.stdout.fileno()):
548 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
552 sys.stdout.write("Parsing .bb files, please wait...")
555 sys.stdout.write("done.")
558 def interactiveMode( self ):
559 """Drop off into a shell"""
562 except ImportError, details:
563 bb.fatal("Sorry, shell not available (%s)" % details )
568 def parseConfigurationFile( self, afile ):
570 make.cfg = bb.parse.handle( afile, make.cfg )
572 bb.fatal( "Unable to open %s" % afile )
573 except bb.parse.ParseError:
574 bb.fatal( "Unable to parse %s" % afile )
576 def handleCollections( self, collections ):
577 """Handle collections"""
579 collection_list = collections.split()
580 for c in collection_list:
581 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
583 bb.error("BBFILE_PATTERN_%s not defined" % c)
585 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
587 bb.error("BBFILE_PRIORITY_%s not defined" % c)
590 cre = re.compile(regex)
592 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
596 self.status.bbfile_config_priorities.append((cre, pri))
598 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
601 def cook( self, args ):
602 if not make.options.cmd:
603 make.options.cmd = "build"
605 if make.options.debug:
606 bb.debug_level = make.options.debug
608 make.cfg = bb.data.init()
610 for f in make.options.file:
611 self.parseConfigurationFile( f )
613 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
615 if not bb.data.getVar("BUILDNAME", make.cfg):
616 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
618 buildname = bb.data.getVar("BUILDNAME", make.cfg)
620 if make.options.interactive:
621 self.interactiveMode()
623 bf = make.options.buildfile
626 bbfile_data = bb.parse.handle(bf, make.cfg)
628 bb.fatal("Unable to open %s" % bf)
630 item = bb.data.getVar('PN', bbfile_data, 1)
631 self.tryBuildPackage( os.path.abspath( bf ), item, bbfile_data )
632 sys.exit( self.stats.show() )
634 # initialise the parsing status now we know we will need deps
635 self.status = BBParsingStatus()
637 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
638 self.status.ignored_dependencies = Set( ignore.split() )
640 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1) )
644 if not pkgs_to_build:
646 pkgs_to_build.extend(args)
647 if not pkgs_to_build:
648 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
650 pkgs_to_build = bbpkgs.split()
651 if not pkgs_to_build and not make.options.show_versions and not make.options.interactive:
652 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
653 print "for usage information."
656 # Import Psyco if available and not disabled
657 if not make.options.disable_psyco:
662 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
664 psyco.bind( make.collect_bbfiles )
666 bb.note("You have disabled Psyco. This decreases performance.")
669 bb.debug(1, "collecting .bb files")
670 make.collect_bbfiles( self.myProgressCallback )
671 bb.debug(1, "parsing complete")
674 if make.options.parse_only:
675 print "Requested parsing .bb files only. Exiting."
678 bb.data.update_data( make.cfg )
681 if make.options.show_versions:
684 if 'world' in pkgs_to_build:
685 pkgs_to_build.remove('world')
686 for t in self.status.world_target:
687 pkgs_to_build.append(t)
689 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
691 for k in pkgs_to_build:
694 if self.buildProvider( k ) == 0:
697 except bb.build.EventException:
698 bb.error("Build of " + k + " failed")
702 if make.options.abort:
705 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
707 sys.exit( self.stats.show() )
709 except KeyboardInterrupt:
710 print "\nNOTE: KeyboardInterrupt - Build not completed."
713 #============================================================================#
715 #============================================================================#
717 if __name__ == "__main__":
719 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
720 usage = """%prog [options] [package ...]
722 Executes the specified task (default is 'build') for a given set of BitBake files.
723 It expects that BBFILES is defined, which is a space seperated list of files to
724 be executed. BBFILES does support wildcards.
725 Default BBFILES are the .bb files in the current directory.""" )
727 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
728 action = "store", dest = "buildfile", default = None )
730 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
731 action = "store_false", dest = "abort", default = True )
733 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
734 action = "store_true", dest = "force", default = False )
736 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
737 action = "store_true", dest = "interactive", default = False )
739 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
740 action = "store", dest = "cmd", default = "build" )
742 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
743 action = "append", dest = "file", default = [] )
745 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
746 action = "store_true", dest = "verbose", default = False )
748 parser.add_option( "-D", "--debug", help = "Increase the debug level",
749 action = "count", dest="debug", default = 0)
751 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
752 action = "store_true", dest = "dry_run", default = False )
754 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
755 action = "store_true", dest = "parse_only", default = False )
757 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
758 action = "store_true", dest = "disable_psyco", default = False )
760 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
761 action = "store_true", dest = "show_versions", default = False )
763 options, args = parser.parse_args( sys.argv )
765 make.options = options
767 cooker.cook( args[1:] )