2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
42 bbfile_config_priorities = []
46 def handle_options( args ):
47 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
48 usage = """%prog [options] [package ...]
50 Executes the specified task (default is 'build') for a given set of BitBake files.
51 It expects that BBFILES is defined, which is a space seperated list of files to
52 be executed. BBFILES does support wildcards.
53 Default BBFILES are the .bb files in the current directory.""" )
55 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
56 action = "store", dest = "buildfile", default = None )
58 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
59 action = "store_false", dest = "abort", default = True )
61 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
62 action = "store_true", dest = "force", default = False )
65 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
66 action = "store", dest = "cmd", default = "build" )
68 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
69 action = "append", dest = "file", default = [] )
71 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
72 action = "store_true", dest = "verbose", default = False )
73 parser.add_option( "-D", "--debug", help = "Increase the debug level",
74 action = "count", dest="debug", default = 0)
76 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
77 action = "store_true", dest = "dry_run", default = False )
79 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
80 action = "store_true", dest = "parse_only", default = False )
82 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
83 action = "store_true", dest = "disable_psyco", default = False )
85 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
86 action = "store_true", dest = "show_versions", default = False )
88 options, args = parser.parse_args( args )
89 return options, args[1:]
91 def try_build(fn, virtual):
92 if fn in __building_list:
93 bb.error("%s depends on itself (eventually)" % fn)
94 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
97 the_data = make.pkgdata[fn]
98 item = bb.data.getVar('PN', the_data, 1)
100 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
103 __building_list.append(fn)
105 pathstr = "%s (%s)" % (item, virtual)
106 __build_path.append(pathstr)
109 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
110 if make.options.verbose:
111 bb.note("current path: %s" % (" -> ".join(__build_path)))
112 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
117 depcmd = make.options.cmd
118 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', make.pkgdata[fn])
119 if bbdepcmd is not None:
126 oldcmd = make.options.cmd
127 make.options.cmd = depcmd
129 for d in depends_list:
130 if d in __ignored_dependencies:
134 if buildPackage(d) == 0:
135 bb.error("dependency %s (for %s) not satisfied" % (d,item))
137 if make.options.abort:
141 make.options.cmd = oldcmd
147 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
149 __stats["attempt"] += 1
150 if not make.options.dry_run:
151 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
152 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
153 __build_cache.append(fn)
155 except bb.build.FuncFailed:
157 bb.error("task stack execution failed")
158 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
159 __build_cache_fail.append(fn)
161 except bb.build.EventException:
163 (type, value, traceback) = sys.exc_info()
165 bb.error("%s event exception, aborting" % bb.event.getName(e))
166 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
167 __build_cache_fail.append(fn)
170 __building_list.remove(fn)
171 __build_path.remove(pathstr)
175 preferred_versions = {}
178 for p in make.pkgdata.keys():
179 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
180 if not pkg_pn.has_key(pn):
185 for pn in pkg_pn.keys():
189 priority = bbfile_priority[f]
190 if not priorities.has_key(priority):
191 priorities[priority] = []
192 priorities[priority].append(f)
193 p_list = priorities.keys()
194 p_list.sort(lambda a, b: a - b)
197 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
199 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
200 # version. If not, find the latest version provided by an bbfile in the
201 # highest-priority set.
202 for pn in pkg_pn.keys():
203 preferred_file = None
205 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
208 m = re.match('(.*)_(.*)', preferred_v)
210 preferred_v = m.group(1)
211 preferred_r = m.group(2)
213 for file_set in pkg_pn[pn]:
215 the_data = make.pkgdata[f]
216 pv = bb.data.getVar('PV', the_data, 1)
217 pr = bb.data.getVar('PR', the_data, 1)
218 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
220 preferred_ver = (pv, pr)
225 pv_str = '%s-%s' % (preferred_v, preferred_r)
228 if preferred_file is None:
229 bb.note("preferred version %s of %s not available" % (pv_str, pn))
231 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
233 # get highest priority file set
234 files = pkg_pn[pn][0]
239 the_data = make.pkgdata[f]
240 pv = bb.data.getVar('PV', the_data, 1)
241 pr = bb.data.getVar('PR', the_data, 1)
242 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
244 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
248 if preferred_file is None:
249 preferred_file = latest_f
250 preferred_ver = latest
252 preferred_versions[pn] = (preferred_ver, preferred_file)
253 latest_versions[pn] = (latest, latest_f)
255 pkg_list = pkg_pn.keys()
259 pref = preferred_versions[p]
260 latest = latest_versions[p]
263 prefstr = pref[0][0] + "-" + pref[0][1]
267 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
270 __consider_msgs_cache = []
271 def buildPackage(item):
274 discriminated = False
276 if not providers.has_key(item):
277 bb.error("Nothing provides %s" % item)
280 all_p = providers[item]
283 if p in __build_cache:
284 bb.debug(1, "already built %s in this run\n" % p)
288 preferred_versions = {}
290 # Collate providers by PN
293 the_data = make.pkgdata[p]
294 pn = bb.data.getVar('PN', the_data, 1)
295 if not pkg_pn.has_key(pn):
299 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
302 for pn in pkg_pn.keys():
306 priority = bbfile_priority[f]
307 if not priorities.has_key(priority):
308 priorities[priority] = []
309 priorities[priority].append(f)
310 p_list = priorities.keys()
311 p_list.sort(lambda a, b: a - b)
314 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
316 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
317 # version. If not, find the latest version provided by an bbfile in the
318 # highest-priority set.
319 for pn in pkg_pn.keys():
320 preferred_file = None
322 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
325 m = re.match('(.*)_(.*)', preferred_v)
327 preferred_v = m.group(1)
328 preferred_r = m.group(2)
330 for file_set in pkg_pn[pn]:
332 the_data = make.pkgdata[f]
333 pv = bb.data.getVar('PV', the_data, 1)
334 pr = bb.data.getVar('PR', the_data, 1)
335 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
337 preferred_ver = (pv, pr)
342 pv_str = '%s-%s' % (preferred_v, preferred_r)
345 if preferred_file is None:
346 bb.note("preferred version %s of %s not available" % (pv_str, pn))
348 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
350 if preferred_file is None:
351 # get highest priority file set
352 files = pkg_pn[pn][0]
357 the_data = make.pkgdata[f]
358 pv = bb.data.getVar('PV', the_data, 1)
359 pr = bb.data.getVar('PR', the_data, 1)
360 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
362 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
366 preferred_file = latest_f
367 preferred_ver = latest
369 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
371 preferred_versions[pn] = (preferred_ver, preferred_file)
372 eligible.append(preferred_file)
375 if p in __build_cache_fail:
376 bb.debug(1, "rejecting already-failed %s" % p)
379 if len(eligible) == 0:
380 bb.error("no eligible providers for %s" % item)
383 # look to see if one of them is already staged, or marked as preferred.
384 # if so, bump it to the head of the queue
386 the_data = make.pkgdata[p]
387 pn = bb.data.getVar('PN', the_data, 1)
388 pv = bb.data.getVar('PV', the_data, 1)
389 pr = bb.data.getVar('PR', the_data, 1)
390 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
391 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
392 if os.path.exists(stamp):
393 (newvers, fn) = preferred_versions[pn]
394 if not fn in eligible:
395 # package was made ineligible by already-failed check
397 oldver = "%s-%s" % (pv, pr)
398 newver = '-'.join(newvers)
399 if (newver != oldver):
400 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
403 if make.options.verbose:
404 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
406 eligible = [fn] + eligible
410 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
412 __preferred[item] = prefervar
414 if __preferred.has_key(item):
416 the_data = make.pkgdata[p]
417 pn = bb.data.getVar('PN', the_data, 1)
418 if __preferred[item] == pn:
419 if make.options.verbose:
420 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
422 eligible = [p] + eligible
426 if len(eligible) > 1 and discriminated == False:
427 if item not in __consider_msgs_cache:
430 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
431 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
432 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
433 __consider_msgs_cache.append(item)
436 # run through the list until we find one that we can build
438 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
439 if try_build(fn, item):
442 bb.note("no buildable providers for %s" % item)
445 def build_depgraph():
450 if bbdebug or progress.p == p: return
452 if os.isatty(sys.stdout.fileno()):
453 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
457 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
459 sys.stdout.write("done.\n")
462 def calc_bbfile_priority(filename):
463 for (regex, pri) in bbfile_config_priorities:
464 if regex.match(filename):
468 # Handle PREFERRED_PROVIDERS
469 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
470 (providee, provider) = p.split(':')
471 if __preferred.has_key(providee) and __preferred[providee] != provider:
472 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
473 __preferred[providee] = provider
475 # Calculate priorities for each file
476 for p in make.pkgdata.keys():
477 bbfile_priority[p] = calc_bbfile_priority(p)
479 n = len(make.pkgdata.keys())
484 bb.debug(1, "building providers hashes")
486 # Build forward and reverse provider hashes
487 # Forward: virtual -> [filenames]
488 # Reverse: PN -> [virtuals]
489 for f in make.pkgdata.keys():
492 pn = bb.data.getVar('PN', d, 1)
493 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
495 if not pn_provides.has_key(pn):
496 pn_provides[pn] = Set()
497 pn_provides[pn] |= provides
499 for provide in provides:
500 if not providers.has_key(provide):
501 providers[provide] = []
502 providers[provide].append(f)
504 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
515 sys.stdout.write("\n")
517 # Build package list for "bitbake world"
518 bb.debug(1, "collating packages for \"world\"")
519 for f in make.pkgdata.keys():
521 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
522 bb.debug(2, "skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
525 pn = bb.data.getVar('PN', d, 1)
526 for p in pn_provides[pn]:
527 if p.startswith('virtual/'):
528 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
531 for pf in providers[p]:
532 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
533 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
537 __world_target.add(pn)
539 def myProgressCallback( x, y, f ):
542 if os.isatty(sys.stdout.fileno()):
543 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
547 sys.stdout.write("Parsing .bb files, please wait...")
550 sys.stdout.write("done.")
553 def executeOneBB( fn ):
555 d = bb.parse.handle(fn, make.cfg)
557 bb.fatal("Unable to open %s" % fn)
559 if make.options.parse_only:
560 print "Requested parsing .bb files only. Exiting."
563 name = bb.data.getVar('PN', d, 1)
564 bb.event.fire(bb.event.PkgStarted(name, d))
566 __stats["attempt"] += 1
567 if make.options.force:
568 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
569 if not make.options.dry_run:
570 bb.build.exec_task('do_%s' % make.options.cmd, d)
571 bb.event.fire(bb.event.PkgSucceeded(name, d))
572 __build_cache.append(fn)
573 except bb.build.FuncFailed:
575 bb.error("task stack execution failed")
576 bb.event.fire(bb.event.PkgFailed(name, d))
577 __build_cache_fail.append(fn)
578 except bb.build.EventException:
580 (type, value, traceback) = sys.exc_info()
582 bb.error("%s event exception, aborting" % bb.event.getName(e))
583 bb.event.fire(bb.event.PkgFailed(name, d))
584 __build_cache_fail.append(fn)
590 __stats["attempt"] = 0
591 __stats["success"] = 0
596 print "Build statistics:"
597 print " Attempted builds: %d" % __stats["attempt"]
598 if __stats["fail"] != 0:
599 print " Failed builds: %d" % __stats["fail"]
600 if __stats["deps"] != 0:
601 print " Dependencies not satisfied: %d" % __stats["deps"]
602 if __stats["fail"] != 0 or __stats["deps"] != 0:
606 if __name__ == "__main__":
608 make.options, args = handle_options( sys.argv )
610 if not make.options.cmd:
611 make.options.cmd = "build"
613 if make.options.debug:
614 bb.debug_level = make.options.debug
617 make.cfg = bb.data.init()
620 for f in make.options.file:
622 make.cfg = bb.parse.handle(f, make.cfg)
624 bb.fatal("Unable to open %s" % f)
627 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
629 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
631 if not bb.data.getVar("BUILDNAME", make.cfg):
632 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
634 buildname = bb.data.getVar("BUILDNAME", make.cfg)
636 bf = make.options.buildfile
638 executeOneBB( os.path.abspath(bf) )
641 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
642 __ignored_dependencies = ignore.split()
644 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
646 collection_list = collections.split()
647 for c in collection_list:
648 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
650 bb.error("BBFILE_PATTERN_%s not defined" % c)
652 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
654 bb.error("BBFILE_PRIORITY_%s not defined" % c)
657 cre = re.compile(regex)
659 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
663 bbfile_config_priorities.append((cre, pri))
665 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
669 if not pkgs_to_build:
671 pkgs_to_build.extend(args)
672 if not pkgs_to_build:
673 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
675 pkgs_to_build = bbpkgs.split()
676 if not pkgs_to_build and not make.options.show_versions:
677 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
678 print "for usage information."
682 # Import Psyco if available and not disabled
683 if not make.options.disable_psyco:
688 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
690 psyco.bind( make.collect_bbfiles )
692 bb.note("You have disabled Psyco. This decreases performance.")
695 bb.debug(1, "collecting .bb files")
696 make.collect_bbfiles( myProgressCallback )
697 bb.debug(1, "parsing complete")
700 if make.options.parse_only:
701 print "Requested parsing .bb files only. Exiting."
706 if make.options.show_versions:
710 if 'world' in pkgs_to_build:
711 pkgs_to_build.remove('world')
712 for t in __world_target:
713 pkgs_to_build.append(t)
715 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
717 for k in pkgs_to_build:
720 if buildPackage(k) == 0:
723 except bb.build.EventException:
724 bb.error("Build of " + k + " failed")
728 if make.options.abort:
731 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
735 except KeyboardInterrupt:
736 print "\nNOTE: KeyboardInterrupt - Build not completed."