2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2005 Holger Hans Peter Freyther
8 # Copyright (C) 2005 ROAD GmbH
10 # This program is free software; you can redistribute it and/or modify it under
11 # the terms of the GNU General Public License as published by the Free Software
12 # Foundation; either version 2 of the License, or (at your option) any later
15 # This program is distributed in the hope that it will be useful, but WITHOUT
16 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
19 # You should have received a copy of the GNU General Public License along with
20 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
21 # Place, Suite 330, Boston, MA 02111-1307 USA.
23 import sys, os, getopt, glob, copy, os.path, re
24 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
28 import itertools, optparse
30 parsespin = itertools.cycle( r'|/-\\' )
33 __build_cache_fail = []
39 __world_target = Set()
40 __ignored_dependencies = Set()
44 bbfile_config_priorities = []
49 def handle_options( args ):
50 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
51 usage = """%prog [options] [package ...]
53 Executes the specified task (default is 'build') for a given set of BitBake files.
54 It expects that BBFILES is defined, which is a space seperated list of files to
55 be executed. BBFILES does support wildcards.
56 Default BBFILES are the .bb files in the current directory.""" )
58 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
59 action = "store", dest = "buildfile", default = None )
61 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
62 action = "store_false", dest = "abort", default = True )
64 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
65 action = "store_true", dest = "force", default = False )
68 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
69 action = "store", dest = "cmd", default = "build" )
71 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
72 action = "append", dest = "file", default = [] )
74 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
75 action = "store_true", dest = "verbose", default = False )
76 parser.add_option( "-D", "--debug", help = "Increase the debug level",
77 action = "count", dest="debug", default = 0)
79 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
80 action = "store_true", dest = "dry_run", default = False )
82 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
83 action = "store_true", dest = "parse_only", default = False )
85 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
86 action = "store_true", dest = "disable_psyco", default = False )
88 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
89 action = "store_true", dest = "show_versions", default = False )
91 options, args = parser.parse_args( args )
92 return options, args[1:]
94 def try_build(fn, virtual):
95 if fn in __building_list:
96 bb.error("%s depends on itself (eventually)" % fn)
97 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
100 the_data = make.pkgdata[fn]
101 item = bb.data.getVar('PN', the_data, 1)
103 __building_list.append(fn)
105 pathstr = "%s (%s)" % (item, virtual)
106 __build_path.append(pathstr)
108 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
109 if make.options.verbose:
110 bb.note("current path: %s" % (" -> ".join(__build_path)))
111 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
116 depcmd = make.options.cmd
117 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
118 if bbdepcmd is not None:
125 oldcmd = make.options.cmd
126 make.options.cmd = depcmd
128 for d in depends_list:
129 if d in __ignored_dependencies:
133 if buildPackage(d) == 0:
134 bb.error("dependency %s (for %s) not satisfied" % (d,item))
136 if make.options.abort:
140 make.options.cmd = oldcmd
146 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
147 __build_cache.append(fn)
150 bb.event.fire(bb.event.PkgStarted(item, the_data))
152 __stats["attempt"] += 1
153 if not make.options.dry_run:
154 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
155 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
156 __build_cache.append(fn)
158 except bb.build.FuncFailed:
160 bb.error("task stack execution failed")
161 bb.event.fire(bb.event.PkgFailed(item, the_data))
162 __build_cache_fail.append(fn)
164 except bb.build.EventException:
166 (type, value, traceback) = sys.exc_info()
168 bb.error("%s event exception, aborting" % bb.event.getName(e))
169 bb.event.fire(bb.event.PkgFailed(item, the_data))
170 __build_cache_fail.append(fn)
173 __building_list.remove(fn)
174 __build_path.remove(pathstr)
178 preferred_versions = {}
181 for p in make.pkgdata.keys():
182 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
183 if not pkg_pn.has_key(pn):
188 for pn in pkg_pn.keys():
192 priority = bbfile_priority[f]
193 if not priorities.has_key(priority):
194 priorities[priority] = []
195 priorities[priority].append(f)
196 p_list = priorities.keys()
197 p_list.sort(lambda a, b: a - b)
200 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
202 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
203 # version. If not, find the latest version provided by an bbfile in the
204 # highest-priority set.
205 for pn in pkg_pn.keys():
206 preferred_file = None
208 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
211 m = re.match('(.*)_(.*)', preferred_v)
213 preferred_v = m.group(1)
214 preferred_r = m.group(2)
216 for file_set in pkg_pn[pn]:
218 the_data = make.pkgdata[f]
219 pv = bb.data.getVar('PV', the_data, 1)
220 pr = bb.data.getVar('PR', the_data, 1)
221 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
223 preferred_ver = (pv, pr)
228 pv_str = '%s-%s' % (preferred_v, preferred_r)
231 if preferred_file is None:
232 bb.note("preferred version %s of %s not available" % (pv_str, pn))
234 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
236 # get highest priority file set
237 files = pkg_pn[pn][0]
242 the_data = make.pkgdata[f]
243 pv = bb.data.getVar('PV', the_data, 1)
244 pr = bb.data.getVar('PR', the_data, 1)
245 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
247 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
251 if preferred_file is None:
252 preferred_file = latest_f
253 preferred_ver = latest
255 preferred_versions[pn] = (preferred_ver, preferred_file)
256 latest_versions[pn] = (latest, latest_f)
258 pkg_list = pkg_pn.keys()
262 pref = preferred_versions[p]
263 latest = latest_versions[p]
266 prefstr = pref[0][0] + "-" + pref[0][1]
270 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
273 __consider_msgs_cache = []
274 def buildPackage(item):
277 discriminated = False
279 if not providers.has_key(item):
280 bb.error("Nothing provides %s" % item)
283 all_p = providers[item]
286 if p in __build_cache:
287 bb.debug(1, "already built %s in this run\n" % p)
291 preferred_versions = {}
293 # Collate providers by PN
296 the_data = make.pkgdata[p]
297 pn = bb.data.getVar('PN', the_data, 1)
298 if not pkg_pn.has_key(pn):
302 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
305 for pn in pkg_pn.keys():
309 priority = bbfile_priority[f]
310 if not priorities.has_key(priority):
311 priorities[priority] = []
312 priorities[priority].append(f)
313 p_list = priorities.keys()
314 p_list.sort(lambda a, b: a - b)
317 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
319 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
320 # version. If not, find the latest version provided by an bbfile in the
321 # highest-priority set.
322 for pn in pkg_pn.keys():
323 preferred_file = None
325 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
328 m = re.match('(.*)_(.*)', preferred_v)
330 preferred_v = m.group(1)
331 preferred_r = m.group(2)
333 for file_set in pkg_pn[pn]:
335 the_data = make.pkgdata[f]
336 pv = bb.data.getVar('PV', the_data, 1)
337 pr = bb.data.getVar('PR', the_data, 1)
338 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
340 preferred_ver = (pv, pr)
345 pv_str = '%s-%s' % (preferred_v, preferred_r)
348 if preferred_file is None:
349 bb.note("preferred version %s of %s not available" % (pv_str, pn))
351 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
353 if preferred_file is None:
354 # get highest priority file set
355 files = pkg_pn[pn][0]
360 the_data = make.pkgdata[f]
361 pv = bb.data.getVar('PV', the_data, 1)
362 pr = bb.data.getVar('PR', the_data, 1)
363 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
365 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
369 preferred_file = latest_f
370 preferred_ver = latest
372 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
374 preferred_versions[pn] = (preferred_ver, preferred_file)
375 eligible.append(preferred_file)
378 if p in __build_cache_fail:
379 bb.debug(1, "rejecting already-failed %s" % p)
382 if len(eligible) == 0:
383 bb.error("no eligible providers for %s" % item)
386 # look to see if one of them is already staged, or marked as preferred.
387 # if so, bump it to the head of the queue
389 the_data = make.pkgdata[p]
390 pn = bb.data.getVar('PN', the_data, 1)
391 pv = bb.data.getVar('PV', the_data, 1)
392 pr = bb.data.getVar('PR', the_data, 1)
393 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
394 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
395 if os.path.exists(stamp):
396 (newvers, fn) = preferred_versions[pn]
397 if not fn in eligible:
398 # package was made ineligible by already-failed check
400 oldver = "%s-%s" % (pv, pr)
401 newver = '-'.join(newvers)
402 if (newver != oldver):
403 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
406 if make.options.verbose:
407 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
409 eligible = [fn] + eligible
413 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
415 __preferred[item] = prefervar
417 if __preferred.has_key(item):
419 the_data = make.pkgdata[p]
420 pn = bb.data.getVar('PN', the_data, 1)
421 if __preferred[item] == pn:
422 if make.options.verbose:
423 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
425 eligible = [p] + eligible
429 if len(eligible) > 1 and discriminated == False:
430 if item not in __consider_msgs_cache:
433 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
434 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
435 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
436 __consider_msgs_cache.append(item)
439 # run through the list until we find one that we can build
441 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
442 if try_build(fn, item):
445 bb.note("no buildable providers for %s" % item)
448 def build_depgraph():
453 if bbdebug or progress.p == p: return
455 if os.isatty(sys.stdout.fileno()):
456 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
460 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
462 sys.stdout.write("done.\n")
465 def calc_bbfile_priority(filename):
466 for (regex, pri) in bbfile_config_priorities:
467 if regex.match(filename):
471 # Handle PREFERRED_PROVIDERS
472 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
473 (providee, provider) = p.split(':')
474 if __preferred.has_key(providee) and __preferred[providee] != provider:
475 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
476 __preferred[providee] = provider
478 # Calculate priorities for each file
479 for p in make.pkgdata.keys():
480 bbfile_priority[p] = calc_bbfile_priority(p)
482 n = len(make.pkgdata.keys())
487 bb.debug(1, "building providers hashes")
489 # Build forward and reverse provider hashes
490 # Forward: virtual -> [filenames]
491 # Reverse: PN -> [virtuals]
492 for f in make.pkgdata.keys():
495 pn = bb.data.getVar('PN', d, 1)
496 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
498 if not pn_provides.has_key(pn):
499 pn_provides[pn] = Set()
500 pn_provides[pn] |= provides
502 for provide in provides:
503 if not providers.has_key(provide):
504 providers[provide] = []
505 providers[provide].append(f)
507 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
518 sys.stdout.write("\n")
520 # Build package list for "bitbake world"
521 bb.debug(1, "collating packages for \"world\"")
522 for f in make.pkgdata.keys():
524 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
525 bb.debug(2, "skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
528 pn = bb.data.getVar('PN', d, 1)
529 for p in pn_provides[pn]:
530 if p.startswith('virtual/'):
531 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
534 for pf in providers[p]:
535 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
536 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
540 __world_target.add(pn)
542 def myProgressCallback( x, y, f ):
545 if os.isatty(sys.stdout.fileno()):
546 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
550 sys.stdout.write("Parsing .bb files, please wait...")
553 sys.stdout.write("done.")
556 def executeOneBB( fn ):
558 d = bb.parse.handle(fn, make.cfg)
560 bb.fatal("Unable to open %s" % fn)
562 if make.options.parse_only:
563 print "Requested parsing .bb files only. Exiting."
566 name = bb.data.getVar('PN', d, 1)
567 bb.event.fire(bb.event.PkgStarted(name, d))
569 __stats["attempt"] += 1
570 if make.options.force:
571 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
572 if not make.options.dry_run:
573 bb.build.exec_task('do_%s' % make.options.cmd, d)
574 bb.event.fire(bb.event.PkgSucceeded(name, d))
575 __build_cache.append(fn)
576 except bb.build.FuncFailed:
578 bb.error("task stack execution failed")
579 bb.event.fire(bb.event.PkgFailed(name, d))
580 __build_cache_fail.append(fn)
581 except bb.build.EventException:
583 (type, value, traceback) = sys.exc_info()
585 bb.error("%s event exception, aborting" % bb.event.getName(e))
586 bb.event.fire(bb.event.PkgFailed(name, d))
587 __build_cache_fail.append(fn)
593 __stats["attempt"] = 0
594 __stats["success"] = 0
599 print "Build statistics:"
600 print " Attempted builds: %d" % __stats["attempt"]
601 if __stats["fail"] != 0:
602 print " Failed builds: %d" % __stats["fail"]
603 if __stats["deps"] != 0:
604 print " Dependencies not satisfied: %d" % __stats["deps"]
605 if __stats["fail"] != 0 or __stats["deps"] != 0:
609 if __name__ == "__main__":
611 make.options, args = handle_options( sys.argv )
613 if not make.options.cmd:
614 make.options.cmd = "build"
616 if make.options.debug:
617 bb.debug_level = make.options.debug
619 make.cfg = bb.data.init()
621 for f in make.options.file:
623 make.cfg = bb.parse.handle(f, make.cfg)
625 bb.fatal("Unable to open %s" % f)
628 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
630 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
632 if not bb.data.getVar("BUILDNAME", make.cfg):
633 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
635 buildname = bb.data.getVar("BUILDNAME", make.cfg)
637 bf = make.options.buildfile
639 executeOneBB( os.path.abspath(bf) )
642 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
643 __ignored_dependencies = ignore.split()
645 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
647 collection_list = collections.split()
648 for c in collection_list:
649 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
651 bb.error("BBFILE_PATTERN_%s not defined" % c)
653 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
655 bb.error("BBFILE_PRIORITY_%s not defined" % c)
658 cre = re.compile(regex)
660 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
664 bbfile_config_priorities.append((cre, pri))
666 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
670 if not pkgs_to_build:
672 pkgs_to_build.extend(args)
673 if not pkgs_to_build:
674 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
676 pkgs_to_build = bbpkgs.split()
677 if not pkgs_to_build and not make.options.show_versions:
678 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
679 print "for usage information."
683 # Import Psyco if available and not disabled
684 if not make.options.disable_psyco:
689 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
691 psyco.bind( make.collect_bbfiles )
693 bb.note("You have disabled Psyco. This decreases performance.")
696 bb.debug(1, "collecting .bb files")
697 make.collect_bbfiles( myProgressCallback )
698 bb.debug(1, "parsing complete")
701 if make.options.parse_only:
702 print "Requested parsing .bb files only. Exiting."
707 if make.options.show_versions:
711 if 'world' in pkgs_to_build:
712 pkgs_to_build.remove('world')
713 for t in __world_target:
714 pkgs_to_build.append(t)
716 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
718 for k in pkgs_to_build:
721 if buildPackage(k) == 0:
724 except bb.build.EventException:
725 bb.error("Build of " + k + " failed")
729 if make.options.abort:
732 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
736 except KeyboardInterrupt:
737 print "\nNOTE: KeyboardInterrupt - Build not completed."