2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
42 bbfile_config_priorities = []
46 def handle_options( args ):
47 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
48 usage = """%prog [options] [package ...]
50 Executes the specified task (default is 'build') for a given set of BitBake files.
51 It expects that BBFILES is defined, which is a space seperated list of files to
52 be executed. BBFILES does support wildcards.
53 Default BBFILES are the .bb files in the current directory.""" )
55 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
56 action = "store", dest = "buildfile", default = None )
58 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
59 action = "store_false", dest = "abort", default = True )
61 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
62 action = "store_true", dest = "force", default = False )
65 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
66 action = "store", dest = "cmd", default = "build" )
68 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
69 action = "append", dest = "file", default = [] )
71 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
72 action = "store_true", dest = "verbose", default = False )
73 parser.add_option( "-D", "--debug", help = "Increase the debug level",
74 action = "count", dest="debug", default = 0)
76 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
77 action = "store_true", dest = "dry_run", default = False )
79 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
80 action = "store_true", dest = "parse_only", default = False )
82 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
83 action = "store_true", dest = "disable_psyco", default = False )
85 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
86 action = "store_true", dest = "show_versions", default = False )
88 options, args = parser.parse_args( args )
89 return options, args[1:]
91 def try_build(fn, virtual):
92 if fn in __building_list:
93 bb.error("%s depends on itself (eventually)" % fn)
94 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
97 __building_list.append(fn)
99 the_data = make.pkgdata[fn]
100 item = bb.data.getVar('PN', the_data, 1)
101 pathstr = "%s (%s)" % (item, virtual)
102 __build_path.append(pathstr)
104 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
105 if make.options.verbose:
106 bb.note("current path: %s" % (" -> ".join(__build_path)))
107 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
112 depcmd = make.options.cmd
113 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', make.pkgdata[fn])
114 if bbdepcmd is not None:
121 oldcmd = make.options.cmd
122 make.options.cmd = depcmd
124 for d in depends_list:
125 if d in __ignored_dependencies:
129 if buildPackage(d) == 0:
130 bb.error("dependency %s (for %s) not satisfied" % (d,item))
132 if make.options.abort:
136 make.options.cmd = oldcmd
142 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
144 __stats["attempt"] += 1
145 if not make.options.dry_run:
146 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
147 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
148 __build_cache.append(fn)
150 except bb.build.FuncFailed:
152 bb.error("task stack execution failed")
153 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
154 __build_cache_fail.append(fn)
156 except bb.build.EventException:
158 (type, value, traceback) = sys.exc_info()
160 bb.error("%s event exception, aborting" % bb.event.getName(e))
161 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
162 __build_cache_fail.append(fn)
165 __building_list.remove(fn)
166 __build_path.remove(pathstr)
170 preferred_versions = {}
173 for p in make.pkgdata.keys():
174 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
175 if not pkg_pn.has_key(pn):
180 for pn in pkg_pn.keys():
184 priority = bbfile_priority[f]
185 if not priorities.has_key(priority):
186 priorities[priority] = []
187 priorities[priority].append(f)
188 p_list = priorities.keys()
189 p_list.sort(lambda a, b: a - b)
192 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
194 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
195 # version. If not, find the latest version provided by an bbfile in the
196 # highest-priority set.
197 for pn in pkg_pn.keys():
198 preferred_file = None
200 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
203 m = re.match('(.*)_(.*)', preferred_v)
205 preferred_v = m.group(1)
206 preferred_r = m.group(2)
208 for file_set in pkg_pn[pn]:
210 the_data = make.pkgdata[f]
211 pv = bb.data.getVar('PV', the_data, 1)
212 pr = bb.data.getVar('PR', the_data, 1)
213 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
215 preferred_ver = (pv, pr)
220 pv_str = '%s-%s' % (preferred_v, preferred_r)
223 if preferred_file is None:
224 bb.note("preferred version %s of %s not available" % (pv_str, pn))
226 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
228 # get highest priority file set
229 files = pkg_pn[pn][0]
234 the_data = make.pkgdata[f]
235 pv = bb.data.getVar('PV', the_data, 1)
236 pr = bb.data.getVar('PR', the_data, 1)
237 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
239 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
243 if preferred_file is None:
244 preferred_file = latest_f
245 preferred_ver = latest
247 preferred_versions[pn] = (preferred_ver, preferred_file)
248 latest_versions[pn] = (latest, latest_f)
250 pkg_list = pkg_pn.keys()
254 pref = preferred_versions[p]
255 latest = latest_versions[p]
258 prefstr = pref[0][0] + "-" + pref[0][1]
262 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
265 def buildPackage(item):
268 discriminated = False
270 if not providers.has_key(item):
271 bb.error("Nothing provides %s" % item)
274 all_p = providers[item]
277 if p in __build_cache:
278 bb.debug(1, "already built %s in this run\n" % p)
282 preferred_versions = {}
284 # Collate providers by PN
287 the_data = make.pkgdata[p]
288 pn = bb.data.getVar('PN', the_data, 1)
289 if not pkg_pn.has_key(pn):
293 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
296 for pn in pkg_pn.keys():
300 priority = bbfile_priority[f]
301 if not priorities.has_key(priority):
302 priorities[priority] = []
303 priorities[priority].append(f)
304 p_list = priorities.keys()
305 p_list.sort(lambda a, b: a - b)
308 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
310 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
311 # version. If not, find the latest version provided by an bbfile in the
312 # highest-priority set.
313 for pn in pkg_pn.keys():
314 preferred_file = None
316 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
319 m = re.match('(.*)_(.*)', preferred_v)
321 preferred_v = m.group(1)
322 preferred_r = m.group(2)
324 for file_set in pkg_pn[pn]:
326 the_data = make.pkgdata[f]
327 pv = bb.data.getVar('PV', the_data, 1)
328 pr = bb.data.getVar('PR', the_data, 1)
329 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
331 preferred_ver = (pv, pr)
336 pv_str = '%s-%s' % (preferred_v, preferred_r)
339 if preferred_file is None:
340 bb.note("preferred version %s of %s not available" % (pv_str, pn))
342 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
344 if preferred_file is None:
345 # get highest priority file set
346 files = pkg_pn[pn][0]
351 the_data = make.pkgdata[f]
352 pv = bb.data.getVar('PV', the_data, 1)
353 pr = bb.data.getVar('PR', the_data, 1)
354 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
356 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
360 preferred_file = latest_f
361 preferred_ver = latest
363 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
365 preferred_versions[pn] = (preferred_ver, preferred_file)
366 eligible.append(preferred_file)
369 if p in __build_cache_fail:
370 bb.debug(1, "rejecting already-failed %s" % p)
373 if len(eligible) == 0:
374 bb.error("no eligible providers for %s" % item)
377 # look to see if one of them is already staged, or marked as preferred.
378 # if so, bump it to the head of the queue
380 the_data = make.pkgdata[p]
381 pn = bb.data.getVar('PN', the_data, 1)
382 pv = bb.data.getVar('PV', the_data, 1)
383 pr = bb.data.getVar('PR', the_data, 1)
384 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
385 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
386 if os.path.exists(stamp):
387 (newvers, fn) = preferred_versions[pn]
388 if not fn in eligible:
389 # package was made ineligible by already-failed check
391 oldver = "%s-%s" % (pv, pr)
392 newver = '-'.join(newvers)
393 if (newver != oldver):
394 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
397 if make.options.verbose:
398 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
400 eligible = [fn] + eligible
404 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
406 __preferred[item] = prefervar
408 if __preferred.has_key(item):
410 the_data = make.pkgdata[p]
411 pn = bb.data.getVar('PN', the_data, 1)
412 if __preferred[item] == pn:
413 if make.options.verbose:
414 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
416 eligible = [p] + eligible
420 if len(eligible) > 1 and discriminated == False:
423 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
424 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
425 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
427 # run through the list until we find one that we can build
429 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
430 if try_build(fn, item):
433 bb.note("no buildable providers for %s" % item)
436 def build_depgraph():
441 if bbdebug or progress.p == p: return
443 if os.isatty(sys.stdout.fileno()):
444 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
448 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
450 sys.stdout.write("done.\n")
453 def calc_bbfile_priority(filename):
454 for (regex, pri) in bbfile_config_priorities:
455 if regex.match(filename):
459 # Handle PREFERRED_PROVIDERS
460 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
461 (providee, provider) = p.split(':')
462 if __preferred.has_key(providee) and __preferred[providee] != provider:
463 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
464 __preferred[providee] = provider
466 # Calculate priorities for each file
467 for p in make.pkgdata.keys():
468 bbfile_priority[p] = calc_bbfile_priority(p)
470 n = len(make.pkgdata.keys())
475 bb.debug(1, "BBMAKE building providers hashes")
477 # Build forward and reverse provider hashes
478 # Forward: virtual -> [filenames]
479 # Reverse: PN -> [virtuals]
480 for f in make.pkgdata.keys():
483 pn = bb.data.getVar('PN', d, 1)
484 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
486 if not pn_provides.has_key(pn):
487 pn_provides[pn] = Set()
488 pn_provides[pn] |= provides
490 for provide in provides:
491 if not providers.has_key(provide):
492 providers[provide] = []
493 providers[provide].append(f)
495 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
506 sys.stdout.write("\n")
508 # Build package list for "bitbake world"
509 bb.debug(1, "BBMAKE collating packages for \"world\"")
510 for f in make.pkgdata.keys():
512 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
513 bb.debug(2, "BBMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
516 pn = bb.data.getVar('PN', d, 1)
517 for p in pn_provides[pn]:
518 if p.startswith('virtual/'):
519 bb.debug(2, "BBMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
522 for pf in providers[p]:
523 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
524 bb.debug(2, "BBMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
528 __world_target.add(pn)
530 def myProgressCallback( x, y, f ):
533 if os.isatty(sys.stdout.fileno()):
534 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
538 sys.stdout.write("Parsing .bb files, please wait...")
541 sys.stdout.write("done.")
544 def executeOneBB( fn ):
546 d = bb.parse.handle(fn, make.cfg)
548 bb.fatal("Unable to open %s" % fn)
550 if make.options.parse_only:
551 print "Requested parsing .bb files only. Exiting."
554 name = bb.data.getVar('PN', d, 1)
555 bb.event.fire(bb.event.PkgStarted(name, d))
557 __stats["attempt"] += 1
558 if make.options.force:
559 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, d)
560 if not make.options.dry_run:
561 bb.build.exec_task('do_%s' % make.options.cmd, d)
562 bb.event.fire(bb.event.PkgSucceeded(name, d))
563 __build_cache.append(fn)
564 except bb.build.FuncFailed:
566 bb.error("task stack execution failed")
567 bb.event.fire(bb.event.PkgFailed(name, d))
568 __build_cache_fail.append(fn)
569 except bb.build.EventException:
571 (type, value, traceback) = sys.exc_info()
573 bb.error("%s event exception, aborting" % bb.event.getName(e))
574 bb.event.fire(bb.event.PkgFailed(name, d))
575 __build_cache_fail.append(fn)
581 __stats["attempt"] = 0
582 __stats["success"] = 0
587 print "Build statistics:"
588 print " Attempted builds: %d" % __stats["attempt"]
589 if __stats["fail"] != 0:
590 print " Failed builds: %d" % __stats["fail"]
591 if __stats["deps"] != 0:
592 print " Dependencies not satisfied: %d" % __stats["deps"]
593 if __stats["fail"] != 0 or __stats["deps"] != 0:
597 if __name__ == "__main__":
599 make.options, args = handle_options( sys.argv )
601 if not make.options.cmd:
602 make.options.cmd = "build"
604 if make.options.debug:
605 bb.debug_level = make.options.debug
608 make.cfg = bb.data.init()
611 for f in make.options.file:
613 make.cfg = bb.parse.handle(f, make.cfg)
615 bb.fatal("Unable to open %s" % f)
618 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
620 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
622 if not bb.data.getVar("BUILDNAME", make.cfg):
623 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
625 buildname = bb.data.getVar("BUILDNAME", make.cfg)
627 bf = make.options.buildfile
629 executeOneBB( os.path.abspath(bf) )
632 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
633 __ignored_dependencies = ignore.split()
635 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
637 collection_list = collections.split()
638 for c in collection_list:
639 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
641 bb.error("BBFILE_PATTERN_%s not defined" % c)
643 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
645 bb.error("BBFILE_PRIORITY_%s not defined" % c)
648 cre = re.compile(regex)
650 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
654 bbfile_config_priorities.append((cre, pri))
656 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
660 if not pkgs_to_build:
662 pkgs_to_build.extend(args)
663 if not pkgs_to_build:
664 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
666 pkgs_to_build = bbpkgs.split()
667 if not pkgs_to_build and not make.options.show_versions:
668 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
669 print "for usage information."
673 # Import Psyco if available and not disabled
674 if not make.options.disable_psyco:
679 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
681 psyco.bind( make.collect_bbfiles )
683 bb.note("You have disabled Psyco. This decreases performance.")
686 bb.debug(1, "BBMAKE collecting .bb files")
687 make.collect_bbfiles( myProgressCallback )
688 bb.debug(1, "BBMAKE parsing complete")
691 if make.options.parse_only:
692 print "Requested parsing .bb files only. Exiting."
697 if make.options.show_versions:
701 if 'world' in pkgs_to_build:
702 pkgs_to_build.remove('world')
703 for t in __world_target:
704 pkgs_to_build.append(t)
706 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
708 for k in pkgs_to_build:
711 if buildPackage(k) == 0:
714 except bb.build.EventException:
715 bb.error("Build of " + k + " failed")
719 if make.options.abort:
722 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
726 except KeyboardInterrupt:
727 print "\nNOTE: KeyboardInterrupt - Build not completed."