2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
8 # This program is free software; you can redistribute it and/or modify it under
9 # the terms of the GNU General Public License as published by the Free Software
10 # Foundation; either version 2 of the License, or (at your option) any later
13 # This program is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License along with
18 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
19 # Place, Suite 330, Boston, MA 02111-1307 USA.
21 import sys, os, getopt, glob, copy, os.path, re
22 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
26 import itertools, optparse
28 parsespin = itertools.cycle( r'|/-\\' )
31 __build_cache_fail = []
37 __world_target = Set()
38 __ignored_dependencies = Set()
39 __depcmds = { "clean": None,
44 bbfile_config_priorities = []
48 def handle_options( args ):
49 parser = optparse.OptionParser( version = "BitBake Build Infrastructure Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
50 usage = """%prog [options] [package ...]
52 Builds specified packages, expecting that the .bb files
53 it has to work from are in BBFILES
54 Default packages are all packages in BBFILES.
55 Default BBFILES are the .bb files in the current directory.""" )
57 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
58 action = "store_false", dest = "abort", default = True )
60 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
61 action = "store_true", dest = "force", default = False )
64 parser.add_option( "-c", "--cmd", help = "specify command to pass to bbbuild. Valid commands are "
65 "'fetch' (fetch all sources), "
66 "'unpack' (unpack the sources), "
67 "'patch' (apply the patches), "
68 "'configure' (configure the source tree), "
69 "'compile' (compile the source tree), "
70 "'stage' (install libraries and headers needed for subsequent packages), "
71 "'install' (install libraries and executables), and"
72 "'package' (package files into the selected package format)",
73 action = "store", dest = "cmd", default = "build" )
75 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
76 action = "append", dest = "file", default = [] )
78 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
79 action = "store_true", dest = "verbose", default = False )
81 parser.add_option( "-n", "--dry-run", help = "don't call bbbuild, just go through the motions",
82 action = "store_true", dest = "dry_run", default = False )
84 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
85 action = "store_true", dest = "parse_only", default = False )
87 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
88 action = "store_true", dest = "disable_psyco", default = False )
90 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
91 action = "store_true", dest = "show_versions", default = False )
93 options, args = parser.parse_args( args )
94 return options, args[1:]
96 def try_build(fn, virtual):
97 if fn in __building_list:
98 bb.error("%s depends on itself (eventually)" % fn)
99 bb.error("upwards chain is: %s" % (" -> ".join(__build_path)))
102 __building_list.append(fn)
104 the_data = make.pkgdata[fn]
105 item = bb.data.getVar('PN', the_data, 1)
106 pathstr = "%s (%s)" % (item, virtual)
107 __build_path.append(pathstr)
109 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
110 if make.options.verbose:
111 bb.note("current path: %s" % (" -> ".join(__build_path)))
112 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
118 oldcmd = make.options.cmd
119 make.options.cmd = __depcmd
121 for d in depends_list:
122 if d in __ignored_dependencies:
126 if buildPackage(d) == 0:
127 bb.error("dependency %s (for %s) not satisfied" % (d,item))
129 if make.options.abort:
133 make.options.cmd = oldcmd
139 bb.event.fire(bb.event.PkgStarted(item, make.pkgdata[fn]))
141 __stats["attempt"] += 1
142 if not make.options.dry_run:
143 bb.build.exec_task('do_%s' % make.options.cmd, make.pkgdata[fn])
144 bb.event.fire(bb.event.PkgSucceeded(item, make.pkgdata[fn]))
145 __build_cache.append(fn)
147 except bb.build.FuncFailed:
149 bb.error("task stack execution failed")
150 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
151 __build_cache_fail.append(fn)
153 except bb.build.EventException:
155 (type, value, traceback) = sys.exc_info()
157 bb.error("%s event exception, aborting" % bb.event.getName(e))
158 bb.event.fire(bb.event.PkgFailed(item, make.pkgdata[fn]))
159 __build_cache_fail.append(fn)
162 __building_list.remove(fn)
163 __build_path.remove(pathstr)
167 preferred_versions = {}
170 for p in make.pkgdata.keys():
171 pn = bb.data.getVar('PN', make.pkgdata[p], 1)
172 if not pkg_pn.has_key(pn):
177 for pn in pkg_pn.keys():
181 priority = bbfile_priority[f]
182 if not priorities.has_key(priority):
183 priorities[priority] = []
184 priorities[priority].append(f)
185 p_list = priorities.keys()
186 p_list.sort(lambda a, b: a - b)
189 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
191 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
192 # version. If not, find the latest version provided by an bbfile in the
193 # highest-priority set.
194 for pn in pkg_pn.keys():
195 preferred_file = None
197 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
200 m = re.match('(.*)_(.*)', preferred_v)
202 preferred_v = m.group(1)
203 preferred_r = m.group(2)
205 for file_set in pkg_pn[pn]:
207 the_data = make.pkgdata[f]
208 pv = bb.data.getVar('PV', the_data, 1)
209 pr = bb.data.getVar('PR', the_data, 1)
210 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
212 preferred_ver = (pv, pr)
217 pv_str = '%s-%s' % (preferred_v, preferred_r)
220 if preferred_file is None:
221 bb.note("preferred version %s of %s not available" % (pv_str, pn))
223 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
225 # get highest priority file set
226 files = pkg_pn[pn][0]
231 the_data = make.pkgdata[f]
232 pv = bb.data.getVar('PV', the_data, 1)
233 pr = bb.data.getVar('PR', the_data, 1)
234 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
236 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
240 if preferred_file is None:
241 preferred_file = latest_f
242 preferred_ver = latest
244 preferred_versions[pn] = (preferred_ver, preferred_file)
245 latest_versions[pn] = (latest, latest_f)
247 pkg_list = pkg_pn.keys()
251 pref = preferred_versions[p]
252 latest = latest_versions[p]
255 prefstr = pref[0][0] + "-" + pref[0][1]
259 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
262 def buildPackage(item):
265 discriminated = False
267 if not providers.has_key(item):
268 bb.error("Nothing provides %s" % item)
271 all_p = providers[item]
274 if p in __build_cache:
275 bb.debug(1, "already built %s in this run\n" % p)
279 preferred_versions = {}
281 # Collate providers by PN
284 the_data = make.pkgdata[p]
285 pn = bb.data.getVar('PN', the_data, 1)
286 if not pkg_pn.has_key(pn):
290 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
293 for pn in pkg_pn.keys():
297 priority = bbfile_priority[f]
298 if not priorities.has_key(priority):
299 priorities[priority] = []
300 priorities[priority].append(f)
301 p_list = priorities.keys()
302 p_list.sort(lambda a, b: a - b)
305 pkg_pn[pn] = [ priorities[p] ] + pkg_pn[pn]
307 # If there is a PREFERRED_VERSION, find the highest-priority bbfile providing that
308 # version. If not, find the latest version provided by an bbfile in the
309 # highest-priority set.
310 for pn in pkg_pn.keys():
311 preferred_file = None
313 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
316 m = re.match('(.*)_(.*)', preferred_v)
318 preferred_v = m.group(1)
319 preferred_r = m.group(2)
321 for file_set in pkg_pn[pn]:
323 the_data = make.pkgdata[f]
324 pv = bb.data.getVar('PV', the_data, 1)
325 pr = bb.data.getVar('PR', the_data, 1)
326 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
328 preferred_ver = (pv, pr)
333 pv_str = '%s-%s' % (preferred_v, preferred_r)
336 if preferred_file is None:
337 bb.note("preferred version %s of %s not available" % (pv_str, pn))
339 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
341 if preferred_file is None:
342 # get highest priority file set
343 files = pkg_pn[pn][0]
348 the_data = make.pkgdata[f]
349 pv = bb.data.getVar('PV', the_data, 1)
350 pr = bb.data.getVar('PR', the_data, 1)
351 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', the_data, 1) or "0")
353 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
357 preferred_file = latest_f
358 preferred_ver = latest
360 bb.debug(1, "selecting %s as latest version of provider %s" % (preferred_file, pn))
362 preferred_versions[pn] = (preferred_ver, preferred_file)
363 eligible.append(preferred_file)
366 if p in __build_cache_fail:
367 bb.debug(1, "rejecting already-failed %s" % p)
370 if len(eligible) == 0:
371 bb.error("no eligible providers for %s" % item)
374 # look to see if one of them is already staged, or marked as preferred.
375 # if so, bump it to the head of the queue
377 the_data = make.pkgdata[p]
378 pn = bb.data.getVar('PN', the_data, 1)
379 pv = bb.data.getVar('PV', the_data, 1)
380 pr = bb.data.getVar('PR', the_data, 1)
381 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
382 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
383 if os.path.exists(stamp):
384 (newvers, fn) = preferred_versions[pn]
385 if not fn in eligible:
386 # package was made ineligible by already-failed check
388 oldver = "%s-%s" % (pv, pr)
389 newver = '-'.join(newvers)
390 if (newver != oldver):
391 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
394 if make.options.verbose:
395 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
397 eligible = [fn] + eligible
401 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
403 __preferred[item] = prefervar
405 if __preferred.has_key(item):
407 the_data = make.pkgdata[p]
408 pn = bb.data.getVar('PN', the_data, 1)
409 if __preferred[item] == pn:
410 if make.options.verbose:
411 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
413 eligible = [p] + eligible
417 if len(eligible) > 1 and discriminated == False:
420 providers_list.append(bb.data.getVar('PN', make.pkgdata[fn], 1))
421 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
422 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
424 # run through the list until we find one that we can build
426 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
427 if try_build(fn, item):
430 bb.note("no buildable providers for %s" % item)
433 def build_depgraph():
438 if bbdebug or progress.p == p: return
440 if os.isatty(sys.stdout.fileno()):
441 sys.stdout.write("\rNOTE: Building provider hash: [%s%s] (%02d%%)" % ( "#" * (p/5), " " * ( 20 - p/5 ), p ) )
445 sys.stdout.write("NOTE: Building provider hash, please wait...\n")
447 sys.stdout.write("done.\n")
450 def calc_bbfile_priority(filename):
451 for (regex, pri) in bbfile_config_priorities:
452 if regex.match(filename):
456 # Handle PREFERRED_PROVIDERS
457 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
458 (providee, provider) = p.split(':')
459 if __preferred.has_key(providee) and __preferred[providee] != provider:
460 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, __preferred[providee]))
461 __preferred[providee] = provider
463 # Calculate priorities for each file
464 for p in make.pkgdata.keys():
465 bbfile_priority[p] = calc_bbfile_priority(p)
467 n = len(make.pkgdata.keys())
472 bb.debug(1, "BBMAKE building providers hashes")
474 # Build forward and reverse provider hashes
475 # Forward: virtual -> [filenames]
476 # Reverse: PN -> [virtuals]
477 for f in make.pkgdata.keys():
480 pn = bb.data.getVar('PN', d, 1)
481 provides = Set([pn] + (bb.data.getVar("PROVIDES", d, 1) or "").split())
483 if not pn_provides.has_key(pn):
484 pn_provides[pn] = Set()
485 pn_provides[pn] |= provides
487 for provide in provides:
488 if not providers.has_key(provide):
489 providers[provide] = []
490 providers[provide].append(f)
492 deps = (bb.data.getVar("DEPENDS", d, 1) or "").split()
503 sys.stdout.write("\n")
505 # Build package list for "bbmake world"
506 bb.debug(1, "BBMAKE collating packages for \"world\"")
507 for f in make.pkgdata.keys():
509 if bb.data.getVar('BROKEN', d, 1) or bb.data.getVar('EXCLUDE_FROM_WORLD', d, 1):
510 bb.debug(2, "BBMAKE skipping %s due to BROKEN/EXCLUDE_FROM_WORLD" % f)
513 pn = bb.data.getVar('PN', d, 1)
514 for p in pn_provides[pn]:
515 if p.startswith('virtual/'):
516 bb.debug(2, "BBMAKE skipping %s due to %s provider starting with virtual/" % (f, p))
519 for pf in providers[p]:
520 if bb.data.getVar('PN', make.pkgdata[pf], 1) != pn:
521 bb.debug(2, "BBMAKE skipping %s due to both us and %s providing %s" % (f, pf, p))
525 __world_target.add(pn)
527 def myProgressCallback( x, y, f ):
530 if os.isatty(sys.stdout.fileno()):
531 sys.stdout.write("\rNOTE: Parsing .bb files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
535 sys.stdout.write("Parsing .bb files, please wait...")
538 sys.stdout.write("done.")
546 if __name__ == "__main__":
548 if "BBDEBUG" in os.environ:
549 bbdebug = int(os.environ["BBDEBUG"])
551 make.options, args = handle_options( sys.argv )
553 if not make.options.cmd:
554 make.options.cmd = "build"
556 if make.options.cmd in __depcmds:
557 __depcmd=__depcmds[make.options.cmd]
559 __depcmd=make.options.cmd
562 make.cfg = bb.data.init()
565 for f in make.options.file:
567 make.cfg = bb.parse.handle(f, make.cfg)
569 bb.fatal("Unable to open %s" % f)
572 make.cfg = bb.parse.handle(os.path.join('conf', 'bitbake.conf'), make.cfg)
574 bb.fatal("Unable to open %s" % os.path.join('conf', 'bitbake.conf'))
576 if not bb.data.getVar("BUILDNAME", make.cfg):
577 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
579 buildname = bb.data.getVar("BUILDNAME", make.cfg)
581 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
582 __ignored_dependencies = ignore.split()
584 collections = bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1)
586 collection_list = collections.split()
587 for c in collection_list:
588 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
590 bb.error("BBFILE_PATTERN_%s not defined" % c)
592 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
594 bb.error("BBFILE_PRIORITY_%s not defined" % c)
597 cre = re.compile(regex)
599 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
603 bbfile_config_priorities.append((cre, pri))
605 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
609 if not pkgs_to_build:
611 pkgs_to_build.extend(args)
612 if not pkgs_to_build:
613 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
615 pkgs_to_build = bbpkgs.split()
616 if not pkgs_to_build and not make.options.show_versions:
617 print "Nothing to build. Use 'bbmake world' to build everything."
620 __stats["attempt"] = 0
621 __stats["success"] = 0
625 # Import Psyco if available and not disabled
626 if not make.options.disable_psyco:
631 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
633 psyco.bind( make.collect_bbfiles )
635 bb.note("You have disabled Psyco. This decreases performance.")
638 bb.debug(1, "BBMAKE collecting .bb files")
639 make.collect_bbfiles( myProgressCallback )
640 bb.debug(1, "BBMAKE parsing complete")
643 if make.options.parse_only:
644 print "Requested parsing .bb files only. Exiting."
649 if make.options.show_versions:
653 if 'world' in pkgs_to_build:
654 pkgs_to_build.remove('world')
655 for t in __world_target:
656 pkgs_to_build.append(t)
658 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
660 for k in pkgs_to_build:
663 if buildPackage(k) == 0:
666 except bb.build.EventException:
667 bb.error("Build of " + k + " failed")
671 if make.options.abort:
674 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
676 print "Build statistics:"
677 print " Attempted builds: %d" % __stats["attempt"]
678 if __stats["fail"] != 0:
679 print " Failed builds: %d" % __stats["fail"]
680 if __stats["deps"] != 0:
681 print " Dependencies not satisfied: %d" % __stats["deps"]
682 if __stats["fail"] != 0 or __stats["deps"] != 0:
686 except KeyboardInterrupt:
687 print "\nNOTE: KeyboardInterrupt - Build not completed."