2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if pn not in self.pkg_pn:
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if pn not in self.pn_provides:
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if provide not in self.providers:
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
143 ParsingStatus = BBParsingStatus # make it visible from the shell
144 Statistics = BBStatistics # make it visible from the shell
146 def __init__( self ):
147 self.build_cache_fail = []
148 self.build_cache = []
149 self.building_list = []
151 self.consider_msgs_cache = []
153 self.stats = BBStatistics()
156 def tryBuildPackage( self, fn, item, the_data ):
157 """Build one package"""
158 bb.event.fire(bb.event.PkgStarted(item, the_data))
160 self.stats.attempt += 1
161 if make.options.force:
162 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, the_data)
163 if not make.options.dry_run:
164 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
165 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
166 self.build_cache.append(fn)
168 except bb.build.FuncFailed:
170 bb.error("task stack execution failed")
171 bb.event.fire(bb.event.PkgFailed(item, the_data))
172 self.build_cache_fail.append(fn)
174 except bb.build.EventException, e:
177 bb.error("%s event exception, aborting" % bb.event.getName(event))
178 bb.event.fire(bb.event.PkgFailed(item, the_data))
179 self.build_cache_fail.append(fn)
182 def tryBuild( self, fn, virtual ):
183 """Build a provider and its dependencies"""
184 if fn in self.building_list:
185 bb.error("%s depends on itself (eventually)" % fn)
186 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
189 the_data = make.pkgdata[fn]
190 item = self.status.pkg_fn[fn]
192 self.building_list.append(fn)
194 pathstr = "%s (%s)" % (item, virtual)
195 self.build_path.append(pathstr)
197 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
198 if make.options.verbose:
199 bb.note("current path: %s" % (" -> ".join(self.build_path)))
200 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
205 depcmd = make.options.cmd
206 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
207 if bbdepcmd is not None:
214 oldcmd = make.options.cmd
215 make.options.cmd = depcmd
217 for dependency in depends_list:
218 if dependency in self.status.ignored_dependencies:
222 if self.buildProvider( dependency ) == 0:
223 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
225 if make.options.abort:
229 make.options.cmd = oldcmd
235 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
236 self.build_cache.append(fn)
239 return self.tryBuildPackage( fn, item, the_data )
242 self.building_list.remove(fn)
243 self.build_path.remove(pathstr)
245 def findBestProvider( self, pn ):
247 If there is a PREFERRED_VERSION, find the highest-priority bbfile
248 providing that version. If not, find the latest version provided by
249 an bbfile in the highest-priority set.
251 pkg_pn = self.status.pkg_pn
256 priority = self.status.bbfile_priority[f]
257 if priority not in priorities:
258 priorities[priority] = []
259 priorities[priority].append(f)
260 p_list = priorities.keys()
261 p_list.sort(lambda a, b: a - b)
264 tmp_pn = priorities[p] + tmp_pn
267 preferred_file = None
269 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
271 m = re.match('(.*)_(.*)', preferred_v)
273 preferred_v = m.group(1)
274 preferred_r = m.group(2)
279 pv,pr = self.status.pkg_pvpr[f]
280 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
282 preferred_ver = (pv, pr)
285 pv_str = '%s-%s' % (preferred_v, preferred_r)
288 if preferred_file is None:
289 bb.note("preferred version %s of %s not available" % (pv_str, pn))
291 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
293 # get highest priority file set
298 for file_name in files:
299 pv,pr = self.status.pkg_pvpr[file_name]
300 dp = self.status.pkg_dp[file_name]
302 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
306 if preferred_file is None:
307 preferred_file = latest_f
308 preferred_ver = latest
310 return (latest,latest_f,preferred_ver, preferred_file)
312 def showVersions( self ):
313 pkg_pn = self.status.pkg_pn
314 preferred_versions = {}
318 for pn in pkg_pn.keys():
319 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
320 preferred_versions[pn] = (pref_ver, pref_file)
321 latest_versions[pn] = (last_ver, last_file)
323 pkg_list = pkg_pn.keys()
327 pref = preferred_versions[p]
328 latest = latest_versions[p]
331 prefstr = pref[0][0] + "-" + pref[0][1]
335 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
338 def buildProvider( self, item ):
341 discriminated = False
343 if item not in self.status.providers:
344 bb.error("Nothing provides %s" % item)
347 all_p = self.status.providers[item]
350 if p in self.build_cache:
351 bb.debug(1, "already built %s in this run\n" % p)
355 preferred_versions = {}
357 # Collate providers by PN
360 pn = self.status.pkg_fn[p]
365 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
367 for pn in pkg_pn.keys():
368 preferred_versions[pn] = self.findBestProvider(pn)[2:4]
369 eligible.append(preferred_versions[pn][1])
372 if p in self.build_cache_fail:
373 bb.debug(1, "rejecting already-failed %s" % p)
376 if len(eligible) == 0:
377 bb.error("no eligible providers for %s" % item)
380 # look to see if one of them is already staged, or marked as preferred.
381 # if so, bump it to the head of the queue
383 the_data = make.pkgdata[p]
384 pn = bb.data.getVar('PN', the_data, 1)
385 pv = bb.data.getVar('PV', the_data, 1)
386 pr = bb.data.getVar('PR', the_data, 1)
387 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
388 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
389 if os.path.exists(stamp):
390 (newvers, fn) = preferred_versions[pn]
391 if not fn in eligible:
392 # package was made ineligible by already-failed check
394 oldver = "%s-%s" % (pv, pr)
395 newver = '-'.join(newvers)
396 if (newver != oldver):
397 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
400 if make.options.verbose:
401 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
403 eligible = [fn] + eligible
407 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
409 self.preferred[item] = prefervar
411 if item in self.preferred:
413 pn = self.status.pkg_fn[p]
414 if self.preferred[item] == pn:
415 if make.options.verbose:
416 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
418 eligible = [p] + eligible
422 if len(eligible) > 1 and discriminated == False:
423 if item not in self.consider_msgs_cache:
426 providers_list.append(self.status.pkg_fn[fn])
427 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
428 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
429 self.consider_msgs_cache.append(item)
432 # run through the list until we find one that we can build
434 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
435 if self.tryBuild(fn, item):
438 bb.note("no buildable providers for %s" % item)
441 def buildDepgraph( self ):
442 all_depends = self.status.all_depends
443 pn_provides = self.status.pn_provides
445 def calc_bbfile_priority(filename):
446 for (regex, pri) in self.status.bbfile_config_priorities:
447 if regex.match(filename):
451 # Handle PREFERRED_PROVIDERS
452 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
453 (providee, provider) = p.split(':')
454 if providee in self.preferred and self.preferred[providee] != provider:
455 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
456 self.preferred[providee] = provider
458 # Calculate priorities for each file
459 for p in make.pkgdata.keys():
460 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
462 # Build package list for "bitbake world"
463 bb.debug(1, "collating packages for \"world\"")
464 for f in self.status.possible_world:
466 pn = self.status.pkg_fn[f]
468 for p in pn_provides[pn]:
469 if p.startswith('virtual/'):
470 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
473 for pf in self.status.providers[p]:
474 if self.status.pkg_fn[pf] != pn:
475 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
479 self.status.world_target.add(pn)
481 # drop reference count now
482 self.status.possible_world = None
483 self.status.all_depends = None
485 def myProgressCallback( self, x, y, f, file_data, from_cache ):
486 # feed the status with new input
487 self.status.handle_bb_data(f, file_data, from_cache)
491 if os.isatty(sys.stdout.fileno()):
492 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
496 sys.stdout.write("Parsing .bb files, please wait...")
499 sys.stdout.write("done.")
502 def interactiveMode( self ):
503 """Drop off into a shell"""
506 except ImportError, details:
507 bb.fatal("Sorry, shell not available (%s)" % details )
512 def parseConfigurationFile( self, afile ):
514 make.cfg = bb.parse.handle( afile, make.cfg )
516 bb.fatal( "Unable to open %s" % afile )
517 except bb.parse.ParseError:
518 bb.fatal( "Unable to parse %s" % afile )
520 def handleCollections( self, collections ):
521 """Handle collections"""
523 collection_list = collections.split()
524 for c in collection_list:
525 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
527 bb.error("BBFILE_PATTERN_%s not defined" % c)
529 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
531 bb.error("BBFILE_PRIORITY_%s not defined" % c)
534 cre = re.compile(regex)
536 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
540 self.status.bbfile_config_priorities.append((cre, pri))
542 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
545 def cook( self, args ):
546 if not make.options.cmd:
547 make.options.cmd = "build"
549 if make.options.debug:
550 bb.debug_level = make.options.debug
552 make.cfg = bb.data.init()
554 for f in make.options.file:
555 self.parseConfigurationFile( f )
557 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
559 if not bb.data.getVar("BUILDNAME", make.cfg):
560 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
562 buildname = bb.data.getVar("BUILDNAME", make.cfg)
564 if make.options.interactive:
565 self.interactiveMode()
567 if make.options.buildfile is not None:
568 bf = os.path.abspath( make.options.buildfile )
570 bbfile_data = bb.parse.handle(bf, make.cfg)
572 bb.fatal("Unable to open %s" % bf)
574 item = bb.data.getVar('PN', bbfile_data, 1)
576 self.tryBuildPackage( bf, item, bbfile_data )
577 except bb.build.EventException:
578 bb.error( "Build of '%s' failed" % item )
580 sys.exit( self.stats.show() )
582 # initialise the parsing status now we know we will need deps
583 self.status = BBParsingStatus()
585 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
586 self.status.ignored_dependencies = Set( ignore.split() )
588 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1) )
592 if not pkgs_to_build:
594 pkgs_to_build.extend(args)
595 if not pkgs_to_build:
596 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
598 pkgs_to_build = bbpkgs.split()
599 if not pkgs_to_build and not make.options.show_versions and not make.options.interactive:
600 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
601 print "for usage information."
604 # Import Psyco if available and not disabled
605 if not make.options.disable_psyco:
610 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
612 psyco.bind( make.collect_bbfiles )
614 bb.note("You have disabled Psyco. This decreases performance.")
617 bb.debug(1, "collecting .bb files")
618 make.collect_bbfiles( self.myProgressCallback )
619 bb.debug(1, "parsing complete")
622 if make.options.parse_only:
623 print "Requested parsing .bb files only. Exiting."
626 bb.data.update_data( make.cfg )
629 if make.options.show_versions:
632 if 'world' in pkgs_to_build:
633 pkgs_to_build.remove('world')
634 for t in self.status.world_target:
635 pkgs_to_build.append(t)
637 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
639 for k in pkgs_to_build:
642 if self.buildProvider( k ) == 0:
645 except bb.build.EventException:
646 bb.error("Build of " + k + " failed")
650 if make.options.abort:
653 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
655 sys.exit( self.stats.show() )
657 except KeyboardInterrupt:
658 print "\nNOTE: KeyboardInterrupt - Build not completed."
661 #============================================================================#
663 #============================================================================#
665 if __name__ == "__main__":
667 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
668 usage = """%prog [options] [package ...]
670 Executes the specified task (default is 'build') for a given set of BitBake files.
671 It expects that BBFILES is defined, which is a space seperated list of files to
672 be executed. BBFILES does support wildcards.
673 Default BBFILES are the .bb files in the current directory.""" )
675 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
676 action = "store", dest = "buildfile", default = None )
678 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
679 action = "store_false", dest = "abort", default = True )
681 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
682 action = "store_true", dest = "force", default = False )
684 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
685 action = "store_true", dest = "interactive", default = False )
687 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
688 action = "store", dest = "cmd", default = "build" )
690 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
691 action = "append", dest = "file", default = [] )
693 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
694 action = "store_true", dest = "verbose", default = False )
696 parser.add_option( "-D", "--debug", help = "Increase the debug level",
697 action = "count", dest="debug", default = 0)
699 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
700 action = "store_true", dest = "dry_run", default = False )
702 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
703 action = "store_true", dest = "parse_only", default = False )
705 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
706 action = "store_true", dest = "disable_psyco", default = False )
708 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
709 action = "store_true", dest = "show_versions", default = False )
711 options, args = parser.parse_args( sys.argv )
713 make.options = options
715 cooker.cook( args[1:] )