2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 # Copyright (C) 2003, 2004 Chris Larson
6 # Copyright (C) 2003, 2004 Phil Blundell
7 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
8 # Copyright (C) 2005 Holger Hans Peter Freyther
9 # Copyright (C) 2005 ROAD GmbH
11 # This program is free software; you can redistribute it and/or modify it under
12 # the terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # This program is distributed in the hope that it will be useful, but WITHOUT
17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # this program; if not, write to the Free Software Foundation, Inc., 59 Temple
22 # Place, Suite 330, Boston, MA 02111-1307 USA.
24 import sys, os, getopt, glob, copy, os.path, re
25 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
29 import itertools, optparse
31 parsespin = itertools.cycle( r'|/-\\' )
36 #============================================================================#
38 #============================================================================#
39 class BBParsingStatus:
41 The initial idea for this status class is to use the data when it is
42 already loaded instead of loading it from various place over and over
47 self.cache_dirty = False
49 self.bbfile_priority = {}
50 self.bbfile_config_priorities = []
51 self.ignored_depedencies = None
52 self.possible_world = []
53 self.world_target = Set()
59 self.all_depends = Set()
61 def handle_bb_data(self, file_name, bb_data, cached):
63 We will fill the dictionaries with the stuff we
64 need for building the tree more fast
70 self.cache_dirty = True
72 pn = bb.data.getVar('PN', bb_data, True)
73 pv = bb.data.getVar('PV', bb_data, True)
74 pr = bb.data.getVar('PR', bb_data, True)
75 dp = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
76 provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
77 depends = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
80 # build PackageName to FileName lookup table
81 if pn not in self.pkg_pn:
83 self.pkg_pn[pn].append(file_name)
85 # build FileName to PackageName lookup table
86 self.pkg_fn[file_name] = pn
87 self.pkg_pvpr[file_name] = (pv,pr)
88 self.pkg_dp[file_name] = dp
90 # Build forward and reverse provider hashes
91 # Forward: virtual -> [filenames]
92 # Reverse: PN -> [virtuals]
93 if pn not in self.pn_provides:
94 self.pn_provides[pn] = Set()
95 self.pn_provides[pn] |= provides
97 for provide in provides:
98 if provide not in self.providers:
99 self.providers[provide] = []
100 self.providers[provide].append(file_name)
103 self.all_depends.add(dep)
105 # Collect files we may need for possible world-dep
107 if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
108 self.possible_world.append(file_name)
111 #============================================================================#
113 #============================================================================#
116 Manage build statistics for one run
125 print "Build statistics:"
126 print " Attempted builds: %d" % self.attempt
128 print " Failed builds: %d" % self.fail
130 print " Dependencies not satisfied: %d" % self.deps
131 if self.fail or self.deps: return 1
135 #============================================================================#
137 #============================================================================#
140 Manages one bitbake build run
143 ParsingStatus = BBParsingStatus # make it visible from the shell
144 Statistics = BBStatistics # make it visible from the shell
146 def __init__( self ):
147 self.build_cache_fail = []
148 self.build_cache = []
149 self.building_list = []
151 self.consider_msgs_cache = []
153 self.stats = BBStatistics()
156 def tryBuildPackage( self, fn, item, the_data ):
157 """Build one package"""
158 bb.event.fire(bb.event.PkgStarted(item, the_data))
160 self.stats.attempt += 1
161 if make.options.force:
162 bb.data.setVarFlag('do_%s' % make.options.cmd, 'force', 1, the_data)
163 if not make.options.dry_run:
164 bb.build.exec_task('do_%s' % make.options.cmd, the_data)
165 bb.event.fire(bb.event.PkgSucceeded(item, the_data))
166 self.build_cache.append(fn)
168 except bb.build.FuncFailed:
170 bb.error("task stack execution failed")
171 bb.event.fire(bb.event.PkgFailed(item, the_data))
172 self.build_cache_fail.append(fn)
174 except bb.build.EventException:
176 (type, value, traceback) = sys.exc_info()
178 bb.error("%s event exception, aborting" % bb.event.getName(e))
179 bb.event.fire(bb.event.PkgFailed(item, the_data))
180 self.build_cache_fail.append(fn)
183 def tryBuild( self, fn, virtual ):
184 """Build a provider and its dependencies"""
185 if fn in self.building_list:
186 bb.error("%s depends on itself (eventually)" % fn)
187 bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
190 the_data = make.pkgdata[fn]
191 item = self.status.pkg_fn[fn]
193 self.building_list.append(fn)
195 pathstr = "%s (%s)" % (item, virtual)
196 self.build_path.append(pathstr)
198 depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
199 if make.options.verbose:
200 bb.note("current path: %s" % (" -> ".join(self.build_path)))
201 bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
206 depcmd = make.options.cmd
207 bbdepcmd = bb.data.getVarFlag('do_%s' % make.options.cmd, 'bbdepcmd', the_data)
208 if bbdepcmd is not None:
215 oldcmd = make.options.cmd
216 make.options.cmd = depcmd
218 for dependency in depends_list:
219 if dependency in self.status.ignored_dependencies:
223 if self.buildProvider( dependency ) == 0:
224 bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
226 if make.options.abort:
230 make.options.cmd = oldcmd
236 if bb.build.stamp_is_current('do_%s' % make.options.cmd, the_data):
237 self.build_cache.append(fn)
240 return self.tryBuildPackage( fn, item, the_data )
243 self.building_list.remove(fn)
244 self.build_path.remove(pathstr)
246 def findBestProvider( self, pn ):
248 If there is a PREFERRED_VERSION, find the highest-priority bbfile
249 providing that version. If not, find the latest version provided by
250 an bbfile in the highest-priority set.
252 pkg_pn = self.status.pkg_pn
257 priority = self.status.bbfile_priority[f]
258 if priority not in priorities:
259 priorities[priority] = []
260 priorities[priority].append(f)
261 p_list = priorities.keys()
262 p_list.sort(lambda a, b: a - b)
265 tmp_pn = priorities[p] + tmp_pn
268 preferred_file = None
270 preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, make.cfg, 1)
272 m = re.match('(.*)_(.*)', preferred_v)
274 preferred_v = m.group(1)
275 preferred_r = m.group(2)
280 pv,pr = self.status.pkg_pvpr[f]
281 if preferred_v == pv and (preferred_r == pr or preferred_r == None):
283 preferred_ver = (pv, pr)
286 pv_str = '%s-%s' % (preferred_v, preferred_r)
289 if preferred_file is None:
290 bb.note("preferred version %s of %s not available" % (pv_str, pn))
292 bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
294 # get highest priority file set
299 for file_name in files:
300 pv,pr = self.status.pkg_pvpr[file_name]
301 dp = self.status.pkg_dp[file_name]
303 if (latest is None) or ((latest_p == dp) and (make.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
307 if preferred_file is None:
308 preferred_file = latest_f
309 preferred_ver = latest
311 return (latest,latest_f,preferred_ver, preferred_file)
313 def showVersions( self ):
314 pkg_pn = self.status.pkg_pn
315 preferred_versions = {}
319 for pn in pkg_pn.keys():
320 (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
321 preferred_versions[pn] = (pref_ver, pref_file)
322 latest_versions[pn] = (last_ver, last_file)
324 pkg_list = pkg_pn.keys()
328 pref = preferred_versions[p]
329 latest = latest_versions[p]
332 prefstr = pref[0][0] + "-" + pref[0][1]
336 print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
339 def buildProvider( self, item ):
342 discriminated = False
344 if item not in self.status.providers:
345 bb.error("Nothing provides %s" % item)
348 all_p = self.status.providers[item]
351 if p in self.build_cache:
352 bb.debug(1, "already built %s in this run\n" % p)
356 preferred_versions = {}
358 # Collate providers by PN
361 pn = self.status.pkg_fn[p]
366 bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
368 for pn in pkg_pn.keys():
369 preferred_versions[pn] = self.findBestProvider(pn)[2:4]
370 eligible.append(preferred_versions[pn][1])
373 if p in self.build_cache_fail:
374 bb.debug(1, "rejecting already-failed %s" % p)
377 if len(eligible) == 0:
378 bb.error("no eligible providers for %s" % item)
381 # look to see if one of them is already staged, or marked as preferred.
382 # if so, bump it to the head of the queue
384 the_data = make.pkgdata[p]
385 pn = bb.data.getVar('PN', the_data, 1)
386 pv = bb.data.getVar('PV', the_data, 1)
387 pr = bb.data.getVar('PR', the_data, 1)
388 tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
389 stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
390 if os.path.exists(stamp):
391 (newvers, fn) = preferred_versions[pn]
392 if not fn in eligible:
393 # package was made ineligible by already-failed check
395 oldver = "%s-%s" % (pv, pr)
396 newver = '-'.join(newvers)
397 if (newver != oldver):
398 extra_chat = "; upgrading from %s to %s" % (oldver, newver)
401 if make.options.verbose:
402 bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
404 eligible = [fn] + eligible
408 prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, make.cfg, 1)
410 self.preferred[item] = prefervar
412 if item in self.preferred:
414 pn = self.status.pkg_fn[p]
415 if self.preferred[item] == pn:
416 if make.options.verbose:
417 bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
419 eligible = [p] + eligible
423 if len(eligible) > 1 and discriminated == False:
424 if item not in self.consider_msgs_cache:
427 providers_list.append(self.status.pkg_fn[fn])
428 bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
429 bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
430 self.consider_msgs_cache.append(item)
433 # run through the list until we find one that we can build
435 bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
436 if self.tryBuild(fn, item):
439 bb.note("no buildable providers for %s" % item)
442 def buildDepgraph( self ):
443 all_depends = self.status.all_depends
444 pn_provides = self.status.pn_provides
446 def calc_bbfile_priority(filename):
447 for (regex, pri) in self.status.bbfile_config_priorities:
448 if regex.match(filename):
452 # Handle PREFERRED_PROVIDERS
453 for p in (bb.data.getVar('PREFERRED_PROVIDERS', make.cfg, 1) or "").split():
454 (providee, provider) = p.split(':')
455 if providee in self.preferred and self.preferred[providee] != provider:
456 bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
457 self.preferred[providee] = provider
459 # Calculate priorities for each file
460 for p in make.pkgdata.keys():
461 self.status.bbfile_priority[p] = calc_bbfile_priority(p)
463 # Build package list for "bitbake world"
464 bb.debug(1, "collating packages for \"world\"")
465 for f in self.status.possible_world:
467 pn = self.status.pkg_fn[f]
469 for p in pn_provides[pn]:
470 if p.startswith('virtual/'):
471 bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
474 for pf in self.status.providers[p]:
475 if self.status.pkg_fn[pf] != pn:
476 bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
480 self.status.world_target.add(pn)
482 # drop reference count now
483 self.status.possible_world = None
484 self.status.all_depends = None
486 def myProgressCallback( self, x, y, f, file_data, from_cache ):
487 # feed the status with new input
488 self.status.handle_bb_data(f, file_data, from_cache)
492 if os.isatty(sys.stdout.fileno()):
493 sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
497 sys.stdout.write("Parsing .bb files, please wait...")
500 sys.stdout.write("done.")
503 def interactiveMode( self ):
504 """Drop off into a shell"""
507 except ImportError, details:
508 bb.fatal("Sorry, shell not available (%s)" % details )
513 def parseConfigurationFile( self, afile ):
515 make.cfg = bb.parse.handle( afile, make.cfg )
517 bb.fatal( "Unable to open %s" % afile )
518 except bb.parse.ParseError:
519 bb.fatal( "Unable to parse %s" % afile )
521 def handleCollections( self, collections ):
522 """Handle collections"""
524 collection_list = collections.split()
525 for c in collection_list:
526 regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, make.cfg, 1)
528 bb.error("BBFILE_PATTERN_%s not defined" % c)
530 priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, make.cfg, 1)
532 bb.error("BBFILE_PRIORITY_%s not defined" % c)
535 cre = re.compile(regex)
537 bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
541 self.status.bbfile_config_priorities.append((cre, pri))
543 bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
546 def cook( self, args ):
547 if not make.options.cmd:
548 make.options.cmd = "build"
550 if make.options.debug:
551 bb.debug_level = make.options.debug
553 make.cfg = bb.data.init()
555 for f in make.options.file:
556 self.parseConfigurationFile( f )
558 self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
560 if not bb.data.getVar("BUILDNAME", make.cfg):
561 bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), make.cfg)
563 buildname = bb.data.getVar("BUILDNAME", make.cfg)
565 if make.options.interactive:
566 self.interactiveMode()
568 if make.options.buildfile is not None:
569 bf = os.path.abspath( make.options.buildfile )
571 bbfile_data = bb.parse.handle(bf, make.cfg)
573 bb.fatal("Unable to open %s" % bf)
575 item = bb.data.getVar('PN', bbfile_data, 1)
576 self.tryBuildPackage( bf, item, bbfile_data )
577 sys.exit( self.stats.show() )
579 # initialise the parsing status now we know we will need deps
580 self.status = BBParsingStatus()
582 ignore = bb.data.getVar("ASSUME_PROVIDED", make.cfg, 1) or ""
583 self.status.ignored_dependencies = Set( ignore.split() )
585 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", make.cfg, 1) )
589 if not pkgs_to_build:
591 pkgs_to_build.extend(args)
592 if not pkgs_to_build:
593 bbpkgs = bb.data.getVar('BBPKGS', make.cfg, 1)
595 pkgs_to_build = bbpkgs.split()
596 if not pkgs_to_build and not make.options.show_versions and not make.options.interactive:
597 print "Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help'"
598 print "for usage information."
601 # Import Psyco if available and not disabled
602 if not make.options.disable_psyco:
607 bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
609 psyco.bind( make.collect_bbfiles )
611 bb.note("You have disabled Psyco. This decreases performance.")
614 bb.debug(1, "collecting .bb files")
615 make.collect_bbfiles( self.myProgressCallback )
616 bb.debug(1, "parsing complete")
619 if make.options.parse_only:
620 print "Requested parsing .bb files only. Exiting."
623 bb.data.update_data( make.cfg )
626 if make.options.show_versions:
629 if 'world' in pkgs_to_build:
630 pkgs_to_build.remove('world')
631 for t in self.status.world_target:
632 pkgs_to_build.append(t)
634 bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, make.cfg))
636 for k in pkgs_to_build:
639 if self.buildProvider( k ) == 0:
642 except bb.build.EventException:
643 bb.error("Build of " + k + " failed")
647 if make.options.abort:
650 bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, make.cfg))
652 sys.exit( self.stats.show() )
654 except KeyboardInterrupt:
655 print "\nNOTE: KeyboardInterrupt - Build not completed."
658 #============================================================================#
660 #============================================================================#
662 if __name__ == "__main__":
664 parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
665 usage = """%prog [options] [package ...]
667 Executes the specified task (default is 'build') for a given set of BitBake files.
668 It expects that BBFILES is defined, which is a space seperated list of files to
669 be executed. BBFILES does support wildcards.
670 Default BBFILES are the .bb files in the current directory.""" )
672 parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
673 action = "store", dest = "buildfile", default = None )
675 parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
676 action = "store_false", dest = "abort", default = True )
678 parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
679 action = "store_true", dest = "force", default = False )
681 parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
682 action = "store_true", dest = "interactive", default = False )
684 parser.add_option( "-c", "--cmd", help = "Specify task to execute",
685 action = "store", dest = "cmd", default = "build" )
687 parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
688 action = "append", dest = "file", default = [] )
690 parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
691 action = "store_true", dest = "verbose", default = False )
693 parser.add_option( "-D", "--debug", help = "Increase the debug level",
694 action = "count", dest="debug", default = 0)
696 parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
697 action = "store_true", dest = "dry_run", default = False )
699 parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
700 action = "store_true", dest = "parse_only", default = False )
702 parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
703 action = "store_true", dest = "disable_psyco", default = False )
705 parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
706 action = "store_true", dest = "show_versions", default = False )
708 options, args = parser.parse_args( sys.argv )
710 make.options = options
712 cooker.cook( args[1:] )