import pickle
print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
-class Cache:
+__cache_version__ = "123"
+class Cache:
+ """
+ BitBake Cache implementation
+ """
def __init__(self, cooker):
+
self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True)
- self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
self.clean = {}
self.depends_cache = {}
self.data = None
self.data_fn = None
if self.cachedir in [None, '']:
+ self.has_cache = False
if cooker.cb is not None:
print "NOTE: Not using a cache. Set CACHE = <directory> to enable."
else:
+ self.has_cache = True
+ self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
+
if cooker.cb is not None:
print "NOTE: Using cache in '%s'" % self.cachedir
try:
except OSError:
bb.mkdirhier( self.cachedir )
- if (self.mtime(self.cachefile)):
- p = pickle.Unpickler( file(self.cachefile,"rb"))
- self.depends_cache = p.load()
+ if self.has_cache and (self.mtime(self.cachefile)):
+ try:
+ p = pickle.Unpickler( file(self.cachefile,"rb"))
+ self.depends_cache, version_data = p.load()
+ if version_data['CACHE_VER'] != __cache_version__:
+ raise ValueError, 'Cache Version Mismatch'
+ if version_data['BITBAKE_VER'] != bb.__version__:
+ raise ValueError, 'Bitbake Version Mismatch'
+ except (ValueError, KeyError):
+ bb.note("Invalid cache found, rebuilding...")
+ self.depends_cache = {}
+
if self.depends_cache:
for fn in self.depends_cache.keys():
self.clean[fn] = ""
self.cacheValidUpdate(fn)
def getVar(self, var, fn, exp = 0):
+ """
+ Gets the value of a variable
+ (similar to getVar in the data class)
+
+ There are two scenarios:
+ 1. We have cached data - serve from depends_cache[fn]
+ 2. We're learning what data to cache - serve from data
+ backend but add a copy of the data to the cache.
+ """
+
if fn in self.clean:
return self.depends_cache[fn][var]
self.depends_cache[fn] = {}
if fn != self.data_fn:
- bb.fatal("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
+ # We're trying to access data in the cache which doesn't exist
+ # yet setData hasn't been called to setup the right access. Very bad.
+ bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
result = bb.data.getVar(var, self.data, exp)
self.depends_cache[fn][var] = result
return result
def setData(self, fn, data):
+ """
+ Called to prime bb_cache ready to learn which variables to cache.
+ Will be followed by calls to self.getVar which aren't cached
+ but can be fulfilled from self.data.
+ """
self.data_fn = fn
self.data = data
self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
def loadDataFull(self, fn, cooker):
-
+ """
+ Return a complete set of data for fn.
+ To do this, we need to parse the file.
+ """
bb_data, skipped = self.load_bbfile(fn, cooker)
- return bb_data, False
+ return bb_data
def loadData(self, fn, cooker):
+ """
+ Load a subset of data for fn.
+ If the cached data is valid we do nothing,
+ To do this, we need to parse the file and set the system
+ to record the variables accessed.
+ Return the cache status and whether the file was skipped when parsed
+ """
if self.cacheValid(fn):
if "SKIPPED" in self.depends_cache[fn]:
return True, True
return False, skipped
def cacheValid(self, fn):
+ """
+ Is the cache valid for fn?
+ Fast version, no timestamps checked.
+ """
# Is cache enabled?
- if self.cachedir in [None, '']:
+ if not self.has_cache:
return False
if fn in self.clean:
return True
return False
def cacheValidUpdate(self, fn):
+ """
+ Is the cache valid for fn?
+ Make thorough (slower) checks including timestamps.
+ """
# Is cache enabled?
- if self.cachedir in [None, '']:
+ if not self.has_cache:
+ return False
+
+ # Check file still exists
+ if self.mtime(fn) == 0:
+ bb.debug(2, "Cache: %s not longer exists" % fn)
+ self.remove(fn)
return False
# File isn't in depends_cache
if not fn in self.depends_cache:
- bb.note("Cache: %s is not cached" % fn)
- if fn in self.clean:
- del self.clean[fn]
+ bb.debug(2, "Cache: %s is not cached" % fn)
+ self.remove(fn)
return False
# Check the file's timestamp
if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
- bb.note("Cache: %s changed" % fn)
- if fn in self.clean:
- del self.clean[fn]
+ bb.debug(2, "Cache: %s changed" % fn)
+ self.remove(fn)
return False
# Check dependencies are still valid
old_mtime = int(old_mtime_s)
new_mtime = bb.parse.cached_mtime(f)
if (new_mtime > old_mtime):
- bb.note("Cache: %s's dependency %s changed" % (fn, f))
- if fn in self.clean:
- del self.clean[fn]
+ bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f))
+ self.remove(fn)
return False
- #bb.note("Depends Cache: %s is clean" % fn)
+ bb.debug(2, "Depends Cache: %s is clean" % fn)
if not fn in self.clean:
self.clean[fn] = ""
return True
def skip(self, fn):
+ """
+ Mark a fn as skipped
+ Called from the parser
+ """
if not fn in self.depends_cache:
self.depends_cache[fn] = {}
self.depends_cache[fn]["SKIPPED"] = "1"
def remove(self, fn):
- bb.note("Removing %s from cache" % fn)
+ """
+ Remove a fn from the cache
+ Called from the parser in error cases
+ """
+ bb.debug(1, "Removing %s from cache" % fn)
if fn in self.depends_cache:
del self.depends_cache[fn]
+ if fn in self.clean:
+ del self.clean[fn]
def sync(self):
+ """
+ Save the cache
+ Called from the parser when complete (or exitting)
+ """
+
+ if not self.has_cache:
+ return
+
+ version_data = {}
+ version_data['CACHE_VER'] = __cache_version__
+ version_data['BITBAKE_VER'] = bb.__version__
+
p = pickle.Pickler(file(self.cachefile, "wb" ), -1 )
- p.dump(self.depends_cache)
+ p.dump([self.depends_cache, version_data])
def mtime(self, cachefile):
try:
return 0
def load_bbfile( self, bbfile , cooker):
- """Load and parse one .bb build file"""
+ """
+ Load and parse one .bb build file
+ Return the data and whether parsing resulted in the file being skipped
+ """
import bb
from bb import utils, data, parse, debug, event, fatal
raise
def init(cooker):
+ """
+ The Objective: Cache the minimum amount of data possible yet get to the
+ stage of building packages (i.e. tryBuild) without reparsing any .bb files.
+
+ To do this, we intercept getVar calls and only cache the variables we see
+ being accessed. We rely on the cache getVar calls being made for all
+ variables bitbake might need to use to reach this stage. For each cached
+ file we need to track:
+
+ * Its mtime
+ * The mtimes of all its dependencies
+ * Whether it caused a parse.SkipPackage exception
+
+ Files causing parsing errors are evicted from the cache.
+
+ """
return Cache(cooker)
projects / vuplus_bitbake / blobdiff