2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 BitBake 'Event' implementation
7 Caching of bitbake variables before task execution
9 # Copyright (C) 2006 Richard Purdie
11 # but small sections based on code from bin/bitbake:
12 # Copyright (C) 2003, 2004 Chris Larson
13 # Copyright (C) 2003, 2004 Phil Blundell
14 # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
15 # Copyright (C) 2005 Holger Hans Peter Freyther
16 # Copyright (C) 2005 ROAD GmbH
18 This program is free software; you can redistribute it and/or modify it under
19 the terms of the GNU General Public License as published by the Free Software
20 Foundation; either version 2 of the License, or (at your option) any later
23 This program is distributed in the hope that it will be useful, but WITHOUT
24 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
25 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
27 You should have received a copy of the GNU General Public License along with
28 this program; if not, write to the Free Software Foundation, Inc., 59 Temple
29 Place, Suite 330, Boston, MA 02111-1307 USA.
38 import cPickle as pickle
41 print "NOTE: Importing cPickle failed. Falling back to a very slow implementation."
45 BitBake Cache implementation
47 def __init__(self, cooker):
49 self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True)
50 self.cachefile = os.path.join(self.cachedir,"bb_cache.dat")
52 self.depends_cache = {}
56 if self.cachedir in [None, '']:
57 if cooker.cb is not None:
58 print "NOTE: Not using a cache. Set CACHE = <directory> to enable."
60 if cooker.cb is not None:
61 print "NOTE: Using cache in '%s'" % self.cachedir
63 os.stat( self.cachedir )
65 bb.mkdirhier( self.cachedir )
67 if (self.mtime(self.cachefile)):
68 p = pickle.Unpickler( file(self.cachefile,"rb"))
69 self.depends_cache = p.load()
70 if self.depends_cache:
71 for fn in self.depends_cache.keys():
73 self.cacheValidUpdate(fn)
75 def getVar(self, var, fn, exp = 0):
77 Gets the value of a variable
78 (similar to getVar in the data class)
80 There are two scenarios:
81 1. We have cached data - serve from depends_cache[fn]
82 2. We're learning what data to cache - serve from data
83 backend but add a copy of the data to the cache.
87 return self.depends_cache[fn][var]
89 if not fn in self.depends_cache:
90 self.depends_cache[fn] = {}
92 if fn != self.data_fn:
93 # We're trying to access data in the cache which doesn't exist
94 # yet setData hasn't been called to setup the right access. Very bad.
95 bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn))
97 result = bb.data.getVar(var, self.data, exp)
98 self.depends_cache[fn][var] = result
101 def setData(self, fn, data):
103 Called to prime bb_cache ready to learn which variables to cache.
104 Will be followed by calls to self.getVar which aren't cached
105 but can be fulfilled from self.data.
110 # Make sure __depends makes the depends_cache
111 self.getVar("__depends", fn, True)
112 self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn)
114 def loadDataFull(self, fn, cooker):
116 Return a complete set of data for fn.
117 To do this, we need to parse the file.
119 bb_data, skipped = self.load_bbfile(fn, cooker)
120 return bb_data, False
122 def loadData(self, fn, cooker):
124 Load a subset of data for fn.
125 If the cached data is valid we do nothing,
126 To do this, we need to parse the file and set the system
127 to record the variables accessed.
128 Return the cache status and whether the file was skipped when parsed
130 if self.cacheValid(fn):
131 if "SKIPPED" in self.depends_cache[fn]:
135 bb_data, skipped = self.load_bbfile(fn, cooker)
136 self.setData(fn, bb_data)
137 return False, skipped
139 def cacheValid(self, fn):
141 Is the cache valid for fn?
142 Fast version, no timestamps checked.
145 if self.cachedir in [None, '']:
151 def cacheValidUpdate(self, fn):
153 Is the cache valid for fn?
154 Make thorough (slower) checks including timestamps.
157 if self.cachedir in [None, '']:
160 # Check file still exists
161 if self.mtime(fn) == 0:
162 bb.debug(2, "Cache: %s not longer exists" % fn)
165 if fn in self.depends_cache:
166 del self.depends_cache[fn]
169 # File isn't in depends_cache
170 if not fn in self.depends_cache:
171 bb.debug(2, "Cache: %s is not cached" % fn)
176 # Check the file's timestamp
177 if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
178 bb.debug(2, "Cache: %s changed" % fn)
183 # Check dependencies are still valid
184 depends = self.getVar("__depends", fn, True)
186 deps = depends.split(" ")
188 (f,old_mtime_s) = dep.split("@")
189 old_mtime = int(old_mtime_s)
190 new_mtime = bb.parse.cached_mtime(f)
191 if (new_mtime > old_mtime):
192 bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f))
197 bb.debug(2, "Depends Cache: %s is clean" % fn)
198 if not fn in self.clean:
206 Called from the parser
208 if not fn in self.depends_cache:
209 self.depends_cache[fn] = {}
210 self.depends_cache[fn]["SKIPPED"] = "1"
212 def remove(self, fn):
214 Remove a fn from the cache
215 Called from the parser in error cases
217 bb.note("Removing %s from cache" % fn)
218 if fn in self.depends_cache:
219 del self.depends_cache[fn]
224 Called from the parser when complete (or exitting)
226 p = pickle.Pickler(file(self.cachefile, "wb" ), -1 )
227 p.dump(self.depends_cache)
229 def mtime(self, cachefile):
231 return os.stat(cachefile)[8]
235 def load_bbfile( self, bbfile , cooker):
237 Load and parse one .bb build file
238 Return the data and whether parsing resulted in the file being skipped
242 from bb import utils, data, parse, debug, event, fatal
244 topdir = data.getVar('TOPDIR', cooker.configuration.data)
246 topdir = os.path.abspath(os.getcwd())
248 data.setVar('TOPDIR', topdir, cooker.configuration)
249 bbfile = os.path.abspath(bbfile)
250 bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
251 # expand tmpdir to include this topdir
252 data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data)
253 # set topdir to location of .bb file
255 #data.setVar('TOPDIR', topdir, cfg)
257 oldpath = os.path.abspath(os.getcwd())
259 bb_data = data.init_db(cooker.configuration.data)
261 parse.handle(bbfile, bb_data) # read .bb data
263 return bb_data, False
264 except bb.parse.SkipPackage:
273 The Objective: Cache the minimum amount of data possible yet get to the
274 stage of building packages (i.e. tryBuild) without reparsing any .bb files.
276 To do this, we intercept getVar calls and only cache the variables we see
277 being accessed. We rely on the cache getVar calls being made for all
278 variables bitbake might need to use to reach this stage. For each cached
279 file we need to track:
282 * The mtimes of all its dependencies
283 * Whether it caused a parse.SkipPackage exception
285 Files causing parsing errors are evicted from the cache.