3 OpenEmbedded 'Fetch' implementations
5 Classes for obtaining upstream sources for the
6 OpenEmbedded (http://openembedded.org) build infrastructure.
8 NOTE that it requires Python 2.x due to its use of static methods.
10 Copyright: (c) 2003 Chris Larson
12 Based on functions from the base oe module, Copyright 2003 Holger Schurig
19 class FetchError(Exception):
20 """Exception raised when a download fails"""
22 class NoMethodError(Exception):
23 """Exception raised when there is no method to obtain a supplied url or set of urls"""
25 class MissingParameterError(Exception):
26 """Exception raised when a fetch method is missing a critical parameter in the url"""
28 #decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81")
29 #('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
31 def uri_replace(uri, uri_find, uri_replace, d = oe.data.init()):
32 # oe.note("uri_replace: operating on %s" % uri)
33 if not uri or not uri_find or not uri_replace:
34 oe.debug(1, "uri_replace: passed an undefined value, not replacing")
35 uri_decoded = list(oe.decodeurl(uri))
36 uri_find_decoded = list(oe.decodeurl(uri_find))
37 uri_replace_decoded = list(oe.decodeurl(uri_replace))
38 result_decoded = ['','','','','',{}]
39 for i in uri_find_decoded:
40 loc = uri_find_decoded.index(i)
41 result_decoded[loc] = uri_decoded[loc]
43 if type(i) == types.StringType:
45 if (re.match(i, uri_decoded[loc])):
46 result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
47 if uri_find_decoded.index(i) == 2:
49 localfn = oe.fetch.localpath(uri, d)
51 result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(oe.fetch.localpath(uri, d))
52 # oe.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
54 # oe.note("uri_replace: no match")
58 # FIXME: apply replacements against options
59 return oe.encodeurl(result_decoded)
63 def init(urls = [], d = oe.data.init()):
73 def go(d = oe.data.init()):
80 """Return a list of the local filenames, assuming successful fetch"""
84 local.append(m.localpath(u, d))
87 def localpath(url, d = oe.data.init()):
89 if m.supports(url, d):
90 return m.localpath(url, d)
94 """Base class for 'fetch'ing data"""
96 def __init__(self, urls = []):
99 if self.supports(oe.decodeurl(url), d) is 1:
100 self.urls.append(url)
102 def supports(url, d):
103 """Check to see if this fetch class supports a given url.
104 Expects supplied url in list form, as outputted by oe.decodeurl().
107 supports = staticmethod(supports)
109 def localpath(url, d = oe.data.init()):
110 """Return the local filename of a given url assuming a successful fetch.
113 localpath = staticmethod(localpath)
115 def setUrls(self, urls):
121 urls = property(getUrls, setUrls, None, "Urls property")
123 def setData(self, data):
129 data = property(getData, setData, None, "Data property")
131 def go(self, urls = []):
133 raise NoMethodError("Missing implementation for url")
136 """Class to fetch urls via 'wget'"""
137 def supports(url, d):
138 """Check to see if a given url can be fetched using wget.
139 Expects supplied url in list form, as outputted by oe.decodeurl().
141 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
142 return type in ['http','https','ftp']
143 supports = staticmethod(supports)
145 def localpath(url, d):
146 # strip off parameters
147 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
148 if "localpath" in parm:
149 # if user overrides local path, use it.
150 return parm["localpath"]
151 url = oe.encodeurl([type, host, path, user, pswd, {}])
152 return os.path.join(oe.data.getVar("DL_DIR", d), os.path.basename(url))
153 localpath = staticmethod(localpath)
155 def go(self, d = oe.data.init(), urls = []):
157 def fetch_uri(uri, basename, dl, md5, d):
158 if os.path.exists(dl):
159 # file exists, but we didnt complete it.. trying again..
160 fetchcmd = oe.data.getVar("RESUMECOMMAND", d, 1)
162 fetchcmd = oe.data.getVar("FETCHCOMMAND", d, 1)
164 oe.note("fetch " + uri)
165 fetchcmd = fetchcmd.replace("${URI}", uri)
166 fetchcmd = fetchcmd.replace("${FILE}", basename)
167 oe.debug(2, "executing " + fetchcmd)
168 ret = os.system(fetchcmd)
172 # supposedly complete.. write out md5sum
173 if oe.which(oe.data.getVar('PATH', d, 1), 'md5sum'):
174 md5pipe = os.popen('md5sum ' + dl)
175 md5data = md5pipe.readline().split()[0]
177 md5out = file(md5, 'w')
178 md5out.write(md5data)
181 md5out = file(md5, 'w')
189 from copy import deepcopy
190 localdata = deepcopy(d)
191 oe.data.setVar('OVERRIDES', "wget:" + oe.data.getVar('OVERRIDES', localdata), localdata)
192 oe.data.update_data(localdata)
195 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(uri, localdata))
196 basename = os.path.basename(path)
197 dl = self.localpath(uri, d)
198 dl = oe.data.expand(dl, localdata)
201 if os.path.exists(md5):
202 # complete, nothing to see here..
205 if fetch_uri(uri, basename, dl, md5, localdata):
210 mirrors = [string.split(i) for i in string.split(oe.data.getVar('MIRRORS', localdata, 1), '\n') if i]
212 for (find, replace) in mirrors:
213 newuri = uri_replace(uri, find, replace)
215 if fetch_uri(newuri, basename, dl, md5, localdata):
220 raise FetchError(uri)
225 methods.append(Wget())
228 """Class to fetch a module or modules from cvs repositories"""
229 def supports(url, d):
230 """Check to see if a given url can be fetched with cvs.
231 Expects supplied url in list form, as outputted by oe.decodeurl().
233 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
234 return type in ['cvs', 'pserver']
235 supports = staticmethod(supports)
237 def localpath(url, d):
238 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
239 if "localpath" in parm:
240 # if user overrides local path, use it.
241 return parm["localpath"]
243 if not "module" in parm:
244 raise MissingParameterError("cvs method needs a 'module' parameter")
246 module = parm["module"]
256 return os.path.join(oe.data.getVar("DL_DIR", d, 1),oe.data.expand('%s_%s_%s_%s.tar.gz' % (string.replace(module, '/', '.'), host, tag, date), d))
257 localpath = staticmethod(localpath)
259 def go(self, d = oe.data.init(), urls = []):
264 from copy import deepcopy
265 localdata = deepcopy(d)
266 oe.data.setVar('OVERRIDES', "cvs:%s" % oe.data.getVar('OVERRIDES', localdata), localdata)
267 oe.data.update_data(localdata)
270 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(loc, localdata))
271 if not "module" in parm:
272 raise MissingParameterError("cvs method needs a 'module' parameter")
274 module = parm["module"]
276 dlfile = self.localpath(loc, localdata)
277 dldir = oe.data.getVar('DL_DIR', localdata, 1)
278 # if local path contains the cvs
279 # module, consider the dir above it to be the
281 # pos = dlfile.find(module)
283 # dldir = dlfile[:pos]
285 # dldir = os.path.dirname(dlfile)
300 method = parm["method"]
304 tarfn = oe.data.expand('%s_%s_%s_%s.tar.gz' % (string.replace(module, '/', '.'), host, tag, date), localdata)
305 oe.data.setVar('TARFILES', dlfile, localdata)
306 oe.data.setVar('TARFN', tarfn, localdata)
308 if os.access(os.path.join(dldir, tarfn), os.R_OK):
309 oe.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
313 options.append("-D %s" % date)
315 options.append("-r %s" % tag)
317 olddir = os.path.abspath(os.getcwd())
318 os.chdir(oe.data.expand(dldir, localdata))
321 cvsroot = ":" + method + ":" + user
323 cvsroot += ":" + pswd
324 cvsroot += "@" + host + ":" + path
326 oe.data.setVar('CVSROOT', cvsroot, localdata)
327 oe.data.setVar('CVSCOOPTS', string.join(options), localdata)
328 oe.data.setVar('CVSMODULE', module, localdata)
329 cvscmd = oe.data.getVar('FETCHCOMMAND', localdata, 1)
331 # create temp directory
332 oe.debug(2, "Fetch: creating temporary directory")
333 oe.mkdirhier(oe.data.expand('${WORKDIR}', localdata))
334 oe.data.setVar('TMPBASE', oe.data.expand('${WORKDIR}/oecvs.XXXXXX', localdata), localdata)
335 tmppipe = os.popen(oe.data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
336 tmpfile = tmppipe.readline().strip()
338 oe.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
339 raise FetchError(module)
341 # check out sources there
343 oe.note("Fetch " + loc)
344 oe.debug(1, "Running %s" % cvscmd)
345 myret = os.system(cvscmd)
351 raise FetchError(module)
353 os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
354 # tar them up to a defined filename
355 myret = os.system("tar -czvf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
362 os.system('rm -rf %s' % tmpfile)
366 methods.append(Cvs())
369 def supports(url, d):
370 """Check to see if a given url can be fetched via bitkeeper.
371 Expects supplied url in list form, as outputted by oe.decodeurl().
373 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
374 return type in ['bk']
375 supports = staticmethod(supports)
380 def supports(url, d):
381 """Check to see if a given url can be fetched in the local filesystem.
382 Expects supplied url in list form, as outputted by oe.decodeurl().
384 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
385 return type in ['file','patch']
386 supports = staticmethod(supports)
388 def localpath(url, d):
389 """Return the local filename of a given url assuming a successful fetch.
391 return url.split("://")[1]
392 localpath = staticmethod(localpath)
394 def go(self, urls = []):
395 """Fetch urls (no-op for Local method)"""
396 # no need to fetch local files, we'll deal with them in place.
399 methods.append(Local())