2 # ex:ts=4:sw=4:sts=4:et
3 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
5 OpenEmbedded 'Fetch' implementations
7 Classes for obtaining upstream sources for the
8 OpenEmbedded (http://openembedded.org) build infrastructure.
10 NOTE that it requires Python 2.x due to its use of static methods.
12 Copyright: (c) 2003 Chris Larson
14 Based on functions from the base oe module, Copyright 2003 Holger Schurig
21 class FetchError(Exception):
22 """Exception raised when a download fails"""
24 class NoMethodError(Exception):
25 """Exception raised when there is no method to obtain a supplied url or set of urls"""
27 class MissingParameterError(Exception):
28 """Exception raised when a fetch method is missing a critical parameter in the url"""
30 #decodeurl("cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;module=familiar/dist/ipkg;tag=V0-99-81")
31 #('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'})
33 def uri_replace(uri, uri_find, uri_replace, d = oe.data.init()):
34 # oe.note("uri_replace: operating on %s" % uri)
35 if not uri or not uri_find or not uri_replace:
36 oe.debug(1, "uri_replace: passed an undefined value, not replacing")
37 uri_decoded = list(oe.decodeurl(uri))
38 uri_find_decoded = list(oe.decodeurl(uri_find))
39 uri_replace_decoded = list(oe.decodeurl(uri_replace))
40 result_decoded = ['','','','','',{}]
41 for i in uri_find_decoded:
42 loc = uri_find_decoded.index(i)
43 result_decoded[loc] = uri_decoded[loc]
45 if type(i) == types.StringType:
47 if (re.match(i, uri_decoded[loc])):
48 result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
49 if uri_find_decoded.index(i) == 2:
51 localfn = oe.fetch.localpath(uri, d)
53 result_decoded[loc] = os.path.dirname(result_decoded[loc]) + "/" + os.path.basename(oe.fetch.localpath(uri, d))
54 # oe.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
56 # oe.note("uri_replace: no match")
60 # FIXME: apply replacements against options
61 return oe.encodeurl(result_decoded)
65 def init(urls = [], d = oe.data.init()):
75 def go(d = oe.data.init()):
82 """Return a list of the local filenames, assuming successful fetch"""
86 local.append(m.localpath(u, d))
89 def localpath(url, d = oe.data.init()):
91 if m.supports(url, d):
92 return m.localpath(url, d)
96 """Base class for 'fetch'ing data"""
98 def __init__(self, urls = []):
101 if self.supports(oe.decodeurl(url), d) is 1:
102 self.urls.append(url)
104 def supports(url, d):
105 """Check to see if this fetch class supports a given url.
106 Expects supplied url in list form, as outputted by oe.decodeurl().
109 supports = staticmethod(supports)
111 def localpath(url, d = oe.data.init()):
112 """Return the local filename of a given url assuming a successful fetch.
115 localpath = staticmethod(localpath)
117 def setUrls(self, urls):
123 urls = property(getUrls, setUrls, None, "Urls property")
125 def setData(self, data):
131 data = property(getData, setData, None, "Data property")
133 def go(self, urls = []):
135 raise NoMethodError("Missing implementation for url")
138 """Class to fetch urls via 'wget'"""
139 def supports(url, d):
140 """Check to see if a given url can be fetched using wget.
141 Expects supplied url in list form, as outputted by oe.decodeurl().
143 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
144 return type in ['http','https','ftp']
145 supports = staticmethod(supports)
147 def localpath(url, d):
148 # strip off parameters
149 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
150 if "localpath" in parm:
151 # if user overrides local path, use it.
152 return parm["localpath"]
153 url = oe.encodeurl([type, host, path, user, pswd, {}])
154 return os.path.join(oe.data.getVar("DL_DIR", d), os.path.basename(url))
155 localpath = staticmethod(localpath)
157 def go(self, d = oe.data.init(), urls = []):
159 def fetch_uri(uri, basename, dl, md5, d):
160 if os.path.exists(dl):
161 # file exists, but we didnt complete it.. trying again..
162 fetchcmd = oe.data.getVar("RESUMECOMMAND", d, 1)
164 fetchcmd = oe.data.getVar("FETCHCOMMAND", d, 1)
166 oe.note("fetch " + uri)
167 fetchcmd = fetchcmd.replace("${URI}", uri)
168 fetchcmd = fetchcmd.replace("${FILE}", basename)
169 oe.debug(2, "executing " + fetchcmd)
170 ret = os.system(fetchcmd)
174 # supposedly complete.. write out md5sum
175 if oe.which(oe.data.getVar('PATH', d), 'md5sum'):
177 md5pipe = os.popen('md5sum ' + dl)
178 md5data = (md5pipe.readline().split() or [ "" ])[0]
182 md5out = file(md5, 'w')
183 md5out.write(md5data)
186 md5out = file(md5, 'w')
194 from copy import deepcopy
195 localdata = deepcopy(d)
196 oe.data.setVar('OVERRIDES', "wget:" + oe.data.getVar('OVERRIDES', localdata), localdata)
197 oe.data.update_data(localdata)
201 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(uri, localdata))
202 basename = os.path.basename(path)
203 dl = self.localpath(uri, d)
204 dl = oe.data.expand(dl, localdata)
207 if os.path.exists(md5):
208 # complete, nothing to see here..
211 premirrors = [ i.split() for i in (oe.data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
212 for (find, replace) in premirrors:
213 newuri = uri_replace(uri, find, replace)
215 if fetch_uri(newuri, basename, dl, md5, localdata):
222 if fetch_uri(uri, basename, dl, md5, localdata):
226 mirrors = [ i.split() for i in (oe.data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
227 for (find, replace) in mirrors:
228 newuri = uri_replace(uri, find, replace)
230 if fetch_uri(newuri, basename, dl, md5, localdata):
235 raise FetchError(uri)
240 methods.append(Wget())
243 """Class to fetch a module or modules from cvs repositories"""
244 def supports(url, d):
245 """Check to see if a given url can be fetched with cvs.
246 Expects supplied url in list form, as outputted by oe.decodeurl().
248 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
249 return type in ['cvs', 'pserver']
250 supports = staticmethod(supports)
252 def localpath(url, d):
253 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
254 if "localpath" in parm:
255 # if user overrides local path, use it.
256 return parm["localpath"]
258 if not "module" in parm:
259 raise MissingParameterError("cvs method needs a 'module' parameter")
261 module = parm["module"]
270 date = oe.data.getVar("CVSDATE", d, 1) or oe.data.getVar("DATE", d, 1)
274 return os.path.join(oe.data.getVar("DL_DIR", d, 1),oe.data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d))
275 localpath = staticmethod(localpath)
277 def go(self, d = oe.data.init(), urls = []):
282 from copy import deepcopy
283 localdata = deepcopy(d)
284 oe.data.setVar('OVERRIDES', "cvs:%s" % oe.data.getVar('OVERRIDES', localdata), localdata)
285 oe.data.update_data(localdata)
288 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(loc, localdata))
289 if not "module" in parm:
290 raise MissingParameterError("cvs method needs a 'module' parameter")
292 module = parm["module"]
294 dlfile = self.localpath(loc, localdata)
295 dldir = oe.data.getVar('DL_DIR', localdata, 1)
296 # if local path contains the cvs
297 # module, consider the dir above it to be the
299 # pos = dlfile.find(module)
301 # dldir = dlfile[:pos]
303 # dldir = os.path.dirname(dlfile)
316 date = oe.data.getVar("CVSDATE", d, 1) or oe.data.getVar("DATE", d, 1)
321 method = parm["method"]
325 if "localdir" in parm:
326 localdir = parm["localdir"]
328 localdir = os.path.basename(module)
333 cvs_rsh = parm["rsh"]
335 tarfn = oe.data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata)
336 oe.data.setVar('TARFILES', dlfile, localdata)
337 oe.data.setVar('TARFN', tarfn, localdata)
339 dl = os.path.join(dldir, tarfn)
340 if os.access(dl, os.R_OK):
341 oe.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
344 pn = oe.data.getVar('PN', d, 1)
345 cvs_tarball_stash = None
347 cvs_tarball_stash = oe.data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
348 if cvs_tarball_stash == None:
349 cvs_tarball_stash = oe.data.getVar('CVS_TARBALL_STASH', d, 1)
350 if cvs_tarball_stash:
351 fetchcmd = oe.data.getVar("FETCHCOMMAND_wget", d, 1)
352 uri = cvs_tarball_stash + tarfn
353 oe.note("fetch " + uri)
354 fetchcmd = fetchcmd.replace("${URI}", uri)
355 ret = os.system(fetchcmd)
357 oe.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
361 options.append("-D %s" % date)
363 options.append("-r %s" % tag)
365 olddir = os.path.abspath(os.getcwd())
366 os.chdir(oe.data.expand(dldir, localdata))
372 cvsroot = ":" + method + ":" + user
374 cvsroot += ":" + pswd
375 cvsroot += "@" + host + ":" + path
377 oe.data.setVar('CVSROOT', cvsroot, localdata)
378 oe.data.setVar('CVSCOOPTS', " ".join(options), localdata)
379 oe.data.setVar('CVSMODULE', module, localdata)
380 cvscmd = oe.data.getVar('FETCHCOMMAND', localdata, 1)
381 cvsupdatecmd = oe.data.getVar('UPDATECOMMAND', localdata, 1)
384 cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
386 # create module directory
387 oe.debug(2, "Fetch: checking for module directory")
388 pkg=oe.data.expand('${PN}', d)
389 pkgdir=os.path.join(oe.data.expand('${CVSDIR}', localdata), pkg)
390 moddir=os.path.join(pkgdir,module)
391 if os.access(os.path.join(moddir,'CVS'), os.R_OK):
392 oe.note("Update " + loc)
393 # update sources there
395 myret = os.system(cvsupdatecmd)
397 oe.note("Fetch " + loc)
398 # check out sources there
401 oe.debug(1, "Running %s" % cvscmd)
402 myret = os.system(cvscmd)
409 raise FetchError(module)
413 # tar them up to a defined filename
414 myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), localdir))
423 methods.append(Cvs())
426 def supports(url, d):
427 """Check to see if a given url can be fetched via bitkeeper.
428 Expects supplied url in list form, as outputted by oe.decodeurl().
430 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
431 return type in ['bk']
432 supports = staticmethod(supports)
437 def supports(url, d):
438 """Check to see if a given url can be fetched in the local filesystem.
439 Expects supplied url in list form, as outputted by oe.decodeurl().
441 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
442 return type in ['file','patch']
443 supports = staticmethod(supports)
445 def localpath(url, d):
446 """Return the local filename of a given url assuming a successful fetch.
448 path = url.split("://")[1]
451 filespath = oe.data.getVar('FILESPATH', d, 1)
453 newpath = oe.which(filespath, path)
455 filesdir = oe.data.getVar('FILESDIR', d, 1)
457 newpath = os.path.join(filesdir, path)
459 localpath = staticmethod(localpath)
461 def go(self, urls = []):
462 """Fetch urls (no-op for Local method)"""
463 # no need to fetch local files, we'll deal with them in place.
466 methods.append(Local())
469 """Class to fetch a module or modules from svn repositories"""
470 def supports(url, d):
471 """Check to see if a given url can be fetched with svn.
472 Expects supplied url in list form, as outputted by oe.decodeurl().
474 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
475 return type in ['svn']
476 supports = staticmethod(supports)
478 def localpath(url, d):
479 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(url, d))
480 if "localpath" in parm:
481 # if user overrides local path, use it.
482 return parm["localpath"]
484 if not "module" in parm:
485 raise MissingParameterError("svn method needs a 'module' parameter")
487 module = parm["module"]
495 if not tag or tag == "HEAD":
496 date = oe.data.getVar("CVSDATE", d, 1) or oe.data.getVar("DATE", d, 1)
500 return os.path.join(oe.data.getVar("DL_DIR", d, 1),oe.data.expand('%s_%s_%s_%s.tar.gz' % ( module.replace('/', '.'), host, tag, date), d))
501 localpath = staticmethod(localpath)
503 def go(self, d = oe.data.init(), urls = []):
508 from copy import deepcopy
509 localdata = deepcopy(d)
510 oe.data.setVar('OVERRIDES', "svn:%s" % oe.data.getVar('OVERRIDES', localdata), localdata)
511 oe.data.update_data(localdata)
514 (type, host, path, user, pswd, parm) = oe.decodeurl(oe.data.expand(loc, localdata))
515 if not "module" in parm:
516 raise MissingParameterError("svn method needs a 'module' parameter")
518 module = parm["module"]
520 dlfile = self.localpath(loc, localdata)
521 dldir = oe.data.getVar('DL_DIR', localdata, 1)
522 # if local path contains the svn
523 # module, consider the dir above it to be the
525 # pos = dlfile.find(module)
527 # dldir = dlfile[:pos]
529 # dldir = os.path.dirname(dlfile)
541 if not tag or tag == "HEAD":
542 date = oe.data.getVar("CVSDATE", d, 1) or oe.data.getVar("DATE", d, 1)
547 method = parm["method"]
554 svn_rsh = parm["rsh"]
556 tarfn = oe.data.expand('%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, tag, date), localdata)
557 oe.data.setVar('TARFILES', dlfile, localdata)
558 oe.data.setVar('TARFN', tarfn, localdata)
560 dl = os.path.join(dldir, tarfn)
561 if os.access(dl, os.R_OK):
562 oe.debug(1, "%s already exists, skipping svn checkout." % tarfn)
565 svn_tarball_stash = oe.data.getVar('CVS_TARBALL_STASH', d, 1)
566 if svn_tarball_stash:
567 fetchcmd = oe.data.getVar("FETCHCOMMAND_wget", d, 1)
568 uri = svn_tarball_stash + tarfn
569 oe.note("fetch " + uri)
570 fetchcmd = fetchcmd.replace("${URI}", uri)
571 ret = os.system(fetchcmd)
573 oe.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
577 options.append("-D %s" % date)
579 options.append("-r %s" % tag)
581 olddir = os.path.abspath(os.getcwd())
582 os.chdir(oe.data.expand(dldir, localdata))
585 # svnroot = ":" + method + ":" + user
587 # svnroot += ":" + pswd
588 svnroot = host + path
590 oe.data.setVar('SVNROOT', svnroot, localdata)
591 oe.data.setVar('SVNCOOPTS', " ".join(options), localdata)
592 oe.data.setVar('SVNMODULE', module, localdata)
593 svncmd = oe.data.getVar('FETCHCOMMAND', localdata, 1)
594 svncmd = "svn co http://%s/%s" % (svnroot, module)
597 svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
599 # create temp directory
600 oe.debug(2, "Fetch: creating temporary directory")
601 oe.mkdirhier(oe.data.expand('${WORKDIR}', localdata))
602 oe.data.setVar('TMPBASE', oe.data.expand('${WORKDIR}/oesvn.XXXXXX', localdata), localdata)
603 tmppipe = os.popen(oe.data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
604 tmpfile = tmppipe.readline().strip()
606 oe.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
607 raise FetchError(module)
609 # check out sources there
611 oe.note("Fetch " + loc)
613 oe.debug(1, "Running %s" % svncmd)
614 myret = os.system(svncmd)
620 raise FetchError(module)
622 os.chdir(os.path.join(tmpfile, os.path.dirname(module)))
623 # tar them up to a defined filename
624 myret = os.system("tar -czf %s %s" % (os.path.join(dldir,tarfn), os.path.basename(module)))
631 os.system('rm -rf %s' % tmpfile)
635 methods.append(Svn())