2 # Populate builds using prebuilt packages where possible to speed up builds
3 # and allow staging to be reconstructed.
5 # To use it add that line to conf/local.conf:
7 # INHERIT += "packaged-staging"
12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
14 PSTAGE_PKGVERSION = "${PV}-${PR}"
15 PSTAGE_PKGARCH = "${BUILD_SYS}"
16 PSTAGE_EXTRAPATH ?= ""
17 PSTAGE_PKGPATH = "${DISTRO}${PSTAGE_EXTRAPATH}"
18 PSTAGE_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
19 PSTAGE_PKGNAME = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
20 PSTAGE_PKG = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
22 # multimachine.bbclass will override this but add a default in case we're not using it
23 MULTIMACH_ARCH ?= "${PACKAGE_ARCH}"
25 PSTAGE_NATIVEDEPENDS = "\
36 update-alternatives-cworth-native \
44 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
50 # These classes encode staging paths into the binary data so can only be
51 # reused if the path doesn't change/
52 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
53 path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
54 path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
55 bb.data.setVar('PSTAGE_PKGPATH', path, d)
57 # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
58 # staging. There will always be some packages we depend on.
59 if bb.data.inherits_class('native', d):
60 pn = bb.data.getVar('PN', d, True)
61 nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
63 pstage_allowed = False
65 # Images aren't of interest to us
66 if bb.data.inherits_class('image', d):
67 pstage_allowed = False
69 # Add task dependencies if we're active, otherwise mark packaged staging
72 deps = bb.data.getVarFlag('do_populate_staging', 'depends', d) or ""
73 deps += " stagemanager-native:do_populate_staging"
74 bb.data.setVarFlag('do_populate_staging', 'depends', deps, d)
76 deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
77 deps += " opkg-native:do_populate_staging ipkg-utils-native:do_populate_staging"
78 bb.data.setVarFlag('do_setscene', 'depends', deps, d)
79 bb.data.setVar("PSTAGING_ACTIVE", "1", d)
81 bb.data.setVar("PSTAGING_ACTIVE", "0", d)
84 DEPLOY_DIR_PSTAGE = "${DEPLOY_DIR}/pstage"
85 PSTAGE_MACHCONFIG = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
87 PSTAGE_BUILD_CMD = "${IPKGBUILDCMD}"
88 PSTAGE_INSTALL_CMD = "opkg-cl install -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
89 PSTAGE_UPDATE_CMD = "opkg-cl update -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
90 PSTAGE_REMOVE_CMD = "opkg-cl remove -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
91 PSTAGE_LIST_CMD = "opkg-cl list_installed -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
93 PSTAGE_TMPDIR_STAGE = "${WORKDIR}/staging-pkg"
95 def pstage_manualclean(srcname, destvarname, d):
98 src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
99 dest = bb.data.getVar(destvarname, d, True)
101 for walkroot, dirs, files in os.walk(src):
103 filepath = os.path.join(walkroot, file).replace(src, dest)
104 bb.note("rm %s" % filepath)
105 os.system("rm %s" % filepath)
107 def pstage_cleanpackage(pkgname, d):
110 path = bb.data.getVar("PATH", d, 1)
111 list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
113 bb.note("Checking if staging package installed")
114 lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
115 ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
117 bb.note("Yes. Uninstalling package from staging...")
118 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
119 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
121 bb.note("Failure removing staging package")
123 bb.note("No. Manually removing any installed files")
124 pstage_manualclean("staging", "STAGING_DIR", d)
125 pstage_manualclean("cross", "CROSS_DIR", d)
126 pstage_manualclean("deploy", "DEPLOY_DIR", d)
128 bb.utils.unlockfile(lf)
132 Clear the build and temp directories
135 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
136 pstage_cleanpackage(removepkg, d)
138 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
139 bb.note("Removing staging package %s" % stagepkg)
140 os.system('rm -rf ' + stagepkg)
144 # Assemble appropriate opkg.conf
145 conffile=${PSTAGE_MACHCONFIG}
146 mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
147 if [ ! -e $conffile ]; then
148 ipkgarchs="${BUILD_SYS}"
150 for arch in $ipkgarchs; do
151 echo "arch $arch $priority" >> $conffile
152 priority=$(expr $priority + 5)
157 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
159 SCENEFUNCS += "packagestage_scenefunc"
161 python packagestage_scenefunc () {
164 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
167 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
168 pstage_cleanpackage(removepkg, d)
170 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
172 if os.path.exists(stagepkg):
173 bb.note("Following speedup\n")
174 path = bb.data.getVar("PATH", d, 1)
175 installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
177 bb.build.exec_func("staging_helper", d)
179 bb.debug(1, "Staging stuff already packaged, using that instead")
180 lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
181 ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
182 bb.utils.unlockfile(lf)
184 bb.note("Failure installing prestage package")
186 bb.build.make_stamp("do_stage_package_populated", d)
189 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
190 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
192 addhandler packagedstage_stampfixing_eventhandler
193 python packagedstage_stampfixing_eventhandler() {
194 from bb.event import getName
197 if getName(e) == "StampUpdate":
198 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
199 for (fn, task) in e.targets:
202 if task in taskscovered:
203 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
204 if os.path.exists(stamp):
205 # We're targetting a task which was skipped with packaged staging
206 # so we need to remove the autogenerated stamps.
207 for task in taskscovered:
208 dir = "%s.do_%s" % (e.stampPrefix[fn], task)
209 os.system('rm -f ' + dir)
210 os.system('rm -f ' + stamp)
215 populate_staging_preamble () {
216 if [ "$PSTAGING_ACTIVE" = "1" ]; then
217 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
218 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
222 populate_staging_postamble () {
223 if [ "$PSTAGING_ACTIVE" = "1" ]; then
224 # list the packages currently installed in staging
225 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
228 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
229 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross
234 do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock"
235 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
236 python do_populate_staging_prepend() {
237 bb.build.exec_func("populate_staging_preamble", d)
240 python do_populate_staging_append() {
241 bb.build.exec_func("populate_staging_postamble", d)
245 staging_packager () {
247 mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
248 mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
250 echo "Package: ${PSTAGE_PKGPN}" > ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
251 echo "Version: ${PSTAGE_PKGVERSION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
252 echo "Description: ${DESCRIPTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
253 echo "Section: ${SECTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
254 echo "Priority: Optional" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
255 echo "Maintainer: ${MAINTAINER}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
256 echo "Architecture: ${PSTAGE_PKGARCH}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
258 # Protect against empty SRC_URI
259 if [ "${SRC_URI}" != "" ] ; then
260 echo "Source: ${SRC_URI}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
262 echo "Source: OpenEmbedded" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
265 ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
266 ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
269 staging_package_installer () {
270 #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
272 STATUSFILE=${TMPDIR}${layout_libdir}/opkg/status
273 echo "Package: ${PSTAGE_PKGPN}" >> $STATUSFILE
274 echo "Version: ${PSTAGE_PKGVERSION}" >> $STATUSFILE
275 echo "Status: install user installed" >> $STATUSFILE
276 echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
277 echo "" >> $STATUSFILE
280 python do_package_stage () {
281 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
285 # Handle deploy/ packages
287 bb.build.exec_func("read_subpackage_metadata", d)
288 stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
289 tmpdir = bb.data.getVar("TMPDIR", d, True)
290 packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
291 if len(packages) > 0:
292 if bb.data.inherits_class('package_ipk', d):
293 ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
294 if bb.data.inherits_class('package_deb', d):
295 debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
298 pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
301 arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
303 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
304 pr = bb.data.getVar('PR_%s' % pkg, d, 1)
306 pr = bb.data.getVar('PR', d, 1)
307 if not packaged(pkg, d):
309 if bb.data.inherits_class('package_ipk', d):
310 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d)
311 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
312 if os.path.exists(srcfile):
313 destpath = ipkpath + "/" + arch + "/"
314 bb.mkdirhier(destpath)
315 bb.copyfile(srcfile, destpath + srcname)
317 if bb.data.inherits_class('package_deb', d):
319 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d)
321 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d)
322 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
323 if os.path.exists(srcfile):
324 destpath = debpath + "/" + arch + "/"
325 bb.mkdirhier(destpath)
326 bb.copyfile(srcfile, destpath + srcname)
329 # Handle stamps/ files
331 stampfn = bb.data.getVar("STAMP", d, True)
332 destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
333 bb.mkdirhier(destdir)
334 # We need to include the package_stage stamp in the staging package so create one
335 bb.build.make_stamp("do_package_stage", d)
336 os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
338 bb.build.exec_func("staging_helper", d)
339 bb.build.exec_func("staging_packager", d)
340 lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
341 bb.build.exec_func("staging_package_installer", d)
342 bb.utils.unlockfile(lf)
346 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
348 addtask package_stage after do_package_write do_populate_staging before do_build
350 do_package_stage_all () {
353 do_package_stage_all[recrdeptask] = "do_package_stage"
354 addtask package_stage_all after do_package_stage before do_build