code.vuplus.com Git - vuplus_openembedded/blob - classes/packaged-staging.bbclass

   1 #
   2 # Populate builds using prebuilt packages where possible to speed up builds
   3 # and allow staging to be reconstructed.
   4 #
   5 # To use it add that line to conf/local.conf:
   6 #
   7 # INHERIT += "packaged-staging"
   8 #
   9
  10
  11 #
  12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
  13 #
  14 PSTAGE_PKGVERSION = "${PV}-${PR}"
  15 PSTAGE_PKGARCH    = "${BUILD_SYS}"
  16 PSTAGE_EXTRAPATH  ?= ""
  17 PSTAGE_PKGPATH    = "${DISTRO}${PSTAGE_EXTRAPATH}"
  18 PSTAGE_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
  19 PSTAGE_PKGNAME    = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
  20 PSTAGE_PKG        = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
  21
  22 # multimachine.bbclass will override this but add a default in case we're not using it
  23 MULTIMACH_ARCH ?= "${PACKAGE_ARCH}"
  24
  25 PSTAGE_NATIVEDEPENDS = "\
  26     pkgconfig-native \
  27     autoconf-native \
  28     automake-native \
  29     curl-native \
  30     zlib-native \
  31     libtool-native \
  32     gnu-config-native \
  33     shasum-native \
  34     libtool-native \
  35     automake-native \
  36     update-alternatives-cworth-native \
  37     ipkg-utils-native \
  38     opkg-native \
  39     m4-native \
  40     quilt-native \
  41     stagemanager-native \
  42     "
  43
  44 python () {
  45     import bb
  46     pstage_allowed = True
  47
  48     # These classes encode staging paths into the binary data so can only be
  49     # reused if the path doesn't change/
  50     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
  51         path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
  52         path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
  53         bb.data.setVar('PSTAGE_PKGPATH', path, d)
  54
  55     # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
  56     # staging. There will always be some packages we depend on.
  57     if bb.data.inherits_class('native', d):
  58         pn = bb.data.getVar('PN', d, True)
  59         nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
  60         if pn in nativedeps:
  61             pstage_allowed = False
  62
  63     # Images aren't of interest to us
  64     if bb.data.inherits_class('image', d):
  65         pstage_allowed = False
  66
  67     # Add task dependencies if we're active, otherwise mark packaged staging
  68     # as inactive.
  69     if pstage_allowed:
  70         deps = bb.data.getVarFlag('do_populate_staging', 'depends', d) or ""
  71         deps += " stagemanager-native:do_populate_staging"
  72         bb.data.setVarFlag('do_populate_staging', 'depends', deps, d)
  73
  74         deps = bb.data.getVarFlag('do_prepackaged_stage', 'depends', d) or ""
  75         deps += " opkg-native:do_populate_staging ipkg-utils-native:do_populate_staging"
  76         bb.data.setVarFlag('do_prepackaged_stage', 'depends', deps, d)
  77         bb.data.setVar("PSTAGING_ACTIVE", "1", d)
  78     else:
  79         bb.data.setVar("PSTAGING_ACTIVE", "0", d)
  80 }
  81
  82 DEPLOY_DIR_PSTAGE       = "${DEPLOY_DIR}/pstage"
  83 PSTAGE_MACHCONFIG       = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
  84
  85 PSTAGE_BUILD_CMD        = "${IPKGBUILDCMD}"
  86 PSTAGE_INSTALL_CMD      = "opkg-cl install -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  87 PSTAGE_UPDATE_CMD       = "opkg-cl update -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  88 PSTAGE_REMOVE_CMD       = "opkg-cl remove -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  89 PSTAGE_LIST_CMD         = "opkg-cl list_installed -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  90
  91 PSTAGE_TMPDIR_STAGE     = "${WORKDIR}/staging-pkg"
  92
  93 do_clean_append() {
  94         """
  95         Clear the build and temp directories
  96         """
  97         bb.note("Uninstalling package from staging...")
  98         path = bb.data.getVar("PATH", d, 1)
  99         removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 100         removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 101         ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, removepkg))
 102         if ret != 0:
 103             bb.note("Failure removing staging package")
 104
 105         stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 106         bb.note("Removing staging package %s" % stagepkg)
 107         os.system('rm -rf ' + stagepkg)
 108 }
 109
 110 staging_helper () {
 111         # Assemble appropriate opkg.conf
 112         conffile=${PSTAGE_MACHCONFIG}
 113         mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
 114         if [ ! -e $conffile ]; then
 115                 ipkgarchs="${BUILD_SYS}"
 116                 priority=1
 117                 for arch in $ipkgarchs; do
 118                         echo "arch $arch $priority" >> $conffile
 119                         priority=$(expr $priority + 5)
 120                 done
 121         fi
 122 }
 123
 124 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
 125
 126 python do_prepackaged_stage () {
 127     import os
 128
 129     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
 130         bb.build.make_stamp("do_prepackaged_stage", d)
 131         return
 132
 133     bb.note("Uninstalling any existing package from staging...")
 134     path = bb.data.getVar("PATH", d, 1)
 135     removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 136     removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 137     lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 138     ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, removepkg))
 139     bb.utils.unlockfile(lf)
 140     if ret != 0:
 141         bb.note("Failure attempting to remove staging package")
 142
 143     stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 144
 145     if os.path.exists(stagepkg):
 146         bb.note("Following speedup\n")
 147         path = bb.data.getVar("PATH", d, 1)
 148         installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
 149
 150         bb.build.exec_func("staging_helper", d)
 151
 152         bb.debug(1, "Staging stuff already packaged, using that instead")
 153         lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 154         ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
 155         bb.utils.unlockfile(lf)
 156         if ret != 0:
 157             bb.note("Failure installing prestage package")
 158
 159         #bb.build.make_stamp("do_prepackaged_stage", d)
 160         #for task in bb.data.getVar("PSTAGE_TASKS_COVERED", d, 1).split():
 161         #    bb.build.make_stamp("do_" + task, d)
 162         bb.build.make_stamp("do_stage_package_populated", d)
 163
 164     else:
 165         bb.build.make_stamp("do_prepackaged_stage", d)
 166 }
 167 do_prepackaged_stage[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
 168 do_prepackaged_stage[selfstamp] = "1"
 169 addtask prepackaged_stage before do_fetch
 170
 171 addhandler packagedstage_stampfixing_eventhandler
 172 python packagedstage_stampfixing_eventhandler() {
 173     from bb.event import getName
 174     import os
 175
 176     if getName(e) == "StampUpdate":
 177         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
 178         for (fn, task) in e.targets:
 179             # strip off 'do_'
 180             task = task[3:]
 181             if task in taskscovered:
 182                 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
 183                 if os.path.exists(stamp):
 184                     # We're targetting a task which was skipped with packaged staging
 185                     # so we need to remove the autogenerated stamps.
 186                     for task in taskscovered:
 187                         dir = "%s.do_%s" % (e.stampPrefix[fn], task)
 188                         os.system('rm -f ' + dir)
 189                     os.system('rm -f ' + stamp)
 190
 191     return NotHandled
 192 }
 193
 194 populate_staging_preamble () {
 195         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 196                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
 197                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
 198         fi
 199 }
 200
 201 populate_staging_postamble () {
 202         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 203                 # list the packages currently installed in staging
 204                 ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
 205
 206                 set +e
 207                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
 208                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross
 209                 set -e
 210         fi
 211 }
 212
 213 do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock"
 214 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
 215 python do_populate_staging_prepend() {
 216     bb.build.exec_func("populate_staging_preamble", d)
 217 }
 218
 219 python do_populate_staging_append() {
 220     bb.build.exec_func("populate_staging_postamble", d)
 221 }
 222
 223
 224 staging_packager () {
 225
 226         mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
 227         mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 228
 229         echo "Package: ${PSTAGE_PKGPN}"         >  ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 230         echo "Version: ${PSTAGE_PKGVERSION}"    >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 231         echo "Description: ${DESCRIPTION}"      >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 232         echo "Section: ${SECTION}"              >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 233         echo "Priority: Optional"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 234         echo "Maintainer: ${MAINTAINER}"        >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 235         echo "Architecture: ${PSTAGE_PKGARCH}"  >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 236
 237         # Protect against empty SRC_URI
 238         if [ "${SRC_URI}" != "" ] ; then
 239                 echo "Source: ${SRC_URI}"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 240         else
 241                 echo "Source: OpenEmbedded"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 242         fi
 243
 244         ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 245         ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 246 }
 247
 248 staging_package_installer () {
 249         ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 250 }
 251
 252 python do_package_stage () {
 253     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
 254         return
 255
 256     #
 257     # Handle deploy/ packages
 258     #
 259     bb.build.exec_func("read_subpackage_metadata", d)
 260     stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
 261     tmpdir = bb.data.getVar("TMPDIR", d, True)
 262     packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
 263     if len(packages) > 0:
 264         if bb.data.inherits_class('package_ipk', d):
 265             ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
 266         if bb.data.inherits_class('package_deb', d):
 267             debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
 268
 269         for pkg in packages:
 270             pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
 271             if not pkgname:
 272                 pkgname = pkg
 273             arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
 274             if not arch:
 275                 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
 276             pr = bb.data.getVar('PR_%s' % pkg, d, 1)
 277             if not pr:
 278                 pr = bb.data.getVar('PR', d, 1)
 279             if not packaged(pkg, d):
 280                 continue
 281             if bb.data.inherits_class('package_ipk', d):
 282                 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d)
 283                 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
 284                 if not os.path.exists(srcfile):
 285                     bb.fatal("Package %s does not exist yet it should" % srcfile)
 286                 destpath = ipkpath + "/" + arch + "/"
 287                 bb.mkdirhier(destpath)
 288                 bb.copyfile(srcfile, destpath + srcname)
 289
 290             if bb.data.inherits_class('package_deb', d):
 291                 if arch == 'all':
 292                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d)
 293                 else:
 294                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d)
 295                 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
 296                 if not os.path.exists(srcfile):
 297                     bb.fatal("Package %s does not exist yet it should" % srcfile)
 298                 destpath = debpath + "/" + arch + "/"
 299                 bb.mkdirhier(destpath)
 300                 bb.copyfile(srcfile, destpath + srcname)
 301
 302     #
 303     # Handle stamps/ files
 304     #
 305     stampfn = bb.data.getVar("STAMP", d, True)
 306     destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
 307     bb.mkdirhier(destdir)
 308     # We need to include the package_stage stamp in the staging package so create one
 309     bb.build.make_stamp("do_package_stage", d)
 310     os.system("cp %s.do_* %s/" % (stampfn, destdir))
 311
 312     bb.build.exec_func("staging_helper", d)
 313     bb.build.exec_func("staging_packager", d)
 314     lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 315     bb.build.exec_func("staging_package_installer", d)
 316     bb.utils.unlockfile(lf)
 317 }
 318
 319 #
 320 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
 321 #
 322 addtask package_stage after do_package_write do_populate_staging before do_build