code.vuplus.com Git - vuplus_openembedded/blob - classes/packaged-staging.bbclass

   1 #
   2 # Populate builds using prebuilt packages where possible to speed up builds
   3 # and allow staging to be reconstructed.
   4 #
   5 # To use it add that line to conf/local.conf:
   6 #
   7 # INHERIT += "packaged-staging"
   8 #
   9
  10
  11 #
  12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
  13 #
  14 PSTAGE_PKGVERSION = "${PV}-${PR}"
  15 PSTAGE_PKGARCH    = "${BUILD_SYS}"
  16 PSTAGE_EXTRAPATH  ?= ""
  17 PSTAGE_PKGPATH    = "${DISTRO}${PSTAGE_EXTRAPATH}"
  18 PSTAGE_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
  19 PSTAGE_PKGNAME    = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
  20 PSTAGE_PKG        = "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
  21
  22 # multimachine.bbclass will override this but add a default in case we're not using it
  23 MULTIMACH_ARCH ?= "${PACKAGE_ARCH}"
  24
  25 PSTAGE_NATIVEDEPENDS = "\
  26     pkgconfig-native \
  27     autoconf-native \
  28     automake-native \
  29     curl-native \
  30     zlib-native \
  31     libtool-native \
  32     gnu-config-native \
  33     shasum-native \
  34     libtool-native \
  35     automake-native \
  36     update-alternatives-cworth-native \
  37     ipkg-utils-native \
  38     opkg-native \
  39     m4-native \
  40     quilt-native \
  41     stagemanager-native \
  42     "
  43
  44 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
  45
  46 python () {
  47     import bb
  48     pstage_allowed = True
  49
  50     # These classes encode staging paths into the binary data so can only be
  51     # reused if the path doesn't change/
  52     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
  53         path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
  54         path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
  55         bb.data.setVar('PSTAGE_PKGPATH', path, d)
  56
  57     # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
  58     # staging. There will always be some packages we depend on.
  59     if bb.data.inherits_class('native', d):
  60         pn = bb.data.getVar('PN', d, True)
  61         nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
  62         if pn in nativedeps:
  63             pstage_allowed = False
  64
  65     # Images aren't of interest to us
  66     if bb.data.inherits_class('image', d):
  67         pstage_allowed = False
  68
  69     # Add task dependencies if we're active, otherwise mark packaged staging
  70     # as inactive.
  71     if pstage_allowed:
  72         deps = bb.data.getVarFlag('do_populate_staging', 'depends', d) or ""
  73         deps += " stagemanager-native:do_populate_staging"
  74         bb.data.setVarFlag('do_populate_staging', 'depends', deps, d)
  75
  76         deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
  77         deps += " opkg-native:do_populate_staging ipkg-utils-native:do_populate_staging"
  78         bb.data.setVarFlag('do_setscene', 'depends', deps, d)
  79         bb.data.setVar("PSTAGING_ACTIVE", "1", d)
  80     else:
  81         bb.data.setVar("PSTAGING_ACTIVE", "0", d)
  82 }
  83
  84 DEPLOY_DIR_PSTAGE       = "${DEPLOY_DIR}/pstage"
  85 PSTAGE_MACHCONFIG       = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
  86
  87 PSTAGE_BUILD_CMD        = "${IPKGBUILDCMD}"
  88 PSTAGE_INSTALL_CMD      = "opkg-cl install -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  89 PSTAGE_UPDATE_CMD       = "opkg-cl update -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  90 PSTAGE_REMOVE_CMD       = "opkg-cl remove -force-depends -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  91 PSTAGE_LIST_CMD         = "opkg-cl list_installed -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR}"
  92
  93 PSTAGE_TMPDIR_STAGE     = "${WORKDIR}/staging-pkg"
  94
  95 def pstage_manualclean(srcname, destvarname, d):
  96         import os, bb
  97
  98         src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
  99         dest = bb.data.getVar(destvarname, d, True)
 100
 101         for walkroot, dirs, files in os.walk(src):
 102                 for file in files:
 103                         filepath = os.path.join(walkroot, file).replace(src, dest)
 104                         bb.note("rm %s" % filepath)
 105                         os.system("rm %s" % filepath)
 106
 107 def pstage_cleanpackage(pkgname, d):
 108         import os, bb
 109
 110         path = bb.data.getVar("PATH", d, 1)
 111         list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
 112
 113         bb.note("Checking if staging package installed")
 114         lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 115         ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
 116         if ret == 0:
 117                 bb.note("Yes. Uninstalling package from staging...")
 118                 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 119                 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
 120                 if ret != 0:
 121                         bb.note("Failure removing staging package")
 122         else:
 123                 bb.note("No. Manually removing any installed files")
 124                 pstage_manualclean("staging", "STAGING_DIR", d)
 125                 pstage_manualclean("cross", "CROSS_DIR", d)
 126                 pstage_manualclean("deploy", "DEPLOY_DIR", d)
 127
 128         bb.utils.unlockfile(lf)
 129
 130 do_clean_prepend() {
 131         """
 132         Clear the build and temp directories
 133         """
 134
 135         removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 136         pstage_cleanpackage(removepkg, d)
 137
 138         stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 139         bb.note("Removing staging package %s" % stagepkg)
 140         os.system('rm -rf ' + stagepkg)
 141 }
 142
 143 staging_helper () {
 144         # Assemble appropriate opkg.conf
 145         conffile=${PSTAGE_MACHCONFIG}
 146         mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
 147         if [ ! -e $conffile ]; then
 148                 ipkgarchs="${BUILD_SYS}"
 149                 priority=1
 150                 for arch in $ipkgarchs; do
 151                         echo "arch $arch $priority" >> $conffile
 152                         priority=$(expr $priority + 5)
 153                 done
 154         fi
 155 }
 156
 157 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
 158
 159 SCENEFUNCS += "packagestage_scenefunc"
 160
 161 python packagestage_scenefunc () {
 162     import os
 163
 164     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
 165         return
 166
 167     removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 168     pstage_cleanpackage(removepkg, d)
 169
 170     stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 171
 172     if os.path.exists(stagepkg):
 173         bb.note("Following speedup\n")
 174         path = bb.data.getVar("PATH", d, 1)
 175         installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
 176
 177         bb.build.exec_func("staging_helper", d)
 178
 179         bb.debug(1, "Staging stuff already packaged, using that instead")
 180         lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 181         ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
 182         bb.utils.unlockfile(lf)
 183         if ret != 0:
 184             bb.note("Failure installing prestage package")
 185
 186         bb.build.make_stamp("do_stage_package_populated", d)
 187
 188 }
 189 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
 190 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
 191
 192 addhandler packagedstage_stampfixing_eventhandler
 193 python packagedstage_stampfixing_eventhandler() {
 194     from bb.event import getName
 195     import os
 196
 197     if getName(e) == "StampUpdate":
 198         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
 199         for (fn, task) in e.targets:
 200             # strip off 'do_'
 201             task = task[3:]
 202             if task in taskscovered:
 203                 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
 204                 if os.path.exists(stamp):
 205                     # We're targetting a task which was skipped with packaged staging
 206                     # so we need to remove the autogenerated stamps.
 207                     for task in taskscovered:
 208                         dir = "%s.do_%s" % (e.stampPrefix[fn], task)
 209                         os.system('rm -f ' + dir)
 210                     os.system('rm -f ' + stamp)
 211
 212     return NotHandled
 213 }
 214
 215 populate_staging_preamble () {
 216         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 217                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
 218                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
 219         fi
 220 }
 221
 222 populate_staging_postamble () {
 223         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 224                 # list the packages currently installed in staging
 225                 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
 226
 227                 set +e
 228                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
 229                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross
 230                 set -e
 231         fi
 232 }
 233
 234 do_populate_staging[lockfiles] = "${STAGING_DIR}/staging.lock"
 235 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
 236 python do_populate_staging_prepend() {
 237     bb.build.exec_func("populate_staging_preamble", d)
 238 }
 239
 240 python do_populate_staging_append() {
 241     bb.build.exec_func("populate_staging_postamble", d)
 242 }
 243
 244
 245 staging_packager () {
 246
 247         mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
 248         mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 249
 250         echo "Package: ${PSTAGE_PKGPN}"         >  ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 251         echo "Version: ${PSTAGE_PKGVERSION}"    >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 252         echo "Description: ${DESCRIPTION}"      >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 253         echo "Section: ${SECTION}"              >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 254         echo "Priority: Optional"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 255         echo "Maintainer: ${MAINTAINER}"        >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 256         echo "Architecture: ${PSTAGE_PKGARCH}"  >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 257
 258         # Protect against empty SRC_URI
 259         if [ "${SRC_URI}" != "" ] ; then
 260                 echo "Source: ${SRC_URI}"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 261         else
 262                 echo "Source: OpenEmbedded"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 263         fi
 264
 265         ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 266         ${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 267 }
 268
 269 staging_package_installer () {
 270         #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 271
 272         STATUSFILE=${TMPDIR}${layout_libdir}/opkg/status
 273         echo "Package: ${PSTAGE_PKGPN}"        >> $STATUSFILE
 274         echo "Version: ${PSTAGE_PKGVERSION}"   >> $STATUSFILE
 275         echo "Status: install user installed"  >> $STATUSFILE
 276         echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
 277         echo "" >> $STATUSFILE
 278 }
 279
 280 python do_package_stage () {
 281     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
 282         return
 283
 284     #
 285     # Handle deploy/ packages
 286     #
 287     bb.build.exec_func("read_subpackage_metadata", d)
 288     stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
 289     tmpdir = bb.data.getVar("TMPDIR", d, True)
 290     packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
 291     if len(packages) > 0:
 292         if bb.data.inherits_class('package_ipk', d):
 293             ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
 294         if bb.data.inherits_class('package_deb', d):
 295             debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
 296
 297         for pkg in packages:
 298             pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
 299             if not pkgname:
 300                 pkgname = pkg
 301             arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
 302             if not arch:
 303                 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
 304             pr = bb.data.getVar('PR_%s' % pkg, d, 1)
 305             if not pr:
 306                 pr = bb.data.getVar('PR', d, 1)
 307             if not packaged(pkg, d):
 308                 continue
 309             if bb.data.inherits_class('package_ipk', d):
 310                 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_" + arch + ".ipk", d)
 311                 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
 312                 if os.path.exists(srcfile):
 313                     destpath = ipkpath + "/" + arch + "/"
 314                     bb.mkdirhier(destpath)
 315                     bb.copyfile(srcfile, destpath + srcname)
 316
 317             if bb.data.inherits_class('package_deb', d):
 318                 if arch == 'all':
 319                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_all.deb", d)
 320                 else:
 321                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "_${DPKG_ARCH}.deb", d)
 322                 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
 323                 if os.path.exists(srcfile):
 324                     destpath = debpath + "/" + arch + "/"
 325                     bb.mkdirhier(destpath)
 326                     bb.copyfile(srcfile, destpath + srcname)
 327
 328     #
 329     # Handle stamps/ files
 330     #
 331     stampfn = bb.data.getVar("STAMP", d, True)
 332     destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
 333     bb.mkdirhier(destdir)
 334     # We need to include the package_stage stamp in the staging package so create one
 335     bb.build.make_stamp("do_package_stage", d)
 336     os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
 337
 338     bb.build.exec_func("staging_helper", d)
 339     bb.build.exec_func("staging_packager", d)
 340     lf = bb.utils.lockfile(bb.data.expand("${STAGING_DIR}/staging.lock", d))
 341     bb.build.exec_func("staging_package_installer", d)
 342     bb.utils.unlockfile(lf)
 343 }
 344
 345 #
 346 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
 347 #
 348 addtask package_stage after do_package_write do_populate_staging before do_build
 349
 350 do_package_stage_all () {
 351         :
 352 }
 353 do_package_stage_all[recrdeptask] = "do_package_stage"
 354 addtask package_stage_all after do_package_stage before do_build
 355
 356
 357
 358