svn commit: r221762 - in projects/portbuild: conf qmanager scripts
Florent Thoumie
flz at FreeBSD.org
Wed May 11 08:42:32 UTC 2011
Author: flz
Date: Wed May 11 08:42:31 2011
New Revision: 221762
URL: http://svn.freebsd.org/changeset/base/221762
Log:
portbuild: don't hardcode package suffix.
This isn't yet perfect, but setting pkg_sufx in portbuild.conf will
now be mandatory, and will set PKG_SUFX during package creation.
Modified:
projects/portbuild/conf/common.conf
projects/portbuild/qmanager/packagebuild
projects/portbuild/scripts/buildscript
projects/portbuild/scripts/chopindex
projects/portbuild/scripts/claim-chroot
projects/portbuild/scripts/dopackages
projects/portbuild/scripts/dopackagestats
projects/portbuild/scripts/pdispatch
projects/portbuild/scripts/portbuild
projects/portbuild/scripts/prunefailure
projects/portbuild/scripts/prunepkgs
projects/portbuild/scripts/stats
Modified: projects/portbuild/conf/common.conf
==============================================================================
--- projects/portbuild/conf/common.conf Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/conf/common.conf Wed May 11 08:42:31 2011 (r221762)
@@ -14,7 +14,6 @@
#
LOCALBASE=/usr/local
-PKGSUFFIX=.tbz
ARCHS_REQUIRING_AOUT_COMPAT="i386"
ARCHS_REQUIRING_LINPROCFS="amd64 i386"
Modified: projects/portbuild/qmanager/packagebuild
==============================================================================
--- projects/portbuild/qmanager/packagebuild Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/qmanager/packagebuild Wed May 11 08:42:31 2011 (r221762)
@@ -70,6 +70,8 @@ DEBUG = False
categories = {}
ports = {}
+pkg_sufx = None
+
# When a build fails we requeue it with a lower priority such that it
# will never preempt a phase 1 build but will build when spare
# capacity is available.
@@ -164,6 +166,7 @@ class Port(object):
self.descr = descr
self.maintainer = maintainer
self.www = www
+ self.sufx = pkg_sufx
# Populated later
self.bdep = []
@@ -325,8 +328,8 @@ they still need to know about us as depe
def packagename(self, arch, branch, buildid):
""" Return the path where a package may be found"""
- return "%s/%s/%s/builds/%s/packages/All/%s.tbz" \
- % (pbd, arch, branch, buildid, self.name)
+ return "%s/%s/%s/builds/%s/packages/All/%s%s" \
+ % (pbd, arch, branch, buildid, self.name, self.sufx)
def is_stale(self, arch, branch, buildid):
""" Does a package need to be (re)-built?
@@ -377,8 +380,8 @@ def gettargets(targets):
if i.endswith("-all"):
cat = i.rpartition("-")[0]
plist.update(p.name for p in categories[cat].ports)
- elif i.rstrip(".tbz") in ports:
- plist.update([ports[i.rstrip(".tbz")].name])
+ elif i.rstrip(pkg_sufx) in ports:
+ plist.update([ports[i.rstrip(pkg_sufx)].name])
else:
raise KeyError, i
@@ -425,18 +428,18 @@ class worker(threading.Thread):
try:
runenv={'HOME':"/root",
'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:%s/scripts' + pbc,
- 'FD':" ".join(["%s.tbz" % p.name for p in pkg.fdep]),
- 'ED':" ".join(["%s.tbz" % p.name for p in pkg.edep]),
- 'PD':" ".join(["%s.tbz" % p.name for p in pkg.pdep]),
- 'BD':" ".join(["%s.tbz" % p.name for p in pkg.bdep]),
- 'RD':" ".join(["%s.tbz" % p.name for p in pkg.rdep])}
+ 'FD':" ".join([p.name + p.sufx for p in pkg.fdep]),
+ 'ED':" ".join([p.name + p.sufx for p in pkg.edep]),
+ 'PD':" ".join([p.name + p.sufx for p in pkg.pdep]),
+ 'BD':" ".join([p.name + p.sufx for p in pkg.bdep]),
+ 'RD':" ".join([p.name + p.sufx for p in pkg.rdep])}
for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN", "PORTBUILD_CHECKOUT", "PORTBUILD_DATA" ]:
if var in os.environ:
runenv[var] = os.environ.get(var)
build = subprocess.Popen(
["/bin/sh", "%s/scripts/pdispatch" % pbc,
self.arch, self.branch, self.buildid, self.machine,
- "/tmp/%s/scripts/portbuild" % self.buildid, "%s.tbz" % pkg.name,
+ "/tmp/%s/scripts/portbuild" % self.buildid, pkg.name + pkg.sufx,
pkg.path],
env=runenv,
stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=0)
@@ -506,7 +509,7 @@ class worker(threading.Thread):
wrk.start()
def main(arch, branch, buildid, args):
- global index
+ global index, pkg_sufx
basedir=os.path.realpath(pbd+"/"+arch+"/"+branch+"/builds/"+buildid)
buildid=basedir.split("/")[-1]
@@ -518,6 +521,18 @@ def main(arch, branch, buildid, args):
branchbase = branchbase.split(".")[ 0 ]
indexfile=portsdir+"/INDEX-"+branchbase
+ archconfig = getConfig(pbd, arch, "portbuild.conf")
+ try:
+ branchconfig = getConfig(pbd, "%s/%s" % (arch, branch), "portbuild.conf")
+ archconfig.merge(branchconfig)
+ except:
+ pass
+
+ pkg_sufx = archconfig.get('pkg_sufx')
+ if not pkg_sufx:
+ print "error: pkg_sufx not defined in portbuild.conf"
+ sys.exit(1)
+
print "[MASTER] parseindex..."
index = Index(indexfile)
index.parse()
Modified: projects/portbuild/scripts/buildscript
==============================================================================
--- projects/portbuild/scripts/buildscript Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/buildscript Wed May 11 08:42:31 2011 (r221762)
@@ -51,8 +51,7 @@ add_pkg() {
echo "adding dependencies"
for i in $pkgs; do
echo "pkg_add $i"
- base=$(basename $i .tgz)
- base=$(basename $base .tbz)
+ base=$(basename $i ${pkg_sufx})
if pkg_info -q -e $base; then
echo "skipping $base, already added"
else
@@ -78,8 +77,7 @@ del_pkg() {
unset delpkg nextpkg
recursion=0
for i in $pkgs; do
- base=$(basename $i .tgz)
- base=$(basename $base .tbz)
+ base=$(basename $i ${pkg_sufx})
if [ -s /var/db/pkg/${base}/+REQUIRED_BY ]; then
recursion=1
nextpkg="${base} ${nextpkg}"
@@ -127,6 +125,9 @@ Z=`ident ${dir}/Makefile | grep 'FreeBSD
cd $dir || exit 1
restr=$(make -V RESTRICTED)
+# Inherit from environment set by portbuild.
+pkg_sufx=${PKG_SUFX}
+
# Keep restricted distfiles in a subdirectory for extra protection
# against leakage
if [ ! -z "$restr" ]; then
Modified: projects/portbuild/scripts/chopindex
==============================================================================
--- projects/portbuild/scripts/chopindex Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/chopindex Wed May 11 08:42:31 2011 (r221762)
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import os, sys
+import re
if len(sys.argv) != 3:
print "%s: <index> <pkgdir>" % sys.argv[0]
@@ -12,7 +13,7 @@ pkgdir = sys.argv[2]
if not pkgdir.endswith("/All"):
pkgdir = pkgdir + "/All"
-packages = [pkg for (pkg, ext) in map(os.path.splitext, os.listdir(pkgdir)) if ext == ".tbz"]
+packages = [pkg for (pkg, ext) in map(os.path.splitext, os.listdir(pkgdir)) if re.match('[.]t[bgx]z', ext)]
index=[]
pkgs=[]
Modified: projects/portbuild/scripts/claim-chroot
==============================================================================
--- projects/portbuild/scripts/claim-chroot Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/claim-chroot Wed May 11 08:42:31 2011 (r221762)
@@ -73,7 +73,7 @@ fi
. ${pbd}/${arch}/portbuild.$(hostname)
buildroot=${scratchdir}
-pkgname=${pkgname%.${PKGSUFFIX}}
+pkgname=${pkgname%.${pkg_sufx}}
chrootdir=${buildroot}/${branch}/${buildid}/chroot
Modified: projects/portbuild/scripts/dopackages
==============================================================================
--- projects/portbuild/scripts/dopackages Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/dopackages Wed May 11 08:42:31 2011 (r221762)
@@ -226,7 +226,7 @@ generatemd5 () {
echo "started generating CHECKSUM.MD5 at $(date)"
cd ${builddir}/packages/All
- find . -name '*.tbz' | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5
+ find . -name "*${pkg_sufx}" | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5
echo "ended generating CHECKSUM.MD5 at $(date)"
}
@@ -250,7 +250,7 @@ dobuild() {
echo "ended at $(date)"
end=$(date +%s)
echo "Build took $(date -u -j -r $((end - start)) | awk '{print $4}')"
- echo $(ls -1 ${builddir}/packages/All | grep tbz | wc -l) "packages built"
+ echo $(ls -1 ${builddir}/packages/All | grep ${pkg_sufx} | wc -l) "packages built"
echo $(wc -l ${PORTSDIR}/${INDEXFILE} | awk '{print $1}') "lines in INDEX"
@@ -648,7 +648,7 @@ if [ "$skipstart" = 0 ]; then
rm ${INDEXFILE}.old1 ${INDEXFILE}.1
cd ${PACKAGES}/All
- sed "s,$,${PKGSUFFIX}," ${builddir}/.oldports | xargs rm -f
+ sed "s,$,${pkg_sufx}," ${builddir}/.oldports | xargs rm -f
# XXX MCL takes an unknown period of time.
# XXX MCL return value not checked.
${pbc}/scripts/prunepkgs ${PORTSDIR}/${INDEXFILE} ${PACKAGES}
@@ -731,7 +731,7 @@ fi
#rm -rf ${builddir}/bad
#mkdir -p ${builddir}/bad
#echo "checking packages"
-#for i in *${PKGSUFFIX}; do
+#for i in *${pkg_sufx}; do
# if ! ${PKGZIPCMD} -t $i; then
# echo "Warning: package $i is bad, moving to ${builddir}/bad"
# # the latest link will be left behind...
Modified: projects/portbuild/scripts/dopackagestats
==============================================================================
--- projects/portbuild/scripts/dopackagestats Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/dopackagestats Wed May 11 08:42:31 2011 (r221762)
@@ -123,7 +123,7 @@ write_row () {
n_packages=0
if [ -d $directory/packages/All ]; then
# MCL removed 20090808 -- this takes way too long
- # n_packages=`find $directory/packages/All -name \*.tbz -or -name \*.tgz |wc -l`
+ # n_packages=`find $directory/packages/All -name '*.t[bgx]z' | wc -l`
n_packages=`ls $directory/packages/All | grep -v CHECKSUM.MD5 | wc -l`
have_packages="yes"
fi
Modified: projects/portbuild/scripts/pdispatch
==============================================================================
--- projects/portbuild/scripts/pdispatch Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/pdispatch Wed May 11 08:42:31 2011 (r221762)
@@ -2,7 +2,7 @@
# $FreeBSD: ports/Tools/portbuild/scripts/pdispatch,v 1.40 2011/01/26 10:41:53 linimon Exp $
#
-# pdispatch <arch> <branch> <buildid> <host> <command> <package.tbz> [<args> ...]
+# pdispatch <arch> <branch> <buildid> <host> <command> <package> [<args> ...]
#
# server-side script to dispatch the job to a host via the ptimeout script.
@@ -50,7 +50,7 @@ if [ -z "${scp_cmd}" ]; then
scp_cmd=scp
fi
-pkgname=$(basename $1 ${PKGSUFFIX})
+pkgname=$(basename $1 ${pkg_sufx})
if [ -z "${pkgname}" ]; then
echo "null packagename"
@@ -150,7 +150,7 @@ error=$?
# Pull in the results of the build from the client
${scp_cmd} ${client_user}@${host}:${chroot}/tmp/${pkgname}.log ${builddir}/logs/${pkgname}.log
-(${ssh_cmd} -a -n ${client_user}@${host} test -f ${chroot}/tmp/work.tbz ) && ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/work.tbz ${builddir}/wrkdirs/${pkgname}.tbz
+(${ssh_cmd} -a -n ${client_user}@${host} test -f ${chroot}/tmp/work.tbz ) && ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/work.tbz ${builddir}/wrkdirs/${pkgname}${pkg_sufx}
# XXX Set dirty flag if any of the scp's fail
@@ -164,8 +164,8 @@ if [ "${error}" = 0 ]; then
tar --unlink -C ${builddir} -xvf -
# XXX why is this needed?
- test -f ${builddir}/packages/All/${pkgname}${PKGSUFFIX} && \
- touch ${builddir}/packages/All/${pkgname}${PKGSUFFIX}
+ test -f ${builddir}/packages/All/${pkgname}${pkg_sufx} && \
+ touch ${builddir}/packages/All/${pkgname}${pkg_sufx}
if [ -f ${builddir}/errors/${pkgname}.log ]; then
rm -f ${builddir}/errors/${pkgname}.log
Modified: projects/portbuild/scripts/portbuild
==============================================================================
--- projects/portbuild/scripts/portbuild Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/portbuild Wed May 11 08:42:31 2011 (r221762)
@@ -130,7 +130,7 @@ unset __MAKE_CONF
# set overrides for make.conf
export BACKUP_FTP_SITE=${CLIENT_BACKUP_FTP_SITE}
-pkgname=$(basename $6 ${PKGSUFFIX})
+pkgname=$(basename $6 ${pkg_sufx})
dirname=$7
shift 2
@@ -143,6 +143,7 @@ export DISTDIR=${CLIENT_DISTDIR}
export LOCALBASE=${LOCALBASE}
export PACKAGES=${CLIENT_PACKAGES_LOCATION}
export SRC_BASE=${CLIENT_SRCBASE}
+export PKG_SUFX=${pkg_sufx}
# to catch missing dependencies
#export DEPENDS_TARGET=/usr/bin/true
Modified: projects/portbuild/scripts/prunefailure
==============================================================================
--- projects/portbuild/scripts/prunefailure Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/prunefailure Wed May 11 08:42:31 2011 (r221762)
@@ -74,7 +74,7 @@ while read dir name ver olddate date cou
newver=$(echo $entry | awk '{print $1}')
- if [ -e "${builddir}/packages/All/$newver${PKGSUFFIX}" ]; then
+ if [ -e "${builddir}/packages/All/$newver${pkg_sufx}" ]; then
echo "$newver package exists, should not still be here!"
rm -f ${pbd}/${arch}/${branch}/latest/${dir}
continue
Modified: projects/portbuild/scripts/prunepkgs
==============================================================================
--- projects/portbuild/scripts/prunepkgs Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/prunepkgs Wed May 11 08:42:31 2011 (r221762)
@@ -30,7 +30,7 @@ trap "rm -rf $tmpdir; exit 1" 1 2 3 5 10
# Check for non-package files
-extras=$(find ${pkgdir} -type f \! \( -name INDEX -o -name CHECKSUM.MD5 -o -name \*.tgz -o -name \*.tbz \) )
+extras=$(find ${pkgdir} -type f \! \( -name INDEX -o -name CHECKSUM.MD5 -o -name '*.t[bgx]z' \) )
echo "==> Removing extra files"
echo $extras
if [ "x${extras}" != "x" ]; then
@@ -42,14 +42,14 @@ fi
# Check for files not present in INDEX
echo "==> Removing extra package files"
-find $pkgdir/All -type f -name \*.tgz -o -name \*.tbz | sed -e "s,${pkgdir}/All/,," -e 's,\.tbz$,,' -e 's,\.tgz$,,' |sort > ${tmpdir}/files
+find $pkgdir/All -type f -name '*.t[bgx]z' | sed -e "s,${pkgdir}/All/,," -e 's,\.t[bgx]z$,,' |sort > ${tmpdir}/files
cut -f 1 -d '|' ${index} |sort > ${tmpdir}/packages
extras=$(comm -2 -3 ${tmpdir}/files ${tmpdir}/packages)
echo $extras
if [ "${dummy}" = "0" ]; then
for i in $extras; do
- rm -f $pkgdir/All/${i}.tgz $pkgdir/All/${i}.tbz
+ rm -f $pkgdir/All/${i}.t[bgx]z
done
fi
Modified: projects/portbuild/scripts/stats
==============================================================================
--- projects/portbuild/scripts/stats Wed May 11 08:42:29 2011 (r221761)
+++ projects/portbuild/scripts/stats Wed May 11 08:42:31 2011 (r221762)
@@ -15,7 +15,7 @@ branch=$1
for i in ${SUPPORTED_ARCHS}; do
all=${pbd}/$i/${branch}/builds/latest/packages/All
if [ -d ${all} ]; then
- count=$(find ${all} -name \*.tbz -o -name \*.tgz |wc -l)
+ count=$(find ${all} -name '*.t[bgx]z' | wc -l)
echo -n "$i: ${count} "
fi
done
More information about the svn-src-projects
mailing list