svn commit: r332275 - in head/Mk: . Scripts

Thu Oct 31 19:07:39 UTC 2013

Author: mandree
Date: Thu Oct 31 19:07:38 2013
New Revision: 332275
URL: http://svnweb.freebsd.org/changeset/ports/332275

Log:
  bsd.stage.mk: Fewer false positives, much faster, easier maintenance.
  
  Bugfixes:
  
  * @cmd in pkg-plist is now properly handled.
    - It was previously treated the same as though there was a directory
      following it, missing the prefix. (ordering matters in case...esac)
  
    - Due to the cwd=${PREFIX} inside the while read line loop, state
      tracking was broken and every new line assumed that cwd were the
      prefix.
  
  * stage-qa no longer complains about unstripped binaries if debugging is
    active (WITH_DEBUG set && WITHOUT_DEBUG unset).
  
  * The compress-man target uses ECHO_MSG, not ECHO_CMD, to print its
    build step.
  
  Additions:
  
  * The plist parser now understands @unexec rmdir ... || ... lines,
    including those with redirections, so that there are no false
    positives for directories stripped with @unexec rmdir (usually
    happens on stuff installed outside $PREFIX, as in /var).
  
  * The system's root and var mtrees are now also expanded to avoid
    false @dirrm positives if a port installs directories under /var
    and has to create parents in the stagedir that are present in a fully
    installed system (i. e. in the real $PREFIX).
  
  * Given that pkg_create is deemed beyond repair with respect to deleting
    files outside prefix, generate @unexec rmdir statements for such
    directories, rather than @dirrmtry, to sidestep the problem.
  
  Speedups:
  
  * the orphan check now generates sorted lists of staged files,
    and plisted/mtree files, and compares them with comm(1).
    This saves us the overhead of running one grep process per file
    and up to two per directory, and defers the actual list
    processing to a shell utility.  Complexity has not changed,
    but overhead per item has.
  
  * the orphan check now uses one file for directories and one file for
    files mentioned in pkg-plist, so we need not decorate them with "dir "
    and parse them out any longer.
  
  * qa.sh's shebang scanner only looks at the first line of a file,
    sed is told to exit from the 2nd line.
  
  Other Changes:
  
  * Split the makeplist/check-orphans logic out of bsd.stage.mk,
    it is too unwieldy to maintain in make-escaped shell syntax,
    and permits shell tracing with "SH=sh -x" (including quotes!)
  
  * Unify the functions "makeplist" and "check-orphans" in one
    script.  The only difference is that makeplist assumes an empty
    pkg-plist, whereas check-orphans parses it.
  
  * overhaul the mtree extractor, avoiding awk.
  
  Reviewed by:	bapt
  Approved by:	portmgr (bapt)

Added:
  head/Mk/Scripts/check-stagedir.sh   (contents, props changed)
Modified:
  head/Mk/Scripts/qa.sh
  head/Mk/bsd.stage.mk   (contents, props changed)

Added: head/Mk/Scripts/check-stagedir.sh
==============================================================================

--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/Mk/Scripts/check-stagedir.sh	Thu Oct 31 19:07:38 2013	(r332275)
@@ -0,0 +1,113 @@
+#!/bin/sh
+# ports/Mk/Scripts/check-stagedir.sh - called from ports/Mk/bsd.stage.mk
+
+set -e
+export LC_ALL=C
+
+# lists an mtree file's contents, prefixed to dir.
+listmtree() { # mtreefile prefix
+	{
+		echo '#mtree'
+		sed 's/nochange$//;' $1
+	} | tar -tf- | sed "s,^,$2/,;s,^$2/\.$,$2,;s,^$,/,"
+}
+
+# obtain operating mode from command line
+makeplist=0
+case "$1" in
+	orphans)	;;
+	makeplist)	makeplist=1 ;;
+	*) echo >&2 "Usage: $0 {orphans|makelist}" ; exit 1 ;;
+esac
+
+# validate environment
+envfault=0
+for i in STAGEDIR PREFIX LOCALBASE WRKDIR WRKSRC MTREE_FILE \
+    TMPPLIST DATADIR DOCSDIR EXAMPLESDIR
+do
+	if eval test -z "\$$i" ; then
+		echo >&2 "Environment variable $i undefined. Aborting."
+		envfault=1
+    fi
+done
+if [ $envfault -ne 0 ] ; then
+	exit 1
+fi
+
+set -u
+
+#### EXPAND TMPPLIST TO ABSOLUTE PATHS, SPLITTING FILES AND DIRS TO
+#    Use file descriptors 1 and 3 so that the while loop can write
+#    files to the pipe and dirs to a separate file.
+if [ $makeplist = 0 ] ; then
+	# check for orphans
+	cwd=${PREFIX}
+	while read line; do
+		case $line in
+		@dirrm*|'@unexec rmdir'*)
+			line="$(printf %s "$line" \
+			    | sed -Ee 's/\|\|.*//;s|[0-9]*[[:space:]]*>[&]?[[:space:]]*[^[:space:]]+||g' \
+				-e "/^@unexec[[:space:]]+rmdir/s|([^%])%D([^%])|\1${PREFIX}\2|g" \
+				-e '/^@unexec[[:space:]]+rmdir/s|"(.*)"[[:space:]]+|\1|g' \
+				-e 's/@unexec[[:space:]]+rmdir[[:space:]]+//' \
+				-e 's/@dirrm(try)?[[:space:]]+//' \
+				-e 's/[[:space:]]+$//')"
+			case "$line" in
+			/*) echo >&3 "$line" ;;
+			*)  echo >&3 "$cwd/$line" ;;
+			esac
+		;;
+		# order matters here - we must check @cwd first because
+		# otherwise the @cwd* would also match it first, shadowing the
+		# @cwd) line.
+		@cwd|@cd) cwd=${PREFIX} ;;
+		@cwd*|@cd*) set -- $line ; cwd=$2 ;;
+		@*) ;;
+		/*) echo "$line" ;;
+		*)  echo "$cwd/$line" ;;
+		esac
+	done < ${TMPPLIST} 3>${WRKDIR}/.plist-dirs-unsorted | sort >${WRKDIR}/.plist-files
+else
+	# generate plist - pretend the plist had been empty
+	: >${WRKDIR}/.plist-dirs-unsorted
+	: >${WRKDIR}/.plist-files
+fi
+
+### PRODUCE MTREE FILE
+{
+	listmtree /etc/mtree/BSD.root.dist ""
+	#listmtree /etc/mtree/BSD.usr.dist /usr
+	listmtree /etc/mtree/BSD.var.dist /var
+
+	if [ -n "${MTREE_FILE}" ]; then
+		listmtree "${MTREE_FILE}" "${PREFIX}"
+	fi
+
+	a=${PREFIX}
+	while :; do
+		a=${a%/*}
+		[ -z "${a}" ] && break
+		echo ${a}
+	done
+} > ${WRKDIR}/.mtree
+
+### HANDLE FILES
+find ${STAGEDIR} -type f -o -type l | sort | sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files
+comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files \
+	| sed \
+	-e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \
+	-e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \
+	-e "s,${DATADIR},%%DATADIR%%,g" \
+	-e "s,${PREFIX}/,,g" | grep -v "^share/licenses" || [ $? = 1 ]
+
+### HANDLE DIRS
+cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree | sort -u >${WRKDIR}/.traced-dirs
+find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | sort >${WRKDIR}/.staged-dirs
+comm -13 ${WRKDIR}/.traced-dirs ${WRKDIR}/.staged-dirs \
+	| sort -r | sed \
+	-e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \
+	-e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \
+	-e "s,${DATADIR},%%DATADIR%%,g" \
+	-e "s,${PREFIX}/,,g" \
+	-e 's,^, at dirrmtry ,' \
+	-e 's, at dirrmtry \(/.*\), at unexec rmdir >/dev/null 2>\&1 \1 || :,' | grep -v "^@dirrmtry share/licenses" || [ $? = 1 ]

Modified: head/Mk/Scripts/qa.sh
==============================================================================
--- head/Mk/Scripts/qa.sh	Thu Oct 31 19:07:29 2013	(r332274)
+++ head/Mk/Scripts/qa.sh	Thu Oct 31 19:07:38 2013	(r332275)
@@ -18,7 +18,7 @@ err() {
 shebang() {
 	rc=0
 	for f in `find ${STAGEDIR} -type f`; do
-		interp=$(sed -n -e '1s/^#![[:space:]]*\([^[:space:]]*\).*/\1/p' $f)
+		interp=$(sed -n -e '1s/^#![[:space:]]*\([^[:space:]]*\).*/\1/p;2q' $f)
 		case "$interp" in
 		"") ;;
 		/usr/bin/env) ;;
@@ -62,7 +62,8 @@ paths() {
 
 # For now do not raise an error, just warnings
 stripped() {
-	[ -x /usr/bin/file ] || return
+	[ -x /usr/bin/file ] || return # this is fatal
+	[ -n "${STRIP}" ] || return 0
 	for f in `find ${STAGEDIR} -type f`; do
 		output=`/usr/bin/file ${f}`
 		case "${output}" in

Modified: head/Mk/bsd.stage.mk
==============================================================================
--- head/Mk/bsd.stage.mk	Thu Oct 31 19:07:29 2013	(r332274)
+++ head/Mk/bsd.stage.mk	Thu Oct 31 19:07:38 2013	(r332275)
@@ -6,10 +6,22 @@ STAGEDIR?=	${WRKDIR}/stage
 DESTDIRNAME?=	DESTDIR
 
 MAKE_ARGS+=	${DESTDIRNAME}=${STAGEDIR}
-QA_ENV+=		STAGEDIR=${STAGEDIR} PREFIX=${PREFIX} \
+QA_ENV+=	STAGEDIR=${STAGEDIR} \
+		PREFIX=${PREFIX} \
 		LOCALBASE=${LOCALBASE} \
 		USESDESKTOPFILEUTILS=${USES:Mdesktop-file-utils} \
-		USESSHAREDMIMEINFO=${USES:Mshared-mime-info}
+		USESSHAREDMIMEINFO=${USES:Mshared-mime-info} \
+		"STRIP=${STRIP}"
+CO_ENV+=	STAGEDIR=${STAGEDIR} \
+		PREFIX=${PREFIX} \
+		LOCALBASE=${LOCALBASE} \
+		WRKDIR=${WRKDIR} \
+		WRKSRC=${WRKSRC} \
+		MTREE_FILE=${MTREE_FILE} \
+		TMPPLIST=${TMPPLIST} \
+		DATADIR=${DATADIR} \
+		DOCSDIR=${DOCSDIR} \
+		EXAMPLESDIR=${EXAMPLESDIR}
 
 .if !target(stage-dir)
 stage-dir:
@@ -28,8 +40,8 @@ stage-dir:
 # Fixes all dead symlinks left by the previous round
 .if !target(compress-man)
 compress-man:
-	@${ECHO_CMD} "====> Compressing man pages" ; \
-	mdirs= ; \
+	@${ECHO_MSG} "====> Compressing man pages (compress-man)"
+	@mdirs= ; \
 	for dir in ${MANDIRS:S/^/${STAGEDIR}/} ; do \
 		[ -d $$dir ] && mdirs="$$mdirs $$dir" ;\
 	done ; \
@@ -76,90 +88,17 @@ add-plist-info:
 
 .if !target(makeplist)
 makeplist: stage
-	@if [ -n "${MTREE_FILE}" ]; then \
-	{ ${ECHO_CMD} "#mtree"; ${CAT} ${MTREE_FILE}; } | ${TAR} tf - | \
-		awk '{ sub(/^\.$$/, "", $$1); \
-		if ($$1 == "") print "${PREFIX}"; else print "${PREFIX}/"$$1; }' ; \
-	fi > ${WRKDIR}/.mtree
-	@a=${PREFIX}; \
-		while :; do \
-			a=$${a%/*} ; \
-			[ -z "$${a}" ] && break ; \
-			${ECHO_CMD} $${a} >> ${WRKDIR}/.mtree ; \
-		done
-	@${FIND} ${STAGEDIR} -type f -o -type l | ${SORT} | ${SED} -e "s,${STAGEDIR},,g" \
-		-e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \
-		-e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \
-		-e "s,${DATADIR},%%DATADIR%%,g" \
-		-e "s,${PREFIX}/,,g" | ${GREP} -v "^share/licenses" || ${TRUE}
-	@${FIND} ${STAGEDIR} -type d | ${SED} -e "s,${STAGEDIR},,g" \
-		| while read line; do \
-		${GREP} -qw "^$${line}$$" ${WRKDIR}/.mtree || { \
-			[ -n "$${line}" ] && ${ECHO_CMD} "@dirrmtry $${line}"; \
-		}; \
-		done | ${SORT} -r | ${SED} \
-		-e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \
-		-e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \
-		-e "s,${DATADIR},%%DATADIR%%,g" \
-		-e "s,${PREFIX}/,,g" | ${GREP} -v "^@dirrmtry share/licenses" || ${TRUE}
+	@${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh makeplist
 .endif
 
 .if !target(check-orphans)
 check-orphans: stage
-	@while read line; do \
-		cwd=${PREFIX} ; \
-		case $$line in \
-		@dirrm*) \
-			set -- $$line ; \
-			case $$2 in \
-			/*) ${ECHO_CMD} "dir $$2" ;; \
-			*) ${ECHO_CMD} "dir $$cwd/$$2" ;; \
-			esac ; \
-		;; \
-		@cwd) cwd=${PREFIX} ;; \
-		@cwd*) set -- $$line ; \
-			cwd=$$2 ;; \
-		@*) ;; \
-		/*) ${ECHO_CMD} $$line ;; \
-		*) ${ECHO_CMD} $$cwd/$$line ;; \
-		esac ; \
-	done < ${TMPPLIST} > ${WRKDIR}/.expanded-plist
-	@if [ -n "${MTREE_FILE}" ]; then \
-		{ ${ECHO_CMD} "#mtree"; ${CAT} ${MTREE_FILE}; } | ${TAR} tf - | \
-		awk '{ sub(/^\.$$/, "", $$1); \
-		if ($$1 == "") print "${PREFIX}"; else print "${PREFIX}/"$$1; }' ; \
-	fi > ${WRKDIR}/.mtree
-	@a=${PREFIX}; \
-		while :; do \
-			a=$${a%/*} ; \
-			[ -z "$${a}" ] && break ; \
-			${ECHO_CMD} $${a} >> ${WRKDIR}/.mtree ; \
-		done
-	@${FIND} ${STAGEDIR} -type f -o -type l | ${SORT} | ${SED} -e "s,${STAGEDIR},,g" \
-		| while read line; do \
-			${GREP} -qw "^$${line}$$" ${WRKDIR}/.expanded-plist || { \
-			[ -n "$${line}" ] && ${ECHO_CMD} "$${line}" ; \
-		} ; \
-		done | ${SED} \
-		-e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \
-		-e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \
-		-e "s,${DATADIR},%%DATADIR%%,g" \
-		-e "s,${PREFIX}/,,g" | ${GREP} -v "^share/licenses" || ${TRUE}
-	@${FIND} ${STAGEDIR} -type d | ${SED} -e "s,${STAGEDIR},,g" \
-		| while read line; do \
-		${GREP} -qw "^$${line}$$" ${WRKDIR}/.mtree || \
-		${GREP} -qw "dir\ $${line}$$" ${WRKDIR}/.expanded-plist || { \
-			[ -n "$${line}" ] && ${ECHO_CMD} "@dirrmtry $${line}"; \
-		} ; \
-		done | ${SORT} -r | ${SED} \
-		-e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \
-		-e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \
-		-e "s,${DATADIR},%%DATADIR%%,g" \
-		-e "s,${PREFIX}/,,g" | ${GREP} -v "^@dirrmtry share/licenses" || ${TRUE}
+	@${ECHO_MSG} "====> Items missing from pkg-plist (check-orphans)"
+	@${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh orphans
 .endif
 
 .if !target(stage-qa)
 stage-qa:
-	@${ECHO_CMD} "====> Running Q/A tests" ; \
-	${SETENV} ${QA_ENV} ${SH} ${SCRIPTSDIR}/qa.sh
+	@${ECHO_MSG} "====> Running Q/A tests (stage-qa)"
+	@${SETENV} ${QA_ENV} ${SH} ${SCRIPTSDIR}/qa.sh
 .endif
