svn commit: r366470 - in head: share/mk tests/sys/capsicum

[Li-Wen Hsu]

Author: lwhsu
Date: Tue Oct  6 06:45:52 2020
New Revision: 366470
URL: https://svnweb.freebsd.org/changeset/base/366470

Log:
  Make capsicum test cases fine-grained
  
  Add a wrapping script to use ATF to run tests written with Googletest
  one by one. This helps locating and tracking the failing case in CI easier.
  
  This is a temporarily solution while Googletest support in Kyua is developing.
  We will revert this once Kyua+Googletest integration is ready.
  
  Reviewed by:	emaste
  Sponsored by:	The FreeBSD Foundation
  Differential Revision:	https://reviews.freebsd.org/D25896

Added:
  head/tests/sys/capsicum/functional.sh   (contents, props changed)
Modified:
  head/share/mk/bsd.test.mk
  head/share/mk/googletest.test.mk
  head/tests/sys/capsicum/Makefile

Modified: head/share/mk/bsd.test.mk
==============================================================================
--- head/share/mk/bsd.test.mk	Tue Oct  6 04:18:42 2020	(r366469)
+++ head/share/mk/bsd.test.mk	Tue Oct  6 06:45:52 2020	(r366470)
@@ -62,10 +62,11 @@ TESTS_ENV+= LD_LIBRARY_PATH=${TESTS_LD_LIBRARY_PATH:tW
 _TESTS=
 
 # Pull in the definitions of all supported test interfaces.
-.include <atf.test.mk>
 .include <googletest.test.mk>
 .include <plain.test.mk>
 .include <tap.test.mk>
+# Include atf last to let other test framework use it
+.include <atf.test.mk>
 
 # Sort the tests alphabetically, so the results are deterministically formed
 # across runs.

Modified: head/share/mk/googletest.test.mk
==============================================================================
--- head/share/mk/googletest.test.mk	Tue Oct  6 04:18:42 2020	(r366469)
+++ head/share/mk/googletest.test.mk	Tue Oct  6 06:45:52 2020	(r366470)
@@ -30,12 +30,18 @@ GTESTS?=
 .include <googletest.test.inc.mk>
 
 PROGS_CXX+= ${GTESTS}
-_TESTS+= ${GTESTS}
 .for _T in ${GTESTS}
 BINDIR.${_T}= ${TESTSDIR}
 CXXFLAGS.${_T}+= ${GTESTS_CXXFLAGS}
 MAN.${_T}?= # empty
 SRCS.${_T}?= ${_T}.cc
+.if !empty(GTESTS_WRAPPER_SH.${_T})
+# A stopgap/workaround to let kyua execute test case one by one
+ATF_TESTS_SH+= ${GTESTS_WRAPPER_SH.${_T}}
+.else
+_TESTS+= ${_T}
 TEST_INTERFACE.${_T}= plain
+.endif
+
 .endfor
 .endif

Modified: head/tests/sys/capsicum/Makefile
==============================================================================
--- head/tests/sys/capsicum/Makefile	Tue Oct  6 04:18:42 2020	(r366469)
+++ head/tests/sys/capsicum/Makefile	Tue Oct  6 06:45:52 2020	(r366470)
@@ -14,6 +14,7 @@ CFLAGS+=	-I${SRCTOP}/tests
 .PATH: ${SRCTOP}/contrib/capsicum-test
 
 GTESTS+=	capsicum-test
+GTESTS_WRAPPER_SH.capsicum-test=	functional
 
 SRCS.capsicum-test+=	\
 	capsicum-test-main.cc \
@@ -50,6 +51,6 @@ BINMODE.mini-me.setuid=	4555
 
 WARNS.capsicum-test=	3
 
-.endif
+.endif # MK_GOOGLETEST
 
 .include <bsd.test.mk>

Added: head/tests/sys/capsicum/functional.sh
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/tests/sys/capsicum/functional.sh	Tue Oct  6 06:45:52 2020	(r366470)
@@ -0,0 +1,68 @@
+#-
+# SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+#
+# Copyright (c) 2020 The FreeBSD Foundation
+#
+# This software was developed by Li-Wen Hsu <lwhsu at FreeBSD.org>
+# under sponsorship from the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+SRCDIR=$(atf_get_srcdir)
+CAPSICUM_TEST_BIN=capsicum-test
+
+check()
+{
+	local tc=${1}
+
+	atf_check -s exit:0 -o match:PASSED  -e ignore \
+		${SRCDIR}/${CAPSICUM_TEST_BIN} --gtest_filter=${tc}
+}
+
+add_testcase()
+{
+	local tc=${1}
+	local tc_escaped word
+
+	tc_escaped=$(echo ${tc} | sed -e 's/\./__/')
+
+	atf_test_case ${tc_escaped}
+	eval "${tc_escaped}_body() { check ${tc}; }"
+	atf_add_test_case ${tc_escaped}
+}
+
+list_tests()
+{
+	${SRCDIR}/${CAPSICUM_TEST_BIN} --gtest_list_tests | awk '
+		/^[^ ]/ { CAT=$0 }
+		/^[ ]/ { print CAT $1}'
+}
+
+atf_init_test_cases()
+{
+	local t
+	for t in `list_tests`; do
+		add_testcase $t
+	done
+}