git: c08f5ad160bf - main - CI: Add full test support
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 30 Apr 2025 08:18:17 UTC
The branch main has been updated by bofh:
URL: https://cgit.FreeBSD.org/src/commit/?id=c08f5ad160bf7c844677e1960b1a882d492006f8
commit c08f5ad160bf7c844677e1960b1a882d492006f8
Author: Muhammad Moinur Rahman <bofh@FreeBSD.org>
AuthorDate: 2025-04-30 08:14:31 +0000
Commit: Muhammad Moinur Rahman <bofh@FreeBSD.org>
CommitDate: 2025-04-30 08:18:08 +0000
CI: Add full test support
The patch adds support for running full tests in the local CI
Environment. New features added:
- New target `ci-full` which runs the full test. This is also the
default now
- Renamed the previous target `ci-smokeit` to `ci-smoke`
- Unlike previous if the available memory is more than 16G a default of
8G will be used
- Removed some unnecessary debug messages
- Added `dummybuf` kernel module to the list of modules to be loaded on
the VM
The features that can be tested:
`make TARGET=<TARGET> TARGET_ARCH=<TARGET_ARCH> CITYPE=full ci`
`make TARGET=<TARGET> TARGET_ARCH=<TARGET_ARCH> ci` is also the same as
above as CITYPE full is the default now
Approved by: lwhsu
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D48015
---
tests/ci/Makefile | 61 +++++++++++++++++++++++++++++++++++++-----------
tests/ci/tools/ci.conf | 8 +++++--
tests/ci/tools/freebsdci | 34 ++++++++++++++++++++-------
3 files changed, 79 insertions(+), 24 deletions(-)
diff --git a/tests/ci/Makefile b/tests/ci/Makefile
index fed66e5cc317..e452d74679af 100644
--- a/tests/ci/Makefile
+++ b/tests/ci/Makefile
@@ -8,8 +8,9 @@
# Makefile for CI testing.
#
# User-driven targets:
-# ci: Run CI tests. Currently only smoke tests are supported.
-# ci-smokeit: Currently same as ci.
+# ci: Run CI tests
+# ci-smoke: Run smoke tests which is simply booting the image
+# ci-full: Run full tests
#
# Variables affecting the build process:
# TARGET/TARGET_ARCH: architecture of built release (default: same as build host)
@@ -68,11 +69,25 @@ SWAPSIZE?= 1g
VMFS?= ufs
FORMAT= raw
CIIMAGE= ci-${OSRELEASE}-${GITREV}-${KERNCONF}.${FORMAT}
+CIDISK?= ${.OBJDIR}/${CIIMAGE}
VMSIZE?= 6g
-CITYPE?=
+CITYPE?= full
+META_TAR!=mktemp /tmp/meta.XXXXXX
+META_DIR!=mktemp -d /tmp/meta.XXXXXX
+META_DIROUT!=mktemp -d /tmp/meta.XXXXXX
+DISC_CAM!=truncate -s 128m /tmp/disk-cam
+EXTRA_DISK_NUM?=5
+DISK_NUMBERS!=jot - 1 ${EXTRA_DISK_NUM}
+BHYVE_EXTRA_DISK_PARAM?=
+BHYVE_EXTRA_DISK_PARAM+=-s 4:0,ahci-hd,/tmp/disk-cam
+.for i in ${DISK_NUMBERS}
+NEW_DISK!=truncate -s 128m /tmp/disk${i}
+BHYVE_EXTRA_DISK_PARAM+=-s $$((${i} + 4)):0,virtio-blk,/tmp/disk${i}
+CLEANFILES+=/tmp/disk${i}
+.endfor
TEST_VM_NAME= ci-${OSRELEASE}-${GITREV}-${KERNCONF}
.if ${TOTAL_MEMORY} >= 16
-VM_MEM!=expr ${TOTAL_MEMORY} / 2
+VM_MEM=8
.elif ${TOTAL_MEMORY} >=4
VM_MEM=${TOTAL_MEMORY}
.else
@@ -117,8 +132,8 @@ KLDFILEMONISLOADED!=kldload -q -n filemon 2>/dev/null && echo "1" || echo "0"
METAMODE?=-DWITH_META_MODE
.endif
-CLEANFILES= ${CIIMAGE} ci.img
-CLEANDIRS= ci-buildimage
+CLEANFILES+= ${.OBJDIR}/${CIIMAGE} ${.OBJDIR}/ci.img ${META_TAR}
+CLEANDIRS+= ${.OBJDIR}/ci-buildimage
portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} .PHONY
@@ -141,11 +156,13 @@ portinstall-expect: portinstall-pkg .PHONY
.endif
beforeclean: .PHONY
- chflags -R noschg .
+ chflags -R noschg ${.OBJDIR}/${.TARGET}
.include <bsd.obj.mk>
clean: beforeclean .PHONY
+cleandir: beforeclean .PHONY
+
ci-buildworld: .PHONY
@echo "Building world for ${TARGET_ARCH}"
${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
@@ -155,7 +172,7 @@ ci-buildworld: .PHONY
ci-buildkernel: ci-buildworld-${TARGET_ARCH:tl} .PHONY
- @echo "Building kenrel for ${TARGET_ARCH"}"
+ @echo "Building kernel for ${TARGET_ARCH"}"
${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \
${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} \
SRCCONF=${SRCCONF} buildkernel > ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || \
@@ -174,19 +191,32 @@ ci-buildimage: ${QEMUTGT} ci-buildkernel-${TARGET_ARCH:tl} .PHONY
(echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false)
touch ${.TARGET}
-ci-setsmokevar: .PHONY
+ci-set-smoke-var: .PHONY
CITYPE=smoke
+ci-set-full-var: .PHONY
+CITYPE=full
+
+ci-create-meta: .PHONY
+ truncate -s 512M ${META_TAR}
+ tar rvf ${META_TAR} -C ${META_DIR} .
+
+ci-extract-meta: .PHONY
+ tar xfv ${META_TAR} -C ${META_DIROUT}
+ @echo "Extracted kyua reports to ${META_DIROUT}"
+
ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY
.if ${MACHINE} == "amd64" && ( ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" ) && ( !defined(USE_QEMU) || empty(USE_QEMU) )
/usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy || true
- /usr/sbin/bhyveload -c stdio -m ${VM_MEM_SIZE} -d ${CIIMAGE} ${TEST_VM_NAME}
+ /usr/sbin/bhyveload -c stdio -m ${VM_MEM_SIZE} -d ${CIDISK} ${TEST_VM_NAME}
expect -c "set timeout ${TIMEOUT_EXPECT}; \
spawn /usr/bin/timeout -k 60 ${TIMEOUT_VM} /usr/sbin/bhyve \
-c ${PARALLEL_JOBS} -m ${VM_MEM_SIZE} -A -H -P \
-s 0:0,hostbridge \
-s 1:0,lpc \
- -s 2:0,virtio-blk,${CIIMAGE} \
+ -s 2:0,virtio-blk,${CIDISK} \
+ -s 3:0,virtio-blk,${META_TAR} \
+ ${BHYVE_EXTRA_DISK_PARAM} \
-l com1,stdio \
${TEST_VM_NAME}; \
expect { eof }"
@@ -199,7 +229,8 @@ ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY
-nographic \
-no-reboot \
${QEMU_EXTRA_PARAM} \
- -drive if=none,file=${CIIMAGE},format=raw,id=hd0 \
+ -drive if=none,file=${CIDISK},format=raw,id=hd0 \
+ -drive if=none,file=${META_TAR},format=raw,id=hd1 \
${QEMU_DEVICES}
.endif
@@ -215,8 +246,10 @@ ci-checktarget: .PHONY
@echo "Error: ${TARGET_ARCH} is not supported on ${TYPE} ${REVISION} ${BRANCH}"
.endif
-ci-smokeit: ci-setsmokevar ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY
+ci-smoke: ci-set-smoke-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY
+
+ci-full: ci-set-full-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} ci-extract-meta .PHONY
-ci: ci-smokeit .PHONY
+ci: ci-${CITYPE:tl} .PHONY
.include "${RELEASEDIR}/Makefile.inc1"
diff --git a/tests/ci/tools/ci.conf b/tests/ci/tools/ci.conf
index 47001d6248c6..a9998a3e5373 100644
--- a/tests/ci/tools/ci.conf
+++ b/tests/ci/tools/ci.conf
@@ -11,7 +11,7 @@
export VM_RC_LIST="auditd freebsdci"
if [ "${CITYPE}" != "smoke" ]; then
-export VM_EXTRA_PACKAGES="coreutils devel/py-pytest gdb jq ksh93 net/py-dpkt net/scapy nist-kat nmap perl5 python python3 sudo tcptestsuite"
+export VM_EXTRA_PACKAGES="coreutils devel/py-pytest gdb jq ksh93 net/py-dpkt net/scapy nist-kat nmap perl5 python python3 sudo sysutils/porch tcptestsuite"
if [ "${TARGET}" = "amd64" ]; then
export VM_EXTRA_PACKAGES="${VM_EXTRA_PACKAGES} linux-c7-ltp"
@@ -41,6 +41,7 @@ cat << EOF >> ${DESTDIR}/etc/rc.conf
kld_list="" # Load modules needed by tests
kld_list="${kld_list} blake2" # sys/opencrypto
kld_list="${kld_list} cryptodev" # sys/opencrypto
+kld_list="${kld_list} dummymbuf" # sys/netpfil
kld_list="${kld_list} fusefs" # sys/fs/fusefs
kld_list="${kld_list} ipsec" # sys/netipsec
kld_list="${kld_list} mac_portacl" # sys/mac/portacl
@@ -64,6 +65,10 @@ if [ "${CITYPE}" = "smoke" ]; then
cat << EOF >> ${DESTDIR}/etc/rc.conf
freebsdci_type="smoke"
EOF
+elif [ "${CITYPE}" = "full" ]; then
+cat << EOF >> ${DESTDIR}/etc/rc.conf
+freebsdci_type="full"
+EOF
fi
cat << EOF >> ${DESTDIR}/etc/sysctl.conf
kern.cryptodevallowsoft=1
@@ -77,7 +82,6 @@ cat << EOF >> ${DESTDIR}/etc/fstab
fdesc /dev/fd fdescfs rw 0 0
EOF
mkdir -p ${DESTDIR}/usr/local/etc/rc.d
- echo $scriptdir
cp -p ${scriptdir}/../../tests/ci/tools/freebsdci ${DESTDIR}/usr/local/etc/rc.d/
touch ${DESTDIR}/firstboot
diff --git a/tests/ci/tools/freebsdci b/tests/ci/tools/freebsdci
index c77216c9fd4d..f0030fe00aba 100755
--- a/tests/ci/tools/freebsdci
+++ b/tests/ci/tools/freebsdci
@@ -26,6 +26,7 @@
. /etc/rc.subr
: ${freebsdci_enable:="NO"}
+: ${freebsdci_type:="full"}
name="freebsdci"
desc="Run FreeBSD CI"
@@ -33,10 +34,13 @@ rcvar=freebsdci_enable
start_cmd="firstboot_ci_run"
stop_cmd=":"
os_arch=$(uname -p)
+tardev=/dev/vtbd1
+metadir=/meta
+istar=$(file -s ${tardev} | grep "POSIX tar archive" | wc -l)
auto_shutdown()
{
- # XXX: Currently RISC-V kernels lack the ability to
+ # NOTE: Currently RISC-V kernels lack the ability to
# make qemu exit on shutdown. Reboot instead;
# it makes qemu exit too.
case "$os_arch" in
@@ -53,8 +57,6 @@ smoke_tests()
{
echo
echo "--------------------------------------------------------------"
- echo "BUILD sequence COMPLETED"
- echo "IMAGE sequence COMPLETED"
echo "BOOT sequence COMPLETED"
echo "INITIATING system SHUTDOWN"
echo "--------------------------------------------------------------"
@@ -62,15 +64,31 @@ smoke_tests()
full_tests()
{
- # Currently this is a placeholder.
- # This will be used to add the full tests scenario those are run in
- # the CI system
echo
echo "--------------------------------------------------------------"
- echo "BUILD sequence COMPLETED"
- echo "IMAGE sequence COMPLETED"
echo "BOOT sequence COMPLETED"
echo "TEST sequence STARTED"
+ if [ "${istar}" -eq 1 ]; then
+ rm -fr ${metadir}
+ mkdir -p ${metadir}
+ tar xvf ${tardev} -C ${metadir}
+ cd /usr/tests
+ set +e
+ kyua test
+ rc=$?
+ set -e
+ if [ ${rc} -ne 0 ] && [ ${rc} -ne 1 ]; then
+ exit ${rc}
+ fi
+ kyua report --verbose --results-filter passed,skipped,xfail,broken,failed --output test-report.txt
+ kyua report-junit --output=test-report.xml
+ mv test-report.* /${metadir}
+ tar cvf ${tardev} -C ${metadir} .
+ else
+ echo "ERROR: no device with POSIX tar archive format found."
+ # Don't shutdown because this is not run in unattended mode
+ exit 1
+ fi
echo "TEST sequence COMPLETED"
echo "INITIATING system SHUTDOWN"
echo "--------------------------------------------------------------"