Re: git: 315ee00fa961 - main - zfs: merge openzfs/zfs@804414aad

From: Gleb Smirnoff <glebius_at_freebsd.org>
Date: Thu, 31 Aug 2023 00:09:29 UTC
  Hi Martin,

we hit this panic with this update:

panic: VERIFY(avl_is_empty(&lwb->lwb_vdev_tree)) failed

cpuid = 8
time = 1693432004
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe09519aa800
vpanic() at vpanic+0x132/frame 0xfffffe09519aa930
spl_panic() at spl_panic+0x3a/frame 0xfffffe09519aa990
zil_free_lwb() at zil_free_lwb+0x1de/frame 0xfffffe09519aa9c0
zil_sync() at zil_sync+0x3cb/frame 0xfffffe09519aaaa0
[Wed Aug 30 21:46:44 2023]dmu_objset_sync() at dmu_objset_sync+0x51b/frame 0xfffffe09519aab70
dsl_pool_sync() at dsl_pool_sync+0x11d/frame 0xfffffe09519aabf0
spa_sync() at spa_sync+0xc68/frame 0xfffffe09519aae20
txg_sync_thread() at txg_sync_thread+0x2eb/frame 0xfffffe09519aaef0
fork_exit() at fork_exit+0x82/frame 0xfffffe09519aaf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe09519aaf30
--- trap 0x26d90a2f, rip = 0xbba54c81b25fd8d4, rsp = 0xff60c2f6f9874bcd, rbp = 0x37940e3e9793bd40 --

I have two cores dumped for this panic.

Our previous version is based on 9c0a2d522f3a18fc795a7dd6266d2be2aff4b084, which
I guess has the most recent ZFS merge openzfs/zfs@009d3288d.

On Sun, Aug 27, 2023 at 05:09:06AM +0000, Martin Matuska wrote:
M> The branch main has been updated by mm:
M> 
M> URL: https://cgit.FreeBSD.org/src/commit/?id=315ee00fa9616b0a192b6834911f98bcf5316a6b
M> 
M> commit 315ee00fa9616b0a192b6834911f98bcf5316a6b
M> Merge: b3a714652ff0 804414aad224
M> Author:     Martin Matuska <mm@FreeBSD.org>
M> AuthorDate: 2023-08-26 21:20:04 +0000
M> Commit:     Martin Matuska <mm@FreeBSD.org>
M> CommitDate: 2023-08-26 21:51:42 +0000
M> 
M>     zfs: merge openzfs/zfs@804414aad
M>     
M>     Notable upstream pull request merges:
M>       #15024 Add missed DMU_PROJECTUSED_OBJECT prefetch
M>       #15029 Do not request data L1 buffers on scan prefetch
M>       #15036 FreeBSD: catch up to __FreeBSD_version 1400093
M>       #15039 Fix raw receive with different indirect block size
M>       #15047 FreeBSD: Fix build on stable/13 after 1302506
M>       #15049 Fix the ZFS checksum error histograms with larger record sizes
M>       #15052 Reduce bloat in ereport.fs.zfs.checksum events
M>       #15056 Avoid extra snprintf() in dsl_deadlist_merge()
M>       #15061 Ignore pool ashift property during vdev attachment
M>       #15063 Don't panic if setting vdev properties is unsupported for this vdev type
M>       #15067 spa_min_alloc should be GCD, not min
M>       #15071 Add explicit prefetches to bpobj_iterate()
M>       #15072 Adjust prefetch parameters
M>       #15076 Refactor dmu_prefetch()
M>       #15079 set autotrim default to 'off' everywhere
M>       #15080 ZIL: Fix config lock deadlock
M>       #15088 metaslab: tuneable to better control force ganging
M>       #15096 Avoid waiting in dmu_sync_late_arrival()
M>       #15097 BRT should return EOPNOTSUPP
M>       #15103 Remove zl_issuer_lock from zil_suspend()
M>       #15107 Remove fastwrite mechanism
M>       #15113 libzfs: sendrecv: send_progress_thread: handle SIGINFO/SIGUSR1
M>       #15122 ZIL: Second attempt to reduce scope of zl_issuer_lock
M>       #15129 zpool_vdev_remove() should handle EALREADY error return
M>       #15132 ZIL: Replay blocks without next block pointer
M>       #15148 zfs_clone_range should return descriptive error codes
M>       #15153 ZIL: Avoid dbuf_read() before dmu_sync()
M>       #15172 copy_file_range: fix fallback when source create on same txg
M>       #15180 Update outdated assertion from zio_write_compress
M>     
M>     Obtained from:  OpenZFS
M>     OpenZFS commit: 804414aad224b432590afe3f9ec114ffb49e0f13
M> 
M>  Makefile.inc1                                      |   5 +-
M>  cddl/lib/libzfs/Makefile                           |   1 +
M>  cddl/lib/libzfs/Makefile.depend                    |   1 +
M>  rescue/rescue/Makefile                             |   2 +-
M>  rescue/rescue/Makefile.depend                      |   1 +
M>  share/mk/src.libnames.mk                           |   2 +-
M>  sys/contrib/openzfs/META                           |   6 +-
M>  sys/contrib/openzfs/cmd/zdb/zdb.c                  |  77 ++-
M>  sys/contrib/openzfs/cmd/zed/agents/zfs_mod.c       |   2 -
M>  sys/contrib/openzfs/cmd/zed/agents/zfs_retire.c    |   5 +
M>  .../openzfs/cmd/zed/zed.d/statechange-slot_off.sh  |  61 ++
M>  sys/contrib/openzfs/cmd/zed/zed.d/zed.rc           |   5 +
M>  sys/contrib/openzfs/cmd/ztest.c                    |   2 +-
M>  sys/contrib/openzfs/config/Shellcheck.am           |   3 +-
M>  sys/contrib/openzfs/config/kernel-blkdev.m4        | 138 +++-
M>  .../config/kernel-block-device-operations.m4       |  35 +-
M>  .../openzfs/config/kernel-filemap-splice-read.m4   |  25 +
M>  .../openzfs/config/kernel-register_sysctl_table.m4 |  27 +
M>  .../config/kernel-vfs-extended-file_range.m4       |  50 ++
M>  .../openzfs/config/kernel-vfs-file_range.m4        | 164 +++++
M>  sys/contrib/openzfs/config/kernel-vfs-iov_iter.m4  |  26 +-
M>  sys/contrib/openzfs/config/kernel.m4               |  16 +
M>  sys/contrib/openzfs/contrib/debian/changelog       |   6 +
M>  .../contrib/debian/openzfs-zfs-test.install        |   2 -
M>  .../contrib/debian/openzfs-zfsutils.install        |   2 +
M>  .../contrib/dracut/90zfs/zfs-env-bootfs.service.in |  11 +-
M>  .../dracut/90zfs/zfs-rollback-bootfs.service.in    |   2 +-
M>  .../openzfs/include/os/freebsd/spl/sys/vnode.h     |   6 +-
M>  .../include/os/linux/kernel/linux/blkdev_compat.h  |  17 +
M>  .../openzfs/include/os/linux/spl/sys/kmem_cache.h  |   8 +
M>  .../openzfs/include/os/linux/spl/sys/types.h       |   2 +-
M>  sys/contrib/openzfs/include/os/linux/spl/sys/uio.h |  12 +
M>  sys/contrib/openzfs/include/os/linux/zfs/sys/zpl.h |  53 ++
M>  sys/contrib/openzfs/include/sys/bpobj.h            |   2 +-
M>  sys/contrib/openzfs/include/sys/brt.h              |   1 +
M>  sys/contrib/openzfs/include/sys/dmu.h              |   5 +
M>  sys/contrib/openzfs/include/sys/dmu_impl.h         |   2 -
M>  sys/contrib/openzfs/include/sys/dmu_zfetch.h       |   2 -
M>  sys/contrib/openzfs/include/sys/fm/fs/zfs.h        |   4 -
M>  sys/contrib/openzfs/include/sys/metaslab.h         |   3 -
M>  sys/contrib/openzfs/include/sys/metaslab_impl.h    |   6 +-
M>  sys/contrib/openzfs/include/sys/spa_impl.h         |   1 +
M>  sys/contrib/openzfs/include/sys/vdev_impl.h        |   2 +-
M>  sys/contrib/openzfs/include/sys/zil_impl.h         |  44 +-
M>  sys/contrib/openzfs/include/sys/zio.h              |   1 -
M>  sys/contrib/openzfs/include/sys/zio_checksum.h     |   2 -
M>  sys/contrib/openzfs/lib/libzfs/Makefile.am         |   2 +-
M>  sys/contrib/openzfs/lib/libzfs/libzfs_pool.c       |   6 +
M>  sys/contrib/openzfs/lib/libzfs/libzfs_sendrecv.c   |  95 ++-
M>  sys/contrib/openzfs/man/man4/zfs.4                 |  10 +-
M>  sys/contrib/openzfs/man/man8/zfs-send.8            |  18 +-
M>  sys/contrib/openzfs/man/man8/zpool-events.8        |  23 +-
M>  sys/contrib/openzfs/module/Kbuild.in               |   1 +
M>  .../openzfs/module/os/freebsd/zfs/zfs_vnops_os.c   |  10 +-
M>  sys/contrib/openzfs/module/os/linux/spl/spl-proc.c |  26 +-
M>  .../openzfs/module/os/linux/zfs/vdev_disk.c        |  65 +-
M>  .../openzfs/module/os/linux/zfs/zfs_ctldir.c       |  26 +-
M>  .../openzfs/module/os/linux/zfs/zfs_vfsops.c       |   7 +
M>  .../openzfs/module/os/linux/zfs/zfs_vnops_os.c     |   9 +-
M>  .../openzfs/module/os/linux/zfs/zfs_znode.c        |   8 +
M>  .../openzfs/module/os/linux/zfs/zpl_ctldir.c       |   2 +-
M>  sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c |  44 +-
M>  .../openzfs/module/os/linux/zfs/zpl_file_range.c   | 272 ++++++++
M>  .../openzfs/module/os/linux/zfs/zpl_super.c        |  39 +-
M>  sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c  |  28 +-
M>  sys/contrib/openzfs/module/zfs/bpobj.c             |  49 +-
M>  sys/contrib/openzfs/module/zfs/brt.c               |  43 +-
M>  sys/contrib/openzfs/module/zfs/dbuf.c              |  11 +-
M>  sys/contrib/openzfs/module/zfs/dmu.c               | 127 ++--
M>  sys/contrib/openzfs/module/zfs/dmu_recv.c          |  22 +-
M>  sys/contrib/openzfs/module/zfs/dmu_zfetch.c        |  12 +-
M>  sys/contrib/openzfs/module/zfs/dnode.c             |  31 +-
M>  sys/contrib/openzfs/module/zfs/dsl_deadlist.c      |  14 +-
M>  sys/contrib/openzfs/module/zfs/dsl_scan.c          |  17 +-
M>  sys/contrib/openzfs/module/zfs/metaslab.c          |  83 +--
M>  sys/contrib/openzfs/module/zfs/spa_log_spacemap.c  |   4 +-
M>  sys/contrib/openzfs/module/zfs/spa_misc.c          |   1 +
M>  sys/contrib/openzfs/module/zfs/vdev.c              |  90 ++-
M>  sys/contrib/openzfs/module/zfs/vdev_indirect.c     |   2 +-
M>  sys/contrib/openzfs/module/zfs/vdev_raidz.c        |   2 +-
M>  sys/contrib/openzfs/module/zfs/zfs_fm.c            |  33 +-
M>  sys/contrib/openzfs/module/zfs/zfs_vnops.c         |  46 +-
M>  sys/contrib/openzfs/module/zfs/zil.c               | 719 +++++++++------------
M>  sys/contrib/openzfs/module/zfs/zio.c               |  45 +-
M>  sys/contrib/openzfs/module/zfs/zio_checksum.c      |   2 -
M>  sys/contrib/openzfs/module/zfs/zvol.c              |   8 +-
M>  sys/contrib/openzfs/tests/runfiles/linux.run       |  11 +
M>  .../openzfs/tests/test-runner/bin/zts-report.py.in |  18 +
M>  sys/contrib/openzfs/tests/zfs-tests/cmd/.gitignore |   1 +
M>  .../openzfs/tests/zfs-tests/cmd/Makefile.am        |   1 +
M>  .../openzfs/tests/zfs-tests/cmd/clonefile.c        | 333 ++++++++++
M>  sys/contrib/openzfs/tests/zfs-tests/cmd/readmmap.c |   1 +
M>  .../openzfs/tests/zfs-tests/include/commands.cfg   |   1 +
M>  .../openzfs/tests/zfs-tests/tests/Makefile.am      |  14 +
M>  .../functional/block_cloning/block_cloning.kshlib  |  54 ++
M>  .../block_cloning/block_cloning_copyfilerange.ksh  |  60 ++
M>  .../block_cloning_copyfilerange_cross_dataset.ksh  |  65 ++
M>  .../block_cloning_copyfilerange_fallback.ksh       |  86 +++
M>  ...ock_cloning_copyfilerange_fallback_same_txg.ksh |  66 ++
M>  .../block_cloning_copyfilerange_partial.ksh        |  68 ++
M>  .../block_cloning_disabled_copyfilerange.ksh       |  60 ++
M>  .../block_cloning_disabled_ficlone.ksh             |  50 ++
M>  .../block_cloning_disabled_ficlonerange.ksh        |  50 ++
M>  .../block_cloning/block_cloning_ficlone.ksh        |  56 ++
M>  .../block_cloning/block_cloning_ficlonerange.ksh   |  56 ++
M>  .../block_cloning_ficlonerange_partial.ksh         |  64 ++
M>  .../tests/functional/block_cloning/cleanup.ksh     |  34 +
M>  .../tests/functional/block_cloning/setup.ksh       |  36 ++
M>  .../cli_root/zpool_attach/attach-o_ashift.ksh      |  30 +-
M>  .../cli_root/zpool_replace/replace-o_ashift.ksh    |  32 +-
M>  .../cli_root/zpool_replace/replace_prop_ashift.ksh |  24 +-
M>  sys/modules/zfs/zfs_config.h                       |  53 +-
M>  sys/modules/zfs/zfs_gitrev.h                       |   2 +-
M>  113 files changed, 3322 insertions(+), 906 deletions(-)
M> 
M> diff --cc Makefile.inc1
M> index 8838d31c795a,000000000000..f740cc5abcca
M> mode 100644,000000..100644
M> --- a/Makefile.inc1
M> +++ b/Makefile.inc1
M> @@@ -1,3692 -1,0 +1,3695 @@@
M>  +#
M>  +#
M>  +# Make command line options:
M>  +#	-DNO_CLEANDIR run ${MAKE} clean, instead of ${MAKE} cleandir
M>  +#	-DNO_CLEAN do not clean at all
M>  +#	-DDB_FROM_SRC use the user/group databases in src/etc instead of
M>  +#	    the system database when installing.
M>  +#	-DNO_SHARE do not go into share subdir
M>  +#	-DKERNFAST define NO_KERNEL{CONFIG,CLEAN,OBJ}
M>  +#	-DNO_KERNELCONFIG do not run config in ${MAKE} buildkernel
M>  +#	-DNO_KERNELCLEAN do not run ${MAKE} clean in ${MAKE} buildkernel
M>  +#	-DNO_KERNELOBJ do not run ${MAKE} obj in ${MAKE} buildkernel
M>  +#	-DNO_ROOT install without using root privilege
M>  +#	-DWITHOUT_CTF do not run the DTrace CTF conversion tools on built objects
M>  +#	LOCAL_DIRS="list of dirs" to add additional dirs to the SUBDIR list
M>  +#	LOCAL_ITOOLS="list of tools" to add additional tools to the ITOOLS list
M>  +#	LOCAL_LIB_DIRS="list of dirs" to add additional dirs to libraries target
M>  +#	LOCAL_MTREE="list of mtree files" to process to allow local directories
M>  +#	    to be created before files are installed
M>  +#	LOCAL_LEGACY_DIRS="list of dirs" to add additional dirs to the legacy
M>  +#	    target
M>  +#	LOCAL_BSTOOL_DIRS="list of dirs" to add additional dirs to the
M>  +#	    bootstrap-tools target
M>  +#	LOCAL_TOOL_DIRS="list of dirs" to add additional dirs to the build-tools
M>  +#	    target
M>  +#	LOCAL_XTOOL_DIRS="list of dirs" to add additional dirs to the
M>  +#	    cross-tools target
M>  +#	METALOG="path to metadata log" to write permission and ownership
M>  +#	    when NO_ROOT is set.  (default: ${DESTDIR}/${DISTDIR}/METALOG,
M>  +#           check /etc/make.conf for DISTDIR)
M>  +#	TARGET="machine" to crossbuild world for a different machine type
M>  +#	TARGET_ARCH= may be required when a TARGET supports multiple endians
M>  +#	BUILDENV_SHELL= shell to launch for the buildenv target (def:${SHELL})
M>  +#	WORLD_FLAGS= additional flags to pass to make(1) during buildworld
M>  +#	KERNEL_FLAGS= additional flags to pass to make(1) during buildkernel
M>  +#	SUBDIR_OVERRIDE="list of dirs" to build rather than everything.
M>  +#	    All libraries and includes, and some build tools will still build.
M>  +
M>  +#
M>  +# The intended user-driven targets are:
M>  +# buildworld  - rebuild *everything*, including glue to help do upgrades
M>  +# installworld- install everything built by "buildworld"
M>  +# checkworld  - run test suite on installed world
M>  +# doxygen     - build API documentation of the kernel
M>  +#
M>  +# Standard targets (not defined here) are documented in the makefiles in
M>  +# /usr/share/mk.  These include:
M>  +#		obj depend all install clean cleandepend cleanobj
M>  +
M>  +.if !defined(TARGET) || !defined(TARGET_ARCH)
M>  +.error Both TARGET and TARGET_ARCH must be defined.
M>  +.endif
M>  +
M>  +.if make(showconfig) || make(test-system-*)
M>  +_MKSHOWCONFIG=	t
M>  +.endif
M>  +
M>  +SRCDIR?=	${.CURDIR}
M>  +LOCALBASE?=	/usr/local
M>  +TIME_ENV ?= time env
M>  +
M>  +.include "share/mk/src.tools.mk"
M>  +
M>  +# Cross toolchain changes must be in effect before bsd.compiler.mk
M>  +# so that gets the right CC, and pass CROSS_TOOLCHAIN to submakes.
M>  +.if defined(CROSS_TOOLCHAIN)
M>  +.if exists(${LOCALBASE}/share/toolchains/${CROSS_TOOLCHAIN}.mk)
M>  +.include "${LOCALBASE}/share/toolchains/${CROSS_TOOLCHAIN}.mk"
M>  +.elif exists(${CROSS_TOOLCHAIN})
M>  +.include "${CROSS_TOOLCHAIN}"
M>  +.else
M>  +.error CROSS_TOOLCHAIN ${CROSS_TOOLCHAIN} not found
M>  +.endif
M>  +CROSSENV+=CROSS_TOOLCHAIN="${CROSS_TOOLCHAIN}"
M>  +.elif defined(UNIVERSE_TOOLCHAIN)
M>  +UNIVERSE_TOOLCHAIN_PATH?=${HOST_OBJTOP}/tmp/usr/bin
M>  +XCC?="${UNIVERSE_TOOLCHAIN_PATH}/cc"
M>  +XCXX?="${UNIVERSE_TOOLCHAIN_PATH}/c++"
M>  +XCPP?="${UNIVERSE_TOOLCHAIN_PATH}/cpp"
M>  +XLD?="${UNIVERSE_TOOLCHAIN_PATH}/ld"
M>  +.endif
M>  +.if defined(CROSS_TOOLCHAIN_PREFIX)
M>  +CROSS_COMPILER_PREFIX?=${CROSS_TOOLCHAIN_PREFIX}
M>  +.endif
M>  +
M>  +XCOMPILERS=	CC CXX CPP
M>  +.for COMPILER in ${XCOMPILERS}
M>  +.if defined(CROSS_COMPILER_PREFIX)
M>  +X${COMPILER}?=	${CROSS_COMPILER_PREFIX}${${COMPILER}}
M>  +.else
M>  +X${COMPILER}?=	${${COMPILER}}
M>  +.endif
M>  +.endfor
M>  +# If a full path to an external cross compiler is given, don't build
M>  +# a cross compiler.
M>  +.if ${XCC:N${CCACHE_BIN}:M/*}
M>  +MK_CLANG_BOOTSTRAP=	no
M>  +# Make sure sub-makes see the option as disabled so the hack in bsd.sys.mk to
M>  +# work around incompatible headers in Clang's resource directory is enabled.
M>  +.MAKEOVERRIDES+=	MK_CLANG_BOOTSTRAP
M>  +.endif
M>  +
M>  +# Pull in compiler metadata from buildworld/toolchain if possible to avoid
M>  +# running CC from bsd.compiler.mk.
M>  +.if make(installworld) || make(install) || make(distributeworld) || \
M>  +    make(stageworld)
M>  +.-include "${OBJTOP}/toolchain-metadata.mk"
M>  +.if !defined(_LOADED_TOOLCHAIN_METADATA)
M>  +.error A build is required first.  You may have the wrong MAKEOBJDIRPREFIX set.
M>  +.endif
M>  +.endif
M>  +
M>  +# Pull in COMPILER_TYPE and COMPILER_FREEBSD_VERSION early. Pull it from the
M>  +# tree to be friendlier to foreign OS builds. It's safe to do so unconditionally
M>  +# here since we will always have the right make, unlike in src/Makefile
M>  +# Don't include bsd.linker.mk yet until XBINUTILS is handled (after src.opts.mk)
M>  +_NO_INCLUDE_LINKERMK=	t
M>  +# We also want the X_COMPILER* variables if we are using an external toolchain.
M>  +_WANT_TOOLCHAIN_CROSS_VARS=	t
M>  +.include "share/mk/bsd.compiler.mk"
M>  +.undef _NO_INCLUDE_LINKERMK
M>  +.undef _WANT_TOOLCHAIN_CROSS_VARS
M>  +# src.opts.mk depends on COMPILER_FEATURES
M>  +.include "share/mk/src.opts.mk"
M>  +
M>  +.if ${TARGET} == ${MACHINE}
M>  +TARGET_CPUTYPE?=${CPUTYPE}
M>  +.else
M>  +TARGET_CPUTYPE?=
M>  +.endif
M>  +.if !empty(TARGET_CPUTYPE)
M>  +_TARGET_CPUTYPE=${TARGET_CPUTYPE}
M>  +.else
M>  +_TARGET_CPUTYPE=dummy
M>  +.endif
M>  +.if ${TARGET} == "arm"
M>  +.if ${TARGET_CPUTYPE:M*soft*} == ""
M>  +TARGET_TRIPLE_ABI=	gnueabihf
M>  +.else
M>  +TARGET_TRIPLE_ABI=	gnueabi
M>  +.endif
M>  +.endif
M>  +MACHINE_TRIPLE_ABI?=	unknown
M>  +MACHINE_TRIPLE?=${MACHINE_ARCH:S/amd64/x86_64/}-${MACHINE_TRIPLE_ABI}-freebsd${OS_REVISION}
M>  +TARGET_TRIPLE_ABI?=	unknown
M>  +TARGET_TRIPLE?=	${TARGET_ARCH:S/amd64/x86_64/}-${TARGET_TRIPLE_ABI}-freebsd${OS_REVISION}
M>  +KNOWN_ARCHES?=	aarch64/arm64 \
M>  +		amd64 \
M>  +		armv6/arm \
M>  +		armv7/arm \
M>  +		i386 \
M>  +		powerpc \
M>  +		powerpc64/powerpc \
M>  +		powerpc64le/powerpc \
M>  +		powerpcspe/powerpc \
M>  +		riscv64/riscv
M>  +
M>  +.if ${TARGET} == ${TARGET_ARCH}
M>  +_t=		${TARGET}
M>  +.else
M>  +_t=		${TARGET_ARCH}/${TARGET}
M>  +.endif
M>  +.for _t in ${_t}
M>  +.if empty(KNOWN_ARCHES:M${_t})
M>  +.error Unknown target ${TARGET_ARCH}:${TARGET}.
M>  +.endif
M>  +.endfor
M>  +
M>  +.if ${TARGET_ARCH} == "amd64"
M>  +LIBCOMPAT_INCLUDE_DIRS+= i386
M>  +.elif ${TARGET_ARCH} == "aarch64"
M>  +LIBCOMPAT_INCLUDE_DIRS+= arm
M>  +.endif
M>  +
M>  +.if ${.MAKE.OS} != "FreeBSD"
M>  +CROSSBUILD_HOST=${.MAKE.OS}
M>  +.if ${.MAKE.OS} != "Linux" && ${.MAKE.OS} != "Darwin"
M>  +.warning Unsupported crossbuild system: ${.MAKE.OS}. Build will probably fail!
M>  +.endif
M>  +# We need to force NO_ROOT/DB_FROM_SRC builds when building on other operating
M>  +# systems since the BSD.foo.dist specs contain users and groups that do not
M>  +# exist by default on a Linux/MacOS system.
M>  +NO_ROOT:=	1
M>  +DB_FROM_SRC:=	1
M>  +.export NO_ROOT
M>  +.endif
M>  +
M>  +# If all targets are disabled for system llvm then don't expect it to work
M>  +# for cross-builds.
M>  +.if !defined(TOOLS_PREFIX) && ${MK_LLVM_TARGET_ALL} == "no" && \
M>  +    ${MACHINE} != ${TARGET} && ${MACHINE_ARCH} != ${TARGET_ARCH} && \
M>  +    !make(showconfig)
M>  +MK_SYSTEM_COMPILER=	no
M>  +MK_SYSTEM_LINKER=	no
M>  +.endif
M>  +
M>  +# Handle external binutils.
M>  +.if defined(CROSS_TOOLCHAIN_PREFIX)
M>  +CROSS_BINUTILS_PREFIX?=${CROSS_TOOLCHAIN_PREFIX}
M>  +.endif
M>  +XBINUTILS=	AS AR ELFCTL LD NM OBJCOPY RANLIB SIZE STRINGS STRIPBIN
M>  +.for BINUTIL in ${XBINUTILS}
M>  +.if defined(CROSS_BINUTILS_PREFIX) && \
M>  +    exists(${CROSS_BINUTILS_PREFIX}/${${BINUTIL}})
M>  +X${BINUTIL}?=	${CROSS_BINUTILS_PREFIX:C,/*$,,}/${${BINUTIL}}
M>  +.else
M>  +X${BINUTIL}?=	${${BINUTIL}}
M>  +.endif
M>  +.endfor
M>  +
M>  +# If a full path to an external linker is given, don't build lld.
M>  +.if ${XLD:M/*}
M>  +MK_LLD_BOOTSTRAP=	no
M>  +.endif
M>  +
M>  +# We also want the X_LINKER* variables if we are using an external toolchain.
M>  +_WANT_TOOLCHAIN_CROSS_VARS=	t
M>  +.include "share/mk/bsd.linker.mk"
M>  +.undef _WANT_TOOLCHAIN_CROSS_VARS
M>  +
M>  +# Begin WITH_SYSTEM_COMPILER / WITH_SYSTEM_LD
M>  +
M>  +# WITH_SYSTEM_COMPILER - Pull in needed values and make a decision.
M>  +
M>  +# Check if there is a local compiler that can satisfy as an external compiler.
M>  +# Which compiler is expected to be used?
M>  +.if ${MK_CLANG_BOOTSTRAP} == "yes"
M>  +WANT_COMPILER_TYPE=	clang
M>  +.else
M>  +WANT_COMPILER_TYPE=
M>  +.endif
M>  +
M>  +.if !defined(WANT_COMPILER_FREEBSD_VERSION) && !make(showconfig) && \
M>  +    !make(test-system-linker)
M>  +.if ${WANT_COMPILER_TYPE} == "clang"
M>  +WANT_COMPILER_FREEBSD_VERSION_FILE= lib/clang/freebsd_cc_version.h
M>  +WANT_COMPILER_FREEBSD_VERSION!= \
M>  +	awk '$$2 == "FREEBSD_CC_VERSION" {printf("%d\n", $$3)}' \
M>  +	${SRCDIR}/${WANT_COMPILER_FREEBSD_VERSION_FILE} || echo unknown
M>  +WANT_COMPILER_VERSION_FILE= lib/clang/include/clang/Basic/Version.inc
M>  +WANT_COMPILER_VERSION!= \
M>  +	awk '$$2 == "CLANG_VERSION" {split($$3, a, "."); print a[1] * 10000 + a[2] * 100 + a[3]}' \
M>  +	${SRCDIR}/${WANT_COMPILER_VERSION_FILE} || echo unknown
M>  +.endif
M>  +.export WANT_COMPILER_FREEBSD_VERSION WANT_COMPILER_VERSION
M>  +.endif	# !defined(WANT_COMPILER_FREEBSD_VERSION)
M>  +
M>  +# It needs to be the same revision as we would build for the bootstrap.
M>  +# If the expected vs CC is different then we can't skip.
M>  +# GCC cannot be used for cross-arch yet.  For clang we pass -target later if
M>  +# TARGET_ARCH!=MACHINE_ARCH.
M>  +.if ${MK_SYSTEM_COMPILER} == "yes" && \
M>  +    defined(WANT_COMPILER_FREEBSD_VERSION) && \
M>  +    ${MK_CLANG_BOOTSTRAP} == "yes" && \
M>  +    !make(xdev*) && \
M>  +    ${X_COMPILER_TYPE} == ${WANT_COMPILER_TYPE} && \
M>  +    (${X_COMPILER_TYPE} == "clang" || ${TARGET_ARCH} == ${MACHINE_ARCH}) && \
M>  +    ${X_COMPILER_VERSION} == ${WANT_COMPILER_VERSION} && \
M>  +    ${X_COMPILER_FREEBSD_VERSION} == ${WANT_COMPILER_FREEBSD_VERSION}
M>  +# Everything matches, disable the bootstrap compiler.
M>  +MK_CLANG_BOOTSTRAP=	no
M>  +USING_SYSTEM_COMPILER=	yes
M>  +.endif	# ${WANT_COMPILER_TYPE} == ${COMPILER_TYPE}
M>  +
M>  +# WITH_SYSTEM_LD - Pull in needed values and make a decision.
M>  +
M>  +# Check if there is a local linker that can satisfy as an external linker.
M>  +# Which linker is expected to be used?
M>  +.if ${MK_LLD_BOOTSTRAP} == "yes"
M>  +WANT_LINKER_TYPE=		lld
M>  +.else
M>  +WANT_LINKER_TYPE=
M>  +.endif
M>  +
M>  +.if !defined(WANT_LINKER_FREEBSD_VERSION) && !make(showconfig) && \
M>  +    !make(test-system-compiler)
M>  +.if ${WANT_LINKER_TYPE} == "lld"
M>  +WANT_LINKER_FREEBSD_VERSION_FILE= lib/clang/include/lld/Common/Version.inc
M>  +WANT_LINKER_FREEBSD_VERSION!= \
M>  +	awk '$$2 == "LLD_FREEBSD_VERSION" {print $$3}' \
M>  +	${SRCDIR}/${WANT_LINKER_FREEBSD_VERSION_FILE} || echo unknown
M>  +WANT_LINKER_VERSION_FILE= lib/clang/include/lld/Common/Version.inc
M>  +WANT_LINKER_VERSION!= \
M>  +	awk '$$2 == "LLD_VERSION_STRING" {gsub("\"", "", $$3); split($$3, a, "."); print a[1] * 10000 + a[2] * 100 + a[3]}' \
M>  +	${SRCDIR}/${WANT_LINKER_VERSION_FILE} || echo unknown
M>  +.else
M>  +WANT_LINKER_FREEBSD_VERSION_FILE=
M>  +WANT_LINKER_FREEBSD_VERSION=
M>  +.endif
M>  +.export WANT_LINKER_FREEBSD_VERSION WANT_LINKER_VERSION
M>  +.endif	# !defined(WANT_LINKER_FREEBSD_VERSION)
M>  +
M>  +.if ${MK_SYSTEM_LINKER} == "yes" && \
M>  +    defined(WANT_LINKER_FREEBSD_VERSION) && \
M>  +    (${MK_LLD_BOOTSTRAP} == "yes") && \
M>  +    !make(xdev*) && \
M>  +    ${X_LINKER_TYPE} == ${WANT_LINKER_TYPE} && \
M>  +    ${X_LINKER_VERSION} == ${WANT_LINKER_VERSION} && \
M>  +    ${X_LINKER_FREEBSD_VERSION} == ${WANT_LINKER_FREEBSD_VERSION}
M>  +# Everything matches, disable the bootstrap linker.
M>  +MK_LLD_BOOTSTRAP=	no
M>  +USING_SYSTEM_LINKER=	yes
M>  +.endif	# ${WANT_LINKER_TYPE} == ${LINKER_TYPE}
M>  +
M>  +# WITH_SYSTEM_COMPILER / WITH_SYSTEM_LINKER - Handle defaults and debug.
M>  +USING_SYSTEM_COMPILER?=	no
M>  +USING_SYSTEM_LINKER?=	no
M>  +
M>  +TEST_SYSTEM_COMPILER_VARS= \
M>  +	USING_SYSTEM_COMPILER MK_SYSTEM_COMPILER \
M>  +	MK_CROSS_COMPILER MK_CLANG_BOOTSTRAP \
M>  +	WANT_COMPILER_TYPE WANT_COMPILER_VERSION WANT_COMPILER_VERSION_FILE \
M>  +	WANT_COMPILER_FREEBSD_VERSION WANT_COMPILER_FREEBSD_VERSION_FILE \
M>  +	CC COMPILER_TYPE COMPILER_FEATURES COMPILER_VERSION \
M>  +	COMPILER_FREEBSD_VERSION \
M>  +	XCC X_COMPILER_TYPE X_COMPILER_FEATURES X_COMPILER_VERSION \
M>  +	X_COMPILER_FREEBSD_VERSION
M>  +TEST_SYSTEM_LINKER_VARS= \
M>  +	USING_SYSTEM_LINKER MK_SYSTEM_LINKER \
M>  +	MK_LLD_BOOTSTRAP \
M>  +	WANT_LINKER_TYPE WANT_LINKER_VERSION WANT_LINKER_VERSION_FILE \
M>  +	WANT_LINKER_FREEBSD_VERSION WANT_LINKER_FREEBSD_VERSION_FILE \
M>  +	LD LINKER_TYPE LINKER_FEATURES LINKER_VERSION \
M>  +	LINKER_FREEBSD_VERSION \
M>  +	XLD X_LINKER_TYPE X_LINKER_FEATURES X_LINKER_VERSION \
M>  +	X_LINKER_FREEBSD_VERSION
M>  +
M>  +.for _t in compiler linker
M>  +test-system-${_t}: .PHONY
M>  +.for v in ${TEST_SYSTEM_${_t:tu}_VARS}
M>  +	${_+_}@printf "%-35s= %s\n" "${v}" "${${v}}"
M>  +.endfor
M>  +.endfor
M>  +.if (make(buildworld) || make(buildkernel) || make(kernel-toolchain) || \
M>  +    make(toolchain) || make(_cross-tools))
M>  +.if ${USING_SYSTEM_COMPILER} == "yes"
M>  +.info SYSTEM_COMPILER: Determined that CC=${CC} matches the source tree.  Not bootstrapping a cross-compiler.
M>  +.elif ${MK_CLANG_BOOTSTRAP} == "yes"
M>  +.info SYSTEM_COMPILER: libclang will be built for bootstrapping a cross-compiler.
M>  +.endif
M>  +.if ${USING_SYSTEM_LINKER} == "yes"
M>  +.info SYSTEM_LINKER: Determined that LD=${LD} matches the source tree.  Not bootstrapping a cross-linker.
M>  +.elif ${MK_LLD_BOOTSTRAP} == "yes"
M>  +.info SYSTEM_LINKER: libclang will be built for bootstrapping a cross-linker.
M>  +.endif
M>  +.endif
M>  +
M>  +# End WITH_SYSTEM_COMPILER / WITH_SYSTEM_LD
M>  +
M>  +# Store some compiler metadata for use in installworld where we don't
M>  +# want to invoke CC at all.
M>  +_TOOLCHAIN_METADATA_VARS=	COMPILER_VERSION \
M>  +				COMPILER_TYPE \
M>  +				COMPILER_FEATURES \
M>  +				COMPILER_FREEBSD_VERSION \
M>  +				COMPILER_RESOURCE_DIR \
M>  +				LINKER_VERSION \
M>  +				LINKER_FEATURES \
M>  +				LINKER_TYPE \
M>  +				LINKER_FREEBSD_VERSION
M>  +toolchain-metadata.mk: .PHONY .META
M>  +	@: > ${.TARGET}
M>  +	@echo ".info Using cached toolchain metadata from build at $$(hostname) on $$(date)" \
M>  +	    > ${.TARGET}
M>  +	@echo "_LOADED_TOOLCHAIN_METADATA=t" >> ${.TARGET}
M>  +.for v in ${_TOOLCHAIN_METADATA_VARS}
M>  +	@echo "${v}=${${v}}" >> ${.TARGET}
M>  +	@echo "X_${v}=${X_${v}}" >> ${.TARGET}
M>  +.endfor
M>  +	@echo ".export ${_TOOLCHAIN_METADATA_VARS}" >> ${.TARGET}
M>  +	@echo ".export ${_TOOLCHAIN_METADATA_VARS:C,^,X_,}" >> ${.TARGET}
M>  +
M>  +
M>  +# We must do lib/ and libexec/ before bin/ in case of a mid-install error to
M>  +# keep the users system reasonably usable.  For static->dynamic root upgrades,
M>  +# we don't want to install a dynamic binary without rtld and the needed
M>  +# libraries.  More commonly, for dynamic root, we don't want to install a
M>  +# binary that requires a newer library version that hasn't been installed yet.
M>  +# This ordering is not a guarantee though.  The only guarantee of a working
M>  +# system here would require fine-grained ordering of all components based
M>  +# on their dependencies.
M>  +.if !empty(SUBDIR_OVERRIDE)
M>  +SUBDIR=	${SUBDIR_OVERRIDE}
M>  +.else
M>  +SUBDIR=	lib libexec
M>  +# Add LOCAL_LIB_DIRS, but only if they will not be picked up as a SUBDIR
M>  +# of a LOCAL_DIRS directory.  This allows LOCAL_DIRS=foo and
M>  +# LOCAL_LIB_DIRS=foo/lib to behave as expected.
M>  +.for _DIR in ${LOCAL_DIRS:M*/} ${LOCAL_DIRS:N*/:S|$|/|}
M>  +_REDUNDANT_LIB_DIRS+=    ${LOCAL_LIB_DIRS:M${_DIR}*}
M>  +.endfor
M>  +.for _DIR in ${LOCAL_LIB_DIRS}
M>  +.if ${_DIR} == ".WAIT" || (empty(_REDUNDANT_LIB_DIRS:M${_DIR}) && exists(${.CURDIR}/${_DIR}/Makefile))
M>  +SUBDIR+=	${_DIR}
M>  +.endif
M>  +.endfor
M>  +.if !defined(NO_ROOT) && (make(installworld) || make(install))
M>  +# Ensure libraries are installed before progressing.
M>  +SUBDIR+=.WAIT
M>  +.endif
M>  +SUBDIR+=bin
M>  +.if ${MK_CDDL} != "no"
M>  +SUBDIR+=cddl
M>  +.endif
M>  +SUBDIR+=gnu include
M>  +.if ${MK_KERBEROS} != "no"
M>  +SUBDIR+=kerberos5
M>  +.endif
M>  +.if ${MK_RESCUE} != "no"
M>  +SUBDIR+=rescue
M>  +.endif
M>  +SUBDIR+=sbin
M>  +.if ${MK_CRYPT} != "no"
M>  +SUBDIR+=secure
M>  +.endif
M>  +.if !defined(NO_SHARE)
M>  +SUBDIR+=share
M>  +.endif
M>  +.if ${MK_BOOT} != "no"
M>  +SUBDIR+=stand
M>  +.endif
M>  +SUBDIR+=sys usr.bin usr.sbin
M>  +.if ${MK_TESTS} != "no"
M>  +SUBDIR+=	tests
M>  +.endif
M>  +
M>  +# Local directories are built in parallel with the base system directories.
M>  +# Users may insert a .WAIT directive at the beginning or elsewhere within
M>  +# the LOCAL_DIRS and LOCAL_LIB_DIRS lists as needed.
M>  +.for _DIR in ${LOCAL_DIRS}
M>  +.if ${_DIR} == ".WAIT" || exists(${.CURDIR}/${_DIR}/Makefile)
M>  +SUBDIR+=	${_DIR}
M>  +.endif
M>  +.endfor
M>  +
M>  +# We must do etc/ last as it hooks into building the man whatis file
M>  +# by calling 'makedb' in share/man.  This is only relevant for
M>  +# install/distribute so they build the whatis file after every manpage is
M>  +# installed.
M>  +.if make(installworld) || make(install)
M>  +SUBDIR+=.WAIT
M>  +.endif
M>  +SUBDIR+=etc
M>  +
M>  +.endif	# !empty(SUBDIR_OVERRIDE)
M>  +
M>  +.if defined(NOCLEAN)
M>  +.warning The src.conf WITHOUT_CLEAN option can now be used instead of NOCLEAN.
M>  +MK_CLEAN:=	no
M>  +.endif
M>  +.if defined(NO_CLEAN)
M>  +.info The src.conf WITHOUT_CLEAN option can now be used instead of NO_CLEAN.
M>  +MK_CLEAN:=	no
M>  +.endif
M>  +.if defined(NO_CLEANDIR)
M>  +CLEANDIR=	clean cleandepend
M>  +.else
M>  +CLEANDIR=	cleandir
M>  +.endif
M>  +
M>  +.if defined(WORLDFAST)
M>  +MK_CLEAN:=	no
M>  +NO_OBJWALK=	t
M>  +.endif
M>  +
M>  +.if ${MK_META_MODE} == "yes"
M>  +# If filemon is used then we can rely on the build being incremental-safe.
M>  +# The .meta files will also track the build command and rebuild should
M>  +# it change.
M>  +.if empty(.MAKE.MODE:Mnofilemon)
M>  +MK_CLEAN:=	no
M>  +.endif
M>  +.endif
M>  +.if defined(NO_OBJWALK) || ${MK_AUTO_OBJ} == "yes"
M>  +NO_OBJWALK=	t
M>  +NO_KERNELOBJ=	t
M>  +.endif
M>  +.if !defined(NO_OBJWALK)
M>  +_obj=		obj
M>  +.endif
M>  +
M>  +LOCAL_TOOL_DIRS?=
M>  +PACKAGEDIR?=	${DESTDIR}/${DISTDIR}
M>  +
M>  +.if empty(SHELL:M*csh*)
M>  +BUILDENV_SHELL?=${SHELL}
M>  +.else
M>  +BUILDENV_SHELL?=/bin/sh
M>  +.endif
M>  +
M>  +.if !defined(_MKSHOWCONFIG)
M>  +.if !defined(VCS_REVISION) || empty(VCS_REVISION)
M>  +.if !defined(SVNVERSION_CMD) || empty(SVNVERSION_CMD)
M>  +. for _D in ${PATH:S,:, ,g}
M>  +.  if exists(${_D}/svnversion)
M>  +SVNVERSION_CMD?=${_D}/svnversion
M>  +.  endif
M>  +.  if exists(${_D}/svnliteversion)
M>  +SVNVERSION_CMD?=${_D}/svnliteversion
M>  +.  endif
M>  +. endfor
M>  +.endif
M>  +.if defined(SVNVERSION_CMD) && !empty(SVNVERSION_CMD)
M>  +_VCS_REVISION?=	$$(eval ${SVNVERSION_CMD} ${SRCDIR})
M>  +. if !empty(_VCS_REVISION)
M>  +VCS_REVISION=	$$(echo r${_VCS_REVISION})
M>  +.export VCS_REVISION
M>  +. endif
M>  +.endif
M>  +.endif
M>  +
M>  +.if !defined(GIT_CMD) || empty(GIT_CMD)
M>  +. for _P in /usr/bin /usr/local/bin
M>  +.  if exists(${_P}/git)
M>  +GIT_CMD=   ${_P}/git
M>  +.  endif
M>  +. endfor
M>  +.export GIT_CMD
M>  +.endif
M>  +
M>  +.if !defined(OSRELDATE)
M>  +.if exists(/usr/include/osreldate.h)
M>  +OSRELDATE!=	awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
M>  +		/usr/include/osreldate.h
M>  +.else
M>  +OSRELDATE=	0
M>  +.endif
M>  +.export OSRELDATE
M>  +.endif
M>  +
M>  +# Set VERSION for CTFMERGE to use via the default CTFFLAGS=-L VERSION.
M>  +.for _V in BRANCH REVISION TYPE
M>  +.if !defined(_${_V})
M>  +_${_V}!=	eval $$(awk '/^${_V}=/{print}' ${SRCTOP}/sys/conf/newvers.sh); echo $$${_V}
M>  +.export _${_V}
M>  +.endif
M>  +.endfor
M>  +.if !defined(SRCRELDATE)
M>  +SRCRELDATE!=	awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
M>  +		${SRCDIR}/sys/sys/param.h
M>  +.export SRCRELDATE
M>  +.endif
M>  +.if !defined(VERSION)
M>  +VERSION=	FreeBSD ${_REVISION}-${_BRANCH:C/-p[0-9]+$//} ${TARGET_ARCH} ${SRCRELDATE}
M>  +.export VERSION
M>  +.endif
M>  +MAJOR_REVISION=	${_REVISION:R}
M>  +
M>  +.if !defined(PKG_VERSION)
M>  +_STRTIMENOW=	%Y%m%d%H%M%S
M>  +_TIMENOW=	${_STRTIMENOW:gmtime}
M>  +.if ${_BRANCH:MCURRENT*} || ${_BRANCH:MSTABLE*} || ${_BRANCH:MPRERELEASE*}
M>  +EXTRA_REVISION=	.snap${_TIMENOW}
M>  +.elif ${_BRANCH:MALPHA*}
M>  +EXTRA_REVISION=	.a${_BRANCH:C/ALPHA([0-9]+).*/\1/}.${_TIMENOW}
M>  +.elif ${_BRANCH:MBETA*}
M>  +EXTRA_REVISION=	.b${_BRANCH:C/BETA([0-9]+).*/\1/}.${_TIMENOW}
M>  +.elif ${_BRANCH:MRC*}
M>  +EXTRA_REVISION=	.rc${_BRANCH:C/RC([0-9]+).*/\1/}.${_TIMENOW}
M>  +.elif ${_BRANCH:M*-p*}
M>  +EXTRA_REVISION=	p${_BRANCH:C/.*-p([0-9]+$)/\1/}
M>  +.endif
M>  +PKG_VERSION:=	${MAJOR_REVISION}${EXTRA_REVISION:C/[[:space:]]//g}
M>  +.endif
M>  +.endif	# !defined(PKG_VERSION)
M>  +
M>  +.if !defined(PKG_TIMESTAMP)
M>  +TIMEEPOCHNOW=		%s
M>  +SOURCE_DATE_EPOCH=	${TIMEEPOCHNOW:gmtime}
M>  +.else
M>  +SOURCE_DATE_EPOCH=	${PKG_TIMESTAMP}
M>  +.endif
M>  +
M>  +PKG_NAME_PREFIX?=	FreeBSD
M>  +PKG_MAINTAINER?=	re@FreeBSD.org
M>  +PKG_WWW?=		https://www.FreeBSD.org
M>  +.export PKG_NAME_PREFIX
M>  +.export PKG_MAINTAINER
M>  +.export PKG_WWW
M>  +
M>  +.if !defined(_MKSHOWCONFIG)
M>  +_CPUTYPE!=	MAKEFLAGS= CPUTYPE=${_TARGET_CPUTYPE} ${MAKE} -f /dev/null \
M>  +		-m ${.CURDIR}/share/mk MK_AUTO_OBJ=no -V CPUTYPE
M>  +.if ${_CPUTYPE} != ${_TARGET_CPUTYPE}
M>  +.error CPUTYPE global should be set with ?=.
M>  +.endif
M>  +.endif
M>  +.if make(buildworld)
M>  +BUILD_ARCH!=	uname -p
M>  +# On some Linux systems uname -p returns "unknown" so skip this check there.
M>  +# This check only exists to tell people to use TARGET_ARCH instead of
M>  +# MACHINE_ARCH so skipping it when crossbuilding on non-FreeBSD should be fine.
M>  +.if ${MACHINE_ARCH} != ${BUILD_ARCH} && ${.MAKE.OS} == "FreeBSD"
M>  +.error To cross-build, set TARGET_ARCH.
M>  +.endif
M>  +.endif
M>  +WORLDTMP?=	${OBJTOP}/tmp
M>  +BPATH=		${CCACHE_WRAPPER_PATH_PFX}${WORLDTMP}/legacy/usr/sbin:${WORLDTMP}/legacy/usr/bin:${WORLDTMP}/legacy/bin:${WORLDTMP}/legacy/usr/libexec
M>  +XPATH=		${WORLDTMP}/bin:${WORLDTMP}/usr/sbin:${WORLDTMP}/usr/bin
M>  +
M>  +# When building we want to find the cross tools before the host tools in ${BPATH}.
M>  +# We also need to add UNIVERSE_TOOLCHAIN_PATH so that we can find the shared
M>  +# toolchain files (clang, lld, etc.) during make universe/tinderbox
M>  +STRICTTMPPATH=	${XPATH}:${BPATH}:${UNIVERSE_TOOLCHAIN_PATH}
M>  +# We should not be using tools from /usr/bin accidentally since this could cause
M>  +# the build to break on other systems that don't have that tool. For now we
M>  +# still allow using the old behaviour (inheriting $PATH) if
M>  +# BUILD_WITH_STRICT_TMPPATH is set to 0 but this will eventually be removed.
M>  +
M>  +# Currently strict $PATH can cause build failures. Once the remaining issues
M>  +# have been resolved it will be turned on by default.
M>  +BUILD_WITH_STRICT_TMPPATH?=0
M>  +.if defined(CROSSBUILD_HOST)
M>  +# When building on non-FreeBSD we can't rely on the tools in /usr/bin being compatible
M>  +# with what FreeBSD expects. Therefore we only use tools from STRICTTMPPATH
M>  +# during the world build stage. We build most tools during the bootstrap-tools
M>  +# phase but symlink host tools that are known to work instead of building them
M>  +BUILD_WITH_STRICT_TMPPATH:=1
M>  +.endif
M>  +.if ${BUILD_WITH_STRICT_TMPPATH} != 0
M>  +TMPPATH=	${STRICTTMPPATH}
M>  +.else
M>  +TMPPATH=	${STRICTTMPPATH}:${PATH}
M>  +.endif
M>  +
M>  +#
M>  +# Avoid running mktemp(1) unless actually needed.
M>  +# It may not be functional, e.g., due to new ABI
M>  +# when in the middle of installing over this system.
M>  +#
M>  +.if make(distributeworld) || make(installworld) || make(stageworld)
M>  +.if ${BUILD_WITH_STRICT_TMPPATH} != 0
M>  +MKTEMP=${WORLDTMP}/legacy/usr/bin/mktemp
M>  +.if !exists(${MKTEMP})
M>  +.error mktemp binary doesn't exist in expected location: ${MKTEMP}
M>  +.endif
M>  +.else
M>  +MKTEMP=mktemp
M>  +.endif
M>  +INSTALLTMP!=	${MKTEMP} -d -u -t install
M>  +
M>  +.if ${.MAKE.OS} == "FreeBSD"
M>  +# When building on FreeBSD we always copy the host tools instead of linking
M>  +# into INSTALLTMP to avoid issues with incompatible libraries (see r364030).
M>  +# Note: we could create links if we don't intend to update the current machine.
M>  +INSTALLTMP_COPY_HOST_TOOL=cp
M>  +.else
M>  +# However, this is not necessary on Linux/macOS. Additionally, copying the host
M>  +# tools to another directory with cp results in AMFI Launch Constraint
M>  +# Violations on macOS Ventura as part of its System Integrity Protection.
M>  +INSTALLTMP_COPY_HOST_TOOL=ln -s
M>  +.endif
M>  +.endif
M>  +
M>  +.if make(stagekernel) || make(distributekernel)
M>  +TAGS+=		kernel
M>  +PACKAGE=	kernel
M>  +.endif
M>  +
M>  +#
M>  +# Building a world goes through the following stages
M>  +#
M>  +# 1. legacy stage [BMAKE]
M>  +#	This stage is responsible for creating compatibility
M>  +#	shims that are needed by the bootstrap-tools,
M>  +#	build-tools and cross-tools stages. These are generally
M>  +#	APIs that tools from one of those three stages need to
M>  +#	build that aren't present on the host.
M>  +# 1. bootstrap-tools stage [BMAKE]
M>  +#	This stage is responsible for creating programs that
M>  +#	are needed for backward compatibility reasons. They
M>  +#	are not built as cross-tools.
M>  +# 2. build-tools stage [TMAKE]
M>  +#	This stage is responsible for creating the object
M>  +#	tree and building any tools that are needed during
M>  +#	the build process. Some programs are listed during
M>  +#	this phase because they build binaries to generate
M>  +#	files needed to build these programs. This stage also
M>  +#	builds the 'build-tools' target rather than 'all'.
M>  +# 3. cross-tools stage [XMAKE]
M>  +#	This stage is responsible for creating any tools that
M>  +#	are needed for building the system. A cross-compiler is one
M>  +#	of them. This differs from build tools in two ways:
M>  +#	1. the 'all' target is built rather than 'build-tools'
M>  +#	2. these tools are installed into TMPPATH for stage 4.
M>  +# 4. world stage [WMAKE]
M>  +#	This stage actually builds the world.
M>  +# 5. install stage (optional) [IMAKE]
M>  +#	This stage installs a previously built world.
M>  +#
M>  +
M>  +BOOTSTRAPPING?=	0
M>  +# Keep these in sync
M>  +MINIMUM_SUPPORTED_OSREL?= 1104001
M>  +MINIMUM_SUPPORTED_REL?= 11.4
M>  +
M>  +# Common environment for world related stages
M>  +CROSSENV+=	\
M>  +		MACHINE_ARCH=${TARGET_ARCH} \
M>  +		MACHINE=${TARGET} \
M>  +		CPUTYPE=${TARGET_CPUTYPE}
M>  +.if ${MK_META_MODE} != "no"
M>  +# Don't rebuild build-tools targets during normal build.
M>  +CROSSENV+=	BUILD_TOOLS_META=.NOMETA
M>  +.endif
M>  +.if defined(TARGET_CFLAGS)
M>  +CROSSENV+=	${TARGET_CFLAGS}
M>  +.endif
M>  +.if (${TARGET} != ${MACHINE} && !defined(WITH_LOCAL_MODULES)) || \
M>  +    defined(WITHOUT_LOCAL_MODULES)
M>  +CROSSENV+=	LOCAL_MODULES=
M>  +.endif
M>  +
M>  +BOOTSTRAPPING_OSRELDATE?=${OSRELDATE}
M>  +
M>  +# bootstrap-tools stage
M>  +BMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
M>  +		TOOLS_PREFIX=${TOOLS_PREFIX_UNDEF:U${WORLDTMP}} \
M>  +		PATH=${BPATH:Q}:${PATH:Q} \
M>  +		WORLDTMP=${WORLDTMP} \
M>  +		MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}"
M>  +# need to keep this in sync with targets/pseudo/bootstrap-tools/Makefile
M>  +BSARGS= 	DESTDIR= \
M>  +		OBJTOP='${WORLDTMP}/obj-tools' \
M>  +		OBJROOT='$${OBJTOP}/' \
M>  +		UNIVERSE_TOOLCHAIN_PATH=${UNIVERSE_TOOLCHAIN_PATH} \
M>  +		MAKEOBJDIRPREFIX= \
M>  +		BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \
M>  +		BWPHASE=${.TARGET:C,^_,,} \
M>  +		-DNO_CPU_CFLAGS \
M>  +		-DNO_LINT \
M>  +		-DNO_PIC \
M>  +		-DNO_SHARED \
M>  +		MK_ASAN=no \
M>  +		MK_CTF=no \
M>  +		MK_CLANG_EXTRAS=no \
M>  +		MK_CLANG_FORMAT=no \
M>  +		MK_CLANG_FULL=no \
M>  +		MK_HTML=no \
M>  +		MK_MAN=no \
M>  +		MK_PROFILE=no \
M>  +		MK_RETPOLINE=no \
M>  +		MK_SSP=no \
M>  +		MK_TESTS=no \
M>  +		MK_UBSAN=no \
M>  +		MK_WERROR=no \
M>  +		MK_INCLUDES=yes \
M>  +		MK_MAN_UTILS=yes
M>  +
M>  +BMAKE=		\
M>  +		${TIME_ENV} ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
M>  +		${BSARGS}
M>  +.if empty(.MAKEOVERRIDES:MMK_LLVM_TARGET_ALL)
M>  +BMAKE+=		MK_LLVM_TARGET_ALL=no
M>  +.endif
M>  +
M>  +# build-tools stage
M>  +TMAKE=		\
M>  +		${TIME_ENV} ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
M>  +		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
M>  +		DESTDIR= \
M>  +		BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \
M>  +		BWPHASE=${.TARGET:C,^_,,} \
M>  +		-DNO_CPU_CFLAGS \
M>  +		-DNO_LINT \
M>  +		MK_ASAN=no \
M>  +		MK_CTF=no \
M>  +		MK_CLANG_EXTRAS=no \
M>  +		MK_CLANG_FORMAT=no \
M>  +		MK_CLANG_FULL=no \
M>  +		MK_LLDB=no \
M>  +		MK_RETPOLINE=no \
M>  +		MK_SSP=no \
M>  +		MK_TESTS=no \
M>  +		MK_UBSAN=no \
M>  +		MK_WERROR=no
M>  +
M>  +# cross-tools stage
M>  +# TOOLS_PREFIX set in BMAKE
M>  +XMAKE=		${BMAKE} \
M>  +		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
M>  +		MK_LLDB=no \
M>  +		MK_LLVM_BINUTILS=no \
M>  +		MK_TESTS=no
M>  +
M>  +# kernel-tools stage
M>  +KTMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
M>  +		PATH=${BPATH:Q}:${PATH:Q} \
M>  +		WORLDTMP=${WORLDTMP} \
M>  +		MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}"
M>  +
M>  +KTMAKE=		${TIME_ENV} \
M>  +		TOOLS_PREFIX=${TOOLS_PREFIX_UNDEF:U${WORLDTMP}} \
M>  +		${KTMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
M>  +		DESTDIR= \
M>  +		OBJTOP='${WORLDTMP}/obj-kernel-tools' \
M>  +		OBJROOT='$${OBJTOP}/' \
M>  +		UNIVERSE_TOOLCHAIN_PATH=${UNIVERSE_TOOLCHAIN_PATH} \
M>  +		MAKEOBJDIRPREFIX= \
M>  +		BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \
M>  +		-DNO_CPU_CFLAGS \
M>  +		-DNO_LINT \
M>  +		-DNO_PIC \
M>  +		-DNO_SHARED \
M>  +		MK_CTF=no \
M>  +		MK_HTML=no \
M>  +		MK_MAN=no \
M>  +		MK_PROFILE=no \
M>  +		MK_SSP=no \
M>  +		MK_RETPOLINE=no \
M>  +		MK_WERROR=no
M>  +
M>  +# world stage
M>  +WMAKEENV=	${CROSSENV} \
M>  +		INSTALL="${INSTALL_CMD} -U" \
M>  +		PATH=${TMPPATH:Q} \
M>  +		SYSROOT=${WORLDTMP}
M>  +
M>  +# make hierarchy
M> *** 5431 LINES SKIPPED ***

-- 
Gleb Smirnoff