git: b731fff69c0f - main - Mk/Uses: Add pytest.mk

The branch main has been updated by amdmi3:

URL: https://cgit.FreeBSD.org/ports/commit/?id=b731fff69c0ffd79f80f5d1d575e016337f3b920

commit b731fff69c0ffd79f80f5d1d575e016337f3b920
Author:     Dmitry Marakasov <amdmi3@FreeBSD.org>
AuthorDate: 2022-04-15 14:17:30 +0000
Commit:     Dmitry Marakasov <amdmi3@FreeBSD.org>
CommitDate: 2022-04-18 10:32:16 +0000

    Mk/Uses: Add pytest.mk
    
    Add USES=pytest, a standardized pytest entry point.
    
    This knob adds dependency on pytest and defines do-test target which
    calls it "the right way" (respecting TEST_ENV, with proper verbosity,
    additional info on skipped tests, and resetting addopts which may
    be set by upstream and introduce unwanted options). This allows to
    remove duplicate (and in many cases not completely correct) do-test
    targets from a lot of pytest consumer ports.
    
    Additionally, it adds PYTEST_IGNORED_TESTS (to list tests which
    fail and are expected to fail) and PYTEST_BROKEN_TESTS (to list
    tests which fail and needs to be fixed). Both knobs lead to
    listed tests being skipped, which is believed to be better than
    always failing `test` target, allowing to monitor status of other
    tests and better document known failures.
    
    Possible room for improvement:
    - Support for pytest plugins (add dependencies and disallow pytest
      from using unlisted plugins which often cause breakages)
    - Treat PYTEST_BROKEN_TESTS as xfail to catch the cases when tests
      are fixed.
    
      Both items seem to require support on pytest side though.
    
    Reviewed by:            tcberner
    Differential Revision:  https://reviews.freebsd.org/D34693
---
 CHANGES           | 13 +++++++++++
 Mk/Uses/pytest.mk | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/CHANGES b/CHANGES
index bb1fdcf4d997..e96e1b0d4a0d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -10,6 +10,19 @@ in the release notes and/or placed into UPDATING.
 
 All ports committers are allowed to commit to this file.
 
+20220415:
+AUTHOR: amdmi3@FreeBSD.org
+
+  A new USES has been added to handle testing with pytest.
+
+    USES=           pytest
+
+  Introduces dependency on pytest and adds do-test target which calls
+  pytest with the right environment and arguments.
+
+  Additionally, PYTEST_{IGNORED,BROKEN}_TESTS knobs are provided for
+  skipping failing tests in a convenient and documented way.
+
 20220218:
 AUTHOR: jrm@FreeBSD.org
 
diff --git a/Mk/Uses/pytest.mk b/Mk/Uses/pytest.mk
new file mode 100644
index 000000000000..62e435270cbd
--- /dev/null
+++ b/Mk/Uses/pytest.mk
@@ -0,0 +1,69 @@
+# handle testing with pytest
+#
+# Feature:	pytest
+# Usage:	USES=pytest[:4]
+#
+# It implies USES=python:test automatically if no USES=python has been
+# specified yet
+#
+# It provides the following additional variables to set by the ports:
+#
+# PYTEST_ARGS		additional args to pytest (defaults to empty).
+# PYTEST_IGNORED_TESTS	lists of `pytest -k` patterns of tests to ignore
+# 			(defaults to empty). For tests which are not
+#			expected to pass, such as ones requiring a database
+#			access.
+# PYTEST_BROKEN_TESTS	lists of `pytest -k` patterns of tests to ignore
+# 			(defaults to empty). For broken tests which require
+#			fixing.
+#
+# The following variables may be set by the user:
+#
+# PYTEST_ENABLE_IGNORED_TESTS	enable tests which are otherwise ignored by
+#				PYTEST_IGNORED_TESTS.
+# PYTEST_ENABLE_BROKEN_TESTS	enable tests which are otherwise ignored by
+#				PYTEST_BROKEN_TESTS.
+# PYTEST_ENABLE_ALL_TESTS	enable tests which are otherwise ignored by
+#				PYTEST_IGNORED_TESTS and PYTEST_BROKEN_TESTS.
+#
+# MAINTAINER: amdmi3@FreeBSD.org
+
+.if !defined(_INCLUDE_USES_PYTEST_MK)
+_INCLUDE_USES_PYTEST_MK=	yes
+
+.  if !${USES:Mpython*}
+python_ARGS=	test
+.    include "${USESDIR}/python.mk"
+.  endif
+
+.  if empty(pytest_ARGS)
+TEST_DEPENDS+=		${PYTHON_PKGNAMEPREFIX}pytest>=0:devel/py-pytest@${PY_FLAVOR}
+.  elif ${pytest_ARGS} == "4"
+TEST_DEPENDS+=		${PYTHON_PKGNAMEPREFIX}pytest4>=0:devel/py-pytest4@${PY_FLAVOR}
+.  else
+IGNORE=	Incorrect 'USES+=pytest:${pytest_ARGS}' expecting 'USES+=pytest[:4]'
+.  endif
+
+PYTEST_IGNORED_TESTS?=	# empty
+PYTEST_BROKEN_TESTS?=	# empty
+PYTEST_ARGS?=		# empty
+
+_PYTEST_ALL_IGNORED_TESTS?=	# empty
+.  if !defined(PYTEST_ENABLE_IGNORED_TESTS) && !defined(PYTEST_ENABLE_ALL_TESTS)
+_PYTEST_ALL_IGNORED_TESTS+=	${PYTEST_IGNORED_TESTS}
+.  endif
+.  if !defined(PYTEST_ENABLE_BROKEN_TESTS) && !defined(PYTEST_ENABLE_ALL_TESTS)
+_PYTEST_ALL_IGNORED_TESTS+=	${PYTEST_BROKEN_TESTS}
+.  endif
+
+_PYTEST_FILTER_EXPRESSION=	${_PYTEST_ALL_IGNORED_TESTS:C/^(.)/and not \1/:tW:C/^and //}
+
+.  if !target(do-test)
+do-test:
+	@cd ${TEST_WRKSRC} && ${SETENV} ${TEST_ENV} ${PYTHON_CMD} -m pytest \
+		-k '${_PYTEST_FILTER_EXPRESSION}' \
+		-v -rs -o addopts= \
+		${PYTEST_ARGS}
+.  endif
+
+.endif