git: 56070dd2e4df - main - Improve timeout precision of pthread_cond_timedwait().
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 04 Mar 2022 03:12:44 UTC
The branch main has been updated by mav:
URL: https://cgit.FreeBSD.org/src/commit/?id=56070dd2e4df70e4de2a10c7ace684ceb8b0f0bd
commit 56070dd2e4df70e4de2a10c7ace684ceb8b0f0bd
Author: Alexander Motin <mav@FreeBSD.org>
AuthorDate: 2022-03-04 03:03:09 +0000
Commit: Alexander Motin <mav@FreeBSD.org>
CommitDate: 2022-03-04 03:03:09 +0000
Improve timeout precision of pthread_cond_timedwait().
This code was not touched when all other user-space sleep functions were
switched to sbintime_t and decoupled from hardclock. When it is possible,
convert supplied times into sbinuptime to supply directly to msleep_sbt()
with C_ABSOLUTE. This provides the timeout resolution of few microseconds
instead of 2 milliseconds, plus avoids few clock reads and conversions.
Reviewed by: vangyzen
MFC after: 1 month
Differential Revision: https://reviews.freebsd.org/D34163
---
sys/kern/kern_umtx.c | 130 ++++++++++++++++++++++++++++++++-------------------
1 file changed, 82 insertions(+), 48 deletions(-)
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 97673ed77e11..b864e6fe9e49 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -175,8 +175,6 @@ static SYSCTL_NODE(_debug_umtx, OID_AUTO, chains, CTLFLAG_RD | CTLFLAG_MPSAFE, 0
static inline void umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
const struct _umtx_time *umtxtime);
-static int umtx_abs_timeout_gethz(struct umtx_abs_timeout *timo);
-static inline void umtx_abs_timeout_update(struct umtx_abs_timeout *timo);
static void umtx_shm_init(void);
static void umtxq_sysinit(void *);
@@ -683,20 +681,14 @@ umtx_abs_timeout_init(struct umtx_abs_timeout *timo, int clockid,
timo->clockid = clockid;
if (!absolute) {
timo->is_abs_real = false;
- umtx_abs_timeout_update(timo);
+ kern_clock_gettime(curthread, timo->clockid, &timo->cur);
timespecadd(&timo->cur, timeout, &timo->end);
} else {
timo->end = *timeout;
timo->is_abs_real = clockid == CLOCK_REALTIME ||
clockid == CLOCK_REALTIME_FAST ||
- clockid == CLOCK_REALTIME_PRECISE;
- /*
- * If is_abs_real, umtxq_sleep will read the clock
- * after setting td_rtcgen; otherwise, read it here.
- */
- if (!timo->is_abs_real) {
- umtx_abs_timeout_update(timo);
- }
+ clockid == CLOCK_REALTIME_PRECISE ||
+ clockid == CLOCK_SECOND;
}
}
@@ -709,22 +701,71 @@ umtx_abs_timeout_init2(struct umtx_abs_timeout *timo,
(umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
}
-static void
-umtx_abs_timeout_update(struct umtx_abs_timeout *timo)
-{
-
- kern_clock_gettime(curthread, timo->clockid, &timo->cur);
-}
-
static int
-umtx_abs_timeout_gethz(struct umtx_abs_timeout *timo)
+umtx_abs_timeout_getsbt(struct umtx_abs_timeout *timo, sbintime_t *sbt,
+ int *flags)
{
+ struct bintime bt, bbt;
struct timespec tts;
- if (timespeccmp(&timo->end, &timo->cur, <=))
- return (-1);
- timespecsub(&timo->end, &timo->cur, &tts);
- return (tstohz(&tts));
+ switch (timo->clockid) {
+
+ /* Clocks that can be converted into absolute time. */
+ case CLOCK_REALTIME:
+ case CLOCK_REALTIME_PRECISE:
+ case CLOCK_REALTIME_FAST:
+ case CLOCK_MONOTONIC:
+ case CLOCK_MONOTONIC_PRECISE:
+ case CLOCK_MONOTONIC_FAST:
+ case CLOCK_UPTIME:
+ case CLOCK_UPTIME_PRECISE:
+ case CLOCK_UPTIME_FAST:
+ case CLOCK_SECOND:
+ timespec2bintime(&timo->end, &bt);
+ switch (timo->clockid) {
+ case CLOCK_REALTIME:
+ case CLOCK_REALTIME_PRECISE:
+ case CLOCK_REALTIME_FAST:
+ case CLOCK_SECOND:
+ getboottimebin(&bbt);
+ bintime_sub(&bt, &bbt);
+ break;
+ }
+ if (bt.sec < 0)
+ return (ETIMEDOUT);
+ if (bt.sec >= (SBT_MAX >> 32)) {
+ *sbt = 0;
+ *flags = 0;
+ return (0);
+ }
+ *sbt = bttosbt(bt);
+ switch (timo->clockid) {
+ case CLOCK_REALTIME_FAST:
+ case CLOCK_MONOTONIC_FAST:
+ case CLOCK_UPTIME_FAST:
+ *sbt += tc_tick_sbt;
+ break;
+ case CLOCK_SECOND:
+ *sbt += SBT_1S;
+ break;
+ }
+ *flags = C_ABSOLUTE;
+ return (0);
+
+ /* Clocks that has to be periodically polled. */
+ case CLOCK_VIRTUAL:
+ case CLOCK_PROF:
+ case CLOCK_THREAD_CPUTIME_ID:
+ case CLOCK_PROCESS_CPUTIME_ID:
+ default:
+ kern_clock_gettime(curthread, timo->clockid, &timo->cur);
+ if (timespeccmp(&timo->end, &timo->cur, <=))
+ return (ETIMEDOUT);
+ timespecsub(&timo->end, &timo->cur, &tts);
+ *sbt = tick_sbt * tstohz(&tts);
+ *flags = C_HARDCLOCK;
+ return (0);
+ }
}
static uint32_t
@@ -746,15 +787,11 @@ umtx_unlock_val(uint32_t flags, bool rb)
*/
int
umtxq_sleep(struct umtx_q *uq, const char *wmesg,
- struct umtx_abs_timeout *abstime)
+ struct umtx_abs_timeout *timo)
{
struct umtxq_chain *uc;
- int error, timo;
-
- if (abstime != NULL && abstime->is_abs_real) {
- curthread->td_rtcgen = atomic_load_acq_int(&rtc_generation);
- umtx_abs_timeout_update(abstime);
- }
+ sbintime_t sbt = 0;
+ int error, flags = 0;
uc = umtxq_getchain(&uq->uq_key);
UMTXQ_LOCKED_ASSERT(uc);
@@ -763,26 +800,22 @@ umtxq_sleep(struct umtx_q *uq, const char *wmesg,
error = 0;
break;
}
- if (abstime != NULL) {
- timo = umtx_abs_timeout_gethz(abstime);
- if (timo < 0) {
- error = ETIMEDOUT;
- break;
- }
- } else
- timo = 0;
- error = msleep(uq, &uc->uc_lock, PCATCH | PDROP, wmesg, timo);
- if (error == EINTR || error == ERESTART) {
- umtxq_lock(&uq->uq_key);
- break;
- }
- if (abstime != NULL) {
- if (abstime->is_abs_real)
+ if (timo != NULL) {
+ if (timo->is_abs_real)
curthread->td_rtcgen =
atomic_load_acq_int(&rtc_generation);
- umtx_abs_timeout_update(abstime);
+ error = umtx_abs_timeout_getsbt(timo, &sbt, &flags);
+ if (error != 0)
+ break;
+ }
+ error = msleep_sbt(uq, &uc->uc_lock, PCATCH, wmesg,
+ sbt, 0, flags);
+ if (error == EINTR || error == ERESTART)
+ break;
+ if (error == EWOULDBLOCK && (flags & C_ABSOLUTE) != 0) {
+ error = ETIMEDOUT;
+ break;
}
- umtxq_lock(&uq->uq_key);
}
curthread->td_rtcgen = 0;
@@ -3652,7 +3685,8 @@ again:
if (error == ERESTART)
error = EINTR;
if (error == EINTR) {
- umtx_abs_timeout_update(&timo);
+ kern_clock_gettime(curthread, timo.clockid,
+ &timo.cur);
timespecsub(&timo.end, &timo.cur,
&timeout->_timeout);
}