git: a1a8f8ada13d - main - cache: deinline state handling
Mateusz Guzik
mjg at FreeBSD.org
Wed Jan 6 07:36:12 UTC 2021
The branch main has been updated by mjg:
URL: https://cgit.FreeBSD.org/src/commit/?id=a1a8f8ada13da1cce2cd3a4b4335a1243bd34f3e
commit a1a8f8ada13da1cce2cd3a4b4335a1243bd34f3e
Author: Mateusz Guzik <mjg at FreeBSD.org>
AuthorDate: 2020-12-31 20:23:08 +0000
Commit: Mateusz Guzik <mjg at FreeBSD.org>
CommitDate: 2021-01-06 06:05:22 +0000
cache: deinline state handling
The intent is to reduce branchfest when finishing the lookup.
Tested by: pho
---
sys/kern/vfs_cache.c | 157 ++++++++++++++++++++++++++-------------------------
1 file changed, 81 insertions(+), 76 deletions(-)
diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index 7260896df452..9d54997c518c 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -3590,18 +3590,23 @@ struct nameidata_saved {
struct cache_fpl {
struct nameidata *ndp;
struct componentname *cnp;
- struct pwd *pwd;
+ struct pwd **pwd;
struct vnode *dvp;
struct vnode *tvp;
seqc_t dvp_seqc;
seqc_t tvp_seqc;
struct nameidata_saved snd;
+ struct nameidata_saved snd_orig;
int line;
enum cache_fpl_status status:8;
bool in_smr;
bool fsearch;
};
+static bool cache_fplookup_is_mp(struct cache_fpl *fpl);
+static int cache_fplookup_cross_mount(struct cache_fpl *fpl);
+static int cache_fplookup_partial_setup(struct cache_fpl *fpl);
+
static void
cache_fpl_cleanup_cnp(struct componentname *cnp)
{
@@ -3676,9 +3681,24 @@ cache_fpl_restore_abort(struct cache_fpl *fpl, struct nameidata_saved *snd)
MPASS(_fpl->in_smr == false); \
VFS_SMR_ASSERT_NOT_ENTERED(); \
})
+static void
+cache_fpl_assert_status(struct cache_fpl *fpl)
+{
+
+ switch (fpl->status) {
+ case CACHE_FPL_STATUS_UNSET:
+ __assert_unreachable();
+ break;
+ case CACHE_FPL_STATUS_ABORTED:
+ case CACHE_FPL_STATUS_PARTIAL:
+ case CACHE_FPL_STATUS_HANDLED:
+ break;
+ }
+}
#else
#define cache_fpl_smr_assert_entered(fpl) do { } while (0)
#define cache_fpl_smr_assert_not_entered(fpl) do { } while (0)
+#define cache_fpl_assert_status(fpl) do { } while (0)
#endif
#define cache_fpl_smr_enter_initial(fpl) ({ \
@@ -3702,6 +3722,23 @@ cache_fpl_restore_abort(struct cache_fpl *fpl, struct nameidata_saved *snd)
})
static int
+cache_fpl_aborted_early_impl(struct cache_fpl *fpl, int line)
+{
+
+ if (fpl->status != CACHE_FPL_STATUS_UNSET) {
+ KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
+ ("%s: converting to abort from %d at %d, set at %d\n",
+ __func__, fpl->status, line, fpl->line));
+ }
+ cache_fpl_smr_assert_not_entered(fpl);
+ fpl->status = CACHE_FPL_STATUS_ABORTED;
+ fpl->line = line;
+ return (CACHE_FPL_FAILED);
+}
+
+#define cache_fpl_aborted_early(x) cache_fpl_aborted_early_impl((x), __LINE__)
+
+static int __noinline
cache_fpl_aborted_impl(struct cache_fpl *fpl, int line)
{
@@ -3712,12 +3749,15 @@ cache_fpl_aborted_impl(struct cache_fpl *fpl, int line)
}
fpl->status = CACHE_FPL_STATUS_ABORTED;
fpl->line = line;
+ if (fpl->in_smr)
+ cache_fpl_smr_exit(fpl);
+ cache_fpl_restore_abort(fpl, &fpl->snd_orig);
return (CACHE_FPL_FAILED);
}
#define cache_fpl_aborted(x) cache_fpl_aborted_impl((x), __LINE__)
-static int
+static int __noinline
cache_fpl_partial_impl(struct cache_fpl *fpl, int line)
{
@@ -3727,7 +3767,7 @@ cache_fpl_partial_impl(struct cache_fpl *fpl, int line)
cache_fpl_smr_assert_entered(fpl);
fpl->status = CACHE_FPL_STATUS_PARTIAL;
fpl->line = line;
- return (CACHE_FPL_FAILED);
+ return (cache_fplookup_partial_setup(fpl));
}
#define cache_fpl_partial(x) cache_fpl_partial_impl((x), __LINE__)
@@ -3766,9 +3806,6 @@ cache_fpl_terminated(struct cache_fpl *fpl)
_Static_assert((CACHE_FPL_SUPPORTED_CN_FLAGS & CACHE_FPL_INTERNAL_CN_FLAGS) == 0,
"supported and internal flags overlap");
-static bool cache_fplookup_is_mp(struct cache_fpl *fpl);
-static int cache_fplookup_cross_mount(struct cache_fpl *fpl);
-
static bool
cache_fpl_islastcn(struct nameidata *ndp)
{
@@ -3798,29 +3835,29 @@ cache_can_fplookup(struct cache_fpl *fpl)
td = cnp->cn_thread;
if (!cache_fast_lookup) {
- cache_fpl_aborted(fpl);
+ cache_fpl_aborted_early(fpl);
return (false);
}
#ifdef MAC
if (mac_vnode_check_lookup_enabled()) {
- cache_fpl_aborted(fpl);
+ cache_fpl_aborted_early(fpl);
return (false);
}
#endif
if ((cnp->cn_flags & ~CACHE_FPL_SUPPORTED_CN_FLAGS) != 0) {
- cache_fpl_aborted(fpl);
+ cache_fpl_aborted_early(fpl);
return (false);
}
if (IN_CAPABILITY_MODE(td)) {
- cache_fpl_aborted(fpl);
+ cache_fpl_aborted_early(fpl);
return (false);
}
if (AUDITING_TD(td)) {
- cache_fpl_aborted(fpl);
+ cache_fpl_aborted_early(fpl);
return (false);
}
if (ndp->ni_startdir != NULL) {
- cache_fpl_aborted(fpl);
+ cache_fpl_aborted_early(fpl);
return (false);
}
return (true);
@@ -3836,7 +3873,6 @@ cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp)
ndp = fpl->ndp;
error = fgetvp_lookup_smr(ndp->ni_dirfd, ndp, vpp, &fsearch);
if (__predict_false(error != 0)) {
- cache_fpl_smr_exit(fpl);
return (cache_fpl_aborted(fpl));
}
fpl->fsearch = fsearch;
@@ -3882,12 +3918,11 @@ cache_fplookup_partial_setup(struct cache_fpl *fpl)
ndp = fpl->ndp;
cnp = fpl->cnp;
- pwd = fpl->pwd;
+ pwd = *(fpl->pwd);
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
if (!pwd_hold_smr(pwd)) {
- cache_fpl_smr_exit(fpl);
return (cache_fpl_aborted(fpl));
}
@@ -4981,7 +5016,6 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl)
struct mount *mp;
int error;
- error = CACHE_FPL_FAILED;
ndp = fpl->ndp;
cnp = fpl->cnp;
@@ -4990,20 +5024,18 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl)
fpl->dvp = dvp;
fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
if (seqc_in_modify(fpl->dvp_seqc)) {
- cache_fpl_aborted(fpl);
- goto out;
+ return (cache_fpl_aborted(fpl));
}
mp = atomic_load_ptr(&dvp->v_mount);
if (__predict_false(mp == NULL || !cache_fplookup_mp_supported(mp))) {
- cache_fpl_aborted(fpl);
- goto out;
+ return (cache_fpl_aborted(fpl));
}
VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp);
error = cache_fplookup_preparse(fpl);
if (__predict_false(cache_fpl_terminated(fpl))) {
- goto out;
+ return (error);
}
for (;;) {
@@ -5043,39 +5075,8 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl)
cache_fplookup_parse_advance(fpl);
cache_fpl_checkpoint(fpl, &fpl->snd);
}
-out:
- switch (fpl->status) {
- case CACHE_FPL_STATUS_UNSET:
- __assert_unreachable();
- break;
- case CACHE_FPL_STATUS_PARTIAL:
- cache_fpl_smr_assert_entered(fpl);
- return (cache_fplookup_partial_setup(fpl));
- case CACHE_FPL_STATUS_ABORTED:
- if (fpl->in_smr)
- cache_fpl_smr_exit(fpl);
- return (CACHE_FPL_FAILED);
- case CACHE_FPL_STATUS_HANDLED:
- MPASS(error != CACHE_FPL_FAILED);
- cache_fpl_smr_assert_not_entered(fpl);
- /*
- * A common error is ENOENT.
- */
- if (error != 0) {
- ndp->ni_dvp = NULL;
- ndp->ni_vp = NULL;
- cache_fpl_cleanup_cnp(cnp);
- return (error);
- }
- ndp->ni_dvp = fpl->dvp;
- ndp->ni_vp = fpl->tvp;
- if (cnp->cn_flags & SAVENAME)
- cnp->cn_flags |= HASBUF;
- else
- cache_fpl_cleanup_cnp(cnp);
- return (error);
- }
- __assert_unreachable();
+
+ return (error);
}
/*
@@ -5162,10 +5163,10 @@ cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status,
struct pwd *pwd;
struct vnode *dvp;
struct componentname *cnp;
- struct nameidata_saved orig;
int error;
fpl.status = CACHE_FPL_STATUS_UNSET;
+ fpl.in_smr = false;
fpl.ndp = ndp;
fpl.cnp = cnp = &ndp->ni_cnd;
@@ -5185,12 +5186,13 @@ cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status,
return (EOPNOTSUPP);
}
- cache_fpl_checkpoint(&fpl, &orig);
+ cache_fpl_checkpoint(&fpl, &fpl.snd_orig);
cache_fpl_smr_enter_initial(&fpl);
fpl.fsearch = false;
+ fpl.pwd = pwdp;
pwd = pwd_get_smr();
- fpl.pwd = pwd;
+ *(fpl.pwd) = pwd;
ndp->ni_rootdir = pwd->pwd_rdir;
ndp->ni_topdir = pwd->pwd_jdir;
@@ -5209,31 +5211,34 @@ cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status,
}
SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true);
-
error = cache_fplookup_impl(dvp, &fpl);
out:
cache_fpl_smr_assert_not_entered(&fpl);
- SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
-
+ cache_fpl_assert_status(&fpl);
*status = fpl.status;
- switch (fpl.status) {
- case CACHE_FPL_STATUS_UNSET:
- __assert_unreachable();
- break;
- case CACHE_FPL_STATUS_HANDLED:
- if (error != 0)
- MPASS(ndp->ni_vp == NULL);
- SDT_PROBE3(vfs, namei, lookup, return, error, ndp->ni_vp, true);
- break;
- case CACHE_FPL_STATUS_PARTIAL:
- *pwdp = fpl.pwd;
+ if (SDT_PROBES_ENABLED()) {
+ SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
+ if (fpl.status == CACHE_FPL_STATUS_HANDLED)
+ SDT_PROBE3(vfs, namei, lookup, return, error, ndp->ni_vp, true);
+ }
+
+ if (__predict_true(fpl.status == CACHE_FPL_STATUS_HANDLED)) {
+ MPASS(error != CACHE_FPL_FAILED);
/*
- * Status restored by cache_fplookup_partial_setup.
+ * A common error is ENOENT.
*/
- break;
- case CACHE_FPL_STATUS_ABORTED:
- cache_fpl_restore_abort(&fpl, &orig);
- break;
+ if (error != 0) {
+ ndp->ni_dvp = NULL;
+ ndp->ni_vp = NULL;
+ cache_fpl_cleanup_cnp(cnp);
+ return (error);
+ }
+ ndp->ni_dvp = fpl.dvp;
+ ndp->ni_vp = fpl.tvp;
+ if (cnp->cn_flags & SAVENAME)
+ cnp->cn_flags |= HASBUF;
+ else
+ cache_fpl_cleanup_cnp(cnp);
}
return (error);
}
More information about the dev-commits-src-all
mailing list