git: c0e0e530ced0 - main - mps/mpr: Add workaround for firmware not responding to IOC_FACTS or IOC_INIT

The branch main has been updated by imp:

URL: https://cgit.FreeBSD.org/src/commit/?id=c0e0e530ced057502f51d7a6086857305e08fae0

commit c0e0e530ced057502f51d7a6086857305e08fae0
Author:     prateek sethi <prateekrootkey@gmail.com>
AuthorDate: 2024-10-13 18:38:54 +0000
Commit:     Warner Losh <imp@FreeBSD.org>
CommitDate: 2024-10-13 21:38:01 +0000

    mps/mpr: Add workaround for firmware not responding to IOC_FACTS or IOC_INIT
    
    Sometimes, especially with older firmware, mps(4) would have trouble
    initializing the card in one of these two steps. Add in a retry after a
    short delay. Sean Bruno and Stephen McConnell thought this was OK in the
    bug discussions, but never committed it.  Steve indicated the delay
    might not be necessary, but the OP clearly needed to make it longer to
    make things work. I've kept the delay, and added the suggested comment.
    
    Ported the iocfacts part to mpr as well, since we see similar errors
    about once every month or two over a few thousand controllers at
    work. We've not seen it with IOC_INIT as far back as I can query the
    error log database, so I didn't port that forward. We'll see if this
    helps, but won't know for sure until next year (so I'm committing it now
    since it won't hurt and might help). We usually see this failure in
    connection with complicated recovery operations with a drive that's
    failing, though, at least in the last year's worth of failures. It's
    not clear this is the same as OP or not.
    
    PR: 212841
    Sponsored by: Netflix
    Co-authored-by: imp
---
 sys/dev/mpr/mpr.c | 23 ++++++++++++++++++-----
 sys/dev/mps/mps.c | 38 ++++++++++++++++++++++++++++++--------
 2 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/sys/dev/mpr/mpr.c b/sys/dev/mpr/mpr.c
index d5c02f9608ca..d1c572e40669 100644
--- a/sys/dev/mpr/mpr.c
+++ b/sys/dev/mpr/mpr.c
@@ -1195,7 +1195,7 @@ mpr_get_iocfacts(struct mpr_softc *sc, MPI2_IOC_FACTS_REPLY *facts)
 {
 	MPI2_DEFAULT_REPLY *reply;
 	MPI2_IOC_FACTS_REQUEST request;
-	int error, req_sz, reply_sz;
+	int error, req_sz, reply_sz, retry = 0;
 
 	MPR_FUNCTRACE(sc);
 	mpr_dprint(sc, MPR_INIT, "%s entered\n", __func__);
@@ -1204,13 +1204,26 @@ mpr_get_iocfacts(struct mpr_softc *sc, MPI2_IOC_FACTS_REPLY *facts)
 	reply_sz = sizeof(MPI2_IOC_FACTS_REPLY);
 	reply = (MPI2_DEFAULT_REPLY *)facts;
 
+	/*
+	 * Retry sending the initialization sequence. Sometimes, especially with
+	 * older firmware, the initialization process fails. Retrying allows the
+	 * error to clear in the firmware.
+	 */
 	bzero(&request, req_sz);
 	request.Function = MPI2_FUNCTION_IOC_FACTS;
-	error = mpr_request_sync(sc, &request, reply, req_sz, reply_sz, 5);
-
-	adjust_iocfacts_endianness(facts);
-	mpr_dprint(sc, MPR_TRACE, "facts->IOCCapabilities 0x%x\n", facts->IOCCapabilities);
+	while (retry < 5) {
+		error = mpr_request_sync(sc, &request, reply, req_sz, reply_sz, 5);
+		if (error == 0)
+			break;
+		mpr_dprint(sc, MPR_FAULT, "%s failed retry %d\n", __func__, retry);
+		DELAY(1000);
+                retry++;
+	}
 
+	if (error == 0) {
+		adjust_iocfacts_endianness(facts);
+		mpr_dprint(sc, MPR_TRACE, "facts->IOCCapabilities 0x%x\n", facts->IOCCapabilities);
+	}
 	mpr_dprint(sc, MPR_INIT, "%s exit, error= %d\n", __func__, error);
 	return (error);
 }
diff --git a/sys/dev/mps/mps.c b/sys/dev/mps/mps.c
index adad2450a3d4..9dfa0471ac0a 100644
--- a/sys/dev/mps/mps.c
+++ b/sys/dev/mps/mps.c
@@ -1127,7 +1127,7 @@ mps_get_iocfacts(struct mps_softc *sc, MPI2_IOC_FACTS_REPLY *facts)
 {
 	MPI2_DEFAULT_REPLY *reply;
 	MPI2_IOC_FACTS_REQUEST request;
-	int error, req_sz, reply_sz;
+	int error, req_sz, reply_sz, retry = 0;
 
 	MPS_FUNCTRACE(sc);
 	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);
@@ -1136,10 +1136,21 @@ mps_get_iocfacts(struct mps_softc *sc, MPI2_IOC_FACTS_REPLY *facts)
 	reply_sz = sizeof(MPI2_IOC_FACTS_REPLY);
 	reply = (MPI2_DEFAULT_REPLY *)facts;
 
+	/*
+	 * Retry sending the initialization sequence. Sometimes, especially with
+	 * older firmware, the initialization process fails. Retrying allows the
+	 * error to clear in the firmware.
+	 */
 	bzero(&request, req_sz);
 	request.Function = MPI2_FUNCTION_IOC_FACTS;
-	error = mps_request_sync(sc, &request, reply, req_sz, reply_sz, 5);
-	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);
+	while (retry < 5) {
+		error = mps_request_sync(sc, &request, reply, req_sz, reply_sz, 5);
+		if (error == 0)
+			break;
+		mps_dprint(sc, MPS_FAULT, "%s failed retry %d\n", __func__, retry);
+		DELAY(1000);
+                retry++;
+	}
 
 	return (error);
 }
@@ -1149,7 +1160,7 @@ mps_send_iocinit(struct mps_softc *sc)
 {
 	MPI2_IOC_INIT_REQUEST	init;
 	MPI2_DEFAULT_REPLY	reply;
-	int req_sz, reply_sz, error;
+	int req_sz, reply_sz, error, retry = 0;
 	struct timeval now;
 	uint64_t time_in_msec;
 
@@ -1193,10 +1204,21 @@ mps_send_iocinit(struct mps_softc *sc)
 	time_in_msec = (now.tv_sec * 1000 + now.tv_usec/1000);
 	init.TimeStamp.High = htole32((time_in_msec >> 32) & 0xFFFFFFFF);
 	init.TimeStamp.Low = htole32(time_in_msec & 0xFFFFFFFF);
-
-	error = mps_request_sync(sc, &init, &reply, req_sz, reply_sz, 5);
-	if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
-		error = ENXIO;
+	/*
+	 * Retry sending the initialization sequence. Sometimes, especially with
+	 * older firmware, the initialization process fails. Retrying allows the
+	 * error to clear in the firmware.
+	 */
+	while (retry < 5) {
+		error = mps_request_sync(sc, &init, &reply, req_sz, reply_sz, 5);
+		if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS)
+			error = ENXIO;
+                if (error == 0)
+                        break;
+                mps_dprint(sc, MPS_FAULT, "%s failed retry %d\n", __func__, retry);
+		DELAY(1000);
+                retry++;
+        }
 
 	mps_dprint(sc, MPS_INIT, "IOCInit status= 0x%x\n", reply.IOCStatus);
 	mps_dprint(sc, MPS_INIT, "%s exit\n", __func__);