git: 70450ecd37fa - main - ntb: Add Intel Xeon Gen4 support
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 09 Feb 2024 16:49:58 UTC
The branch main has been updated by dab:
URL: https://cgit.FreeBSD.org/src/commit/?id=70450ecd37fa4ce06bd957195b00669dc3445e04
commit 70450ecd37fa4ce06bd957195b00669dc3445e04
Author: Austin Zhang <austin.zhang@dell.com>
AuthorDate: 2024-02-07 18:55:02 +0000
Commit: David Bright <dab@FreeBSD.org>
CommitDate: 2024-02-07 21:14:58 +0000
ntb: Add Intel Xeon Gen4 support
The NTB hardware of XEON Ice lake and Sapphire Rapids has register mapping changes
Add a new NTB_XEON_GEN4 device type and use it to conditionalize driver logic differs
Reviewed by: vangyzen, dab
Sponsored by: Dell Technologies
Differential Revision: https://reviews.freebsd.org/D43291
---
sys/dev/ntb/ntb_hw/ntb_hw_intel.c | 462 +++++++++++++++++++++++++++++++++++---
sys/dev/ntb/ntb_hw/ntb_hw_intel.h | 115 ++++++++++
2 files changed, 543 insertions(+), 34 deletions(-)
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw_intel.c b/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
index 5c07fb8d7e7d..d4852917085d 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
+++ b/sys/dev/ntb/ntb_hw/ntb_hw_intel.c
@@ -87,6 +87,7 @@
enum ntb_device_type {
NTB_XEON_GEN1,
NTB_XEON_GEN3,
+ NTB_XEON_GEN4,
NTB_ATOM
};
@@ -171,6 +172,7 @@ struct ntb_reg {
struct ntb_alt_reg {
uint32_t db_bell;
uint32_t db_mask;
+ uint32_t db_clear;
uint32_t spad;
};
@@ -356,9 +358,12 @@ static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb);
+static int intel_ntb_detect_xeon_gen4(struct ntb_softc *ntb);
+static int intel_ntb_detect_xeon_gen4_cfg(struct ntb_softc *ntb);
static int intel_ntb_detect_atom(struct ntb_softc *ntb);
static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb);
+static int intel_ntb_xeon_gen4_init_dev(struct ntb_softc *ntb);
static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
@@ -371,6 +376,7 @@ static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
static int xeon_setup_b2b_mw(struct ntb_softc *,
const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
static int xeon_gen3_setup_b2b_mw(struct ntb_softc *);
+static int xeon_gen4_setup_b2b_mw(struct ntb_softc *);
static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr,
size_t size);
static inline bool link_is_up(struct ntb_softc *ntb);
@@ -482,12 +488,15 @@ SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx,
#define NTB_SDOORBELL_LOCKUP (1 << 1)
#define NTB_SB01BASE_LOCKUP (1 << 2)
#define NTB_B2BDOORBELL_BIT14 (1 << 3)
+#define NTB_BAR_ALIGN (1 << 4)
+#define NTB_LTR_BAD (1 << 5)
/* Software/configuration owns the top 16 bits. */
#define NTB_SPLIT_BAR (1ull << 16)
#define NTB_ONE_MSIX (1ull << 17)
#define NTB_FEATURES_STR \
- "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
+ "\20\21SPLIT_BAR4\06LTR_BAD\05BAR_ALIGN" \
+ "\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
"\02SDOORBELL_LOCKUP\01BAR_SIZE_4K"
static struct ntb_hw_info pci_ids[] = {
@@ -511,6 +520,9 @@ static struct ntb_hw_info pci_ids[] = {
{ 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B",
NTB_XEON_GEN3, 0 },
+
+ { 0x347e8086, "ICX/SPR Xeon Non-Transparent Bridge B2B",
+ NTB_XEON_GEN4, 0 },
};
static const struct ntb_reg atom_reg = {
@@ -626,6 +638,32 @@ static const struct ntb_xlat_reg xeon_gen3_sec_xlat = {
.bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE,
};
+static const struct ntb_reg xeon_gen4_reg = {
+ .ntb_ctl = XEON_GEN4_REG_IMNTB_CTL,
+ .lnk_sta = XEON_GEN4_REG_LINK_STATUS, /* mmio */
+ .db_size = sizeof(uint32_t),
+ .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
+};
+
+static const struct ntb_alt_reg xeon_gen4_pri_reg = {
+ .db_clear = XEON_GEN4_REG_IMINT_STATUS,
+ .db_mask = XEON_GEN4_REG_IMINT_DISABLE,
+ .spad = XEON_GEN4_REG_IMSPAD,
+};
+
+static const struct ntb_alt_reg xeon_gen4_b2b_reg = {
+ .db_bell = XEON_GEN4_REG_IMDOORBELL,
+ .spad = XEON_GEN4_REG_EMSPAD,
+};
+
+static const struct ntb_xlat_reg xeon_gen4_sec_xlat = {
+ .bar2_limit = XEON_GEN4_REG_IMBAR1XLIMIT,
+ .bar2_xlat = XEON_GEN4_REG_IMBAR1XBASE,
+
+ .bar4_limit = XEON_GEN4_REG_IMBAR1XLIMIT,
+ .bar4_xlat = XEON_GEN4_REG_IMBAR2XBASE,
+};
+
SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"B2B MW segment overrides -- MUST be the same on both sides");
@@ -704,6 +742,8 @@ intel_ntb_attach(device_t device)
error = intel_ntb_detect_atom(ntb);
else if (ntb->type == NTB_XEON_GEN3)
error = intel_ntb_detect_xeon_gen3(ntb);
+ else if (ntb->type == NTB_XEON_GEN4)
+ error = intel_ntb_detect_xeon_gen4(ntb);
else
error = intel_ntb_detect_xeon(ntb);
if (error != 0)
@@ -720,6 +760,8 @@ intel_ntb_attach(device_t device)
error = intel_ntb_atom_init_dev(ntb);
else if (ntb->type == NTB_XEON_GEN3)
error = intel_ntb_xeon_gen3_init_dev(ntb);
+ else if (ntb->type == NTB_XEON_GEN4)
+ error = intel_ntb_xeon_gen4_init_dev(ntb);
else
error = intel_ntb_xeon_init_dev(ntb);
if (error != 0)
@@ -873,6 +915,10 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb)
bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ;
bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ;
bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE;
+ } else if (ntb->type == NTB_XEON_GEN4) {
+ bar->psz_off = XEON_GEN4_CFG_REG_IMBAR1SZ;
+ bar->ssz_off = XEON_GEN4_CFG_REG_EMBAR1SZ;
+ bar->pbarxlat_off = XEON_GEN4_REG_EXT_BAR1BASE;
} else {
bar->psz_off = XEON_PBAR23SZ_OFFSET;
bar->ssz_off = XEON_SBAR23SZ_OFFSET;
@@ -888,6 +934,10 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb)
bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ;
bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ;
bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE;
+ } else if (ntb->type == NTB_XEON_GEN4) {
+ bar->psz_off = XEON_GEN4_CFG_REG_IMBAR2SZ;
+ bar->ssz_off = XEON_GEN4_CFG_REG_EMBAR2SZ;
+ bar->pbarxlat_off = XEON_GEN4_REG_EXT_BAR2BASE;
} else {
bar->psz_off = XEON_PBAR4SZ_OFFSET;
bar->ssz_off = XEON_SBAR4SZ_OFFSET;
@@ -897,7 +947,8 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb)
if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
goto out;
- if (ntb->type == NTB_XEON_GEN3) {
+ if (ntb->type == NTB_XEON_GEN3 ||
+ ntb->type == NTB_XEON_GEN4) {
device_printf(ntb->device, "no split bar support\n");
return (ENXIO);
}
@@ -1176,7 +1227,63 @@ intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb)
return (ENXIO);
}
- return (0);
+ return (rc);
+}
+
+static int
+intel_ntb_xeon_gen4_init_isr(struct ntb_softc *ntb)
+{
+ uint64_t i, reg;
+ uint32_t desired_vectors, num_vectors;
+ int rc;
+
+ ntb->allocated_interrupts = 0;
+ ntb->last_ts = ticks;
+
+ /* Mask all the interrupts, including hardware interrupt */
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_DISABLE, ~0ULL);
+
+ /* Clear Interrupt Status */
+ reg = intel_ntb_reg_read(8, XEON_GEN4_REG_IMINT_STATUS);
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_STATUS, reg);
+
+ num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
+ XEON_GEN4_DB_MSIX_VECTOR_COUNT);
+
+ rc = pci_alloc_msix(ntb->device, &num_vectors);
+ if (rc != 0) {
+ device_printf(ntb->device,
+ "Interrupt allocation failed %d\n", rc);
+ return (rc);
+ }
+ if (desired_vectors != num_vectors) {
+ device_printf(ntb->device, "Couldn't get %d vectors\n",
+ XEON_GEN4_DB_MSIX_VECTOR_COUNT);
+ return (ENXIO);
+ }
+ if (num_vectors != XEON_GEN4_DB_MSIX_VECTOR_COUNT) {
+ device_printf(ntb->device,
+ "Need to remap interrupts, giving up\n");
+ return (ENXIO);
+ }
+
+ /*
+ * The MSIX vectors and the interrupt status bits are not lined up
+ * on Gen3 (Skylake) and Gen4. By default the link status bit is bit
+ * 32, however it is by default MSIX vector0. We need to fixup to
+ * line them up. The vectors at reset is 1-32,0. We need to reprogram
+ * to 0-32.
+ */
+ for (i = 0; i < XEON_GEN4_DB_MSIX_VECTOR_COUNT; i++)
+ intel_ntb_reg_write(1, XEON_GEN4_REG_INTVEC + i, i);
+
+ intel_ntb_create_msix_vec(ntb, num_vectors);
+ rc = intel_ntb_setup_msix(ntb, num_vectors);
+
+ /* enable all interrupts */
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_DISABLE, 0ULL);
+
+ return (rc);
}
static int
@@ -1304,6 +1411,7 @@ db_ioread(struct ntb_softc *ntb, uint64_t regoff)
switch (ntb->type) {
case NTB_ATOM:
case NTB_XEON_GEN3:
+ case NTB_XEON_GEN4:
return (intel_ntb_reg_read(8, regoff));
case NTB_XEON_GEN1:
return (intel_ntb_reg_read(2, regoff));
@@ -1332,6 +1440,7 @@ db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
switch (ntb->type) {
case NTB_ATOM:
case NTB_XEON_GEN3:
+ case NTB_XEON_GEN4:
intel_ntb_reg_write(8, regoff, val);
break;
case NTB_XEON_GEN1:
@@ -1448,7 +1557,8 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
ntb->last_ts = ticks;
vec_mask = intel_ntb_vec_mask(ntb, vec);
- if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX)
+ if ((ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4) &&
+ vec == XEON_GEN3_LINK_VECTOR_INDEX)
vec_mask |= ntb->db_link_mask;
if ((vec_mask & ntb->db_link_mask) != 0) {
if (intel_ntb_poll_link(ntb))
@@ -1456,6 +1566,9 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
if (ntb->type == NTB_XEON_GEN3)
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS,
intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
+ if (ntb->type == NTB_XEON_GEN4)
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_STATUS,
+ intel_ntb_reg_read(8, XEON_GEN4_REG_IMINT_STATUS));
}
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
@@ -1591,6 +1704,7 @@ intel_ntb_detect_max_mw(struct ntb_softc *ntb)
ntb->mw_count = XEON_SNB_MW_COUNT;
break;
case NTB_XEON_GEN3:
+ case NTB_XEON_GEN4:
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT;
else
@@ -1720,6 +1834,86 @@ intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb)
return (0);
}
+static int
+intel_ntb_is_ICX(struct ntb_softc *ntb)
+{
+ uint8_t revision;
+
+ revision = pci_get_revid(ntb->device);
+ if (ntb->type == NTB_XEON_GEN4 &&
+ revision >= PCI_DEV_REV_ICX_MIN &&
+ revision <= PCI_DEV_REV_ICX_MAX)
+ return (1);
+
+ return (0);
+}
+
+static int
+intel_ntb_is_SPR(struct ntb_softc *ntb)
+{
+ uint8_t revision;
+
+ revision = pci_get_revid(ntb->device);
+ if (ntb->type == NTB_XEON_GEN4 &&
+ revision > PCI_DEV_REV_ICX_MAX)
+ return (1);
+
+ return (0);
+}
+
+static int
+intel_ntb_detect_xeon_gen4(struct ntb_softc *ntb)
+{
+ if (intel_ntb_is_ICX(ntb)) {
+ ntb->features |= NTB_BAR_ALIGN;
+ ntb->features |= NTB_LTR_BAD;
+ }
+ return (0);
+}
+
+static int
+intel_ntb_detect_xeon_gen4_cfg(struct ntb_softc *ntb)
+{
+ uint32_t ppd1;
+
+ ppd1 = intel_ntb_reg_read(4, XEON_GEN4_REG_PPD1);
+ ntb->ppd = ppd1;
+ if (intel_ntb_is_ICX(ntb)) {
+ if ((ppd1 & GEN4_PPD_TOPO_MASK) == GEN4_PPD_TOPO_B2B_USD) {
+ /* NTB Port is configured as USD/DSP */
+ ntb->conn_type = NTB_CONN_B2B;
+ ntb->dev_type = NTB_DEV_USD;
+ } else if ((ppd1 & GEN4_PPD_TOPO_MASK) == GEN4_PPD_TOPO_B2B_DSD) {
+ /* NTB Port is configured as DSD/USP */
+ ntb->conn_type = NTB_CONN_B2B;
+ ntb->dev_type = NTB_DEV_DSD;
+ } else {
+ device_printf(ntb->device, "Unsupported connection type: %u\n",
+ (ppd1 & GEN4_PPD_CONN_MASK));
+ return (ENXIO);
+ }
+ } else if (intel_ntb_is_SPR(ntb)) {
+ if ((ppd1 & SPR_PPD_TOPO_MASK) == SPR_PPD_TOPO_B2B_USD) {
+ /* NTB Port is configured as USD/DSP */
+ ntb->conn_type = NTB_CONN_B2B;
+ ntb->dev_type = NTB_DEV_USD;
+ } else if ((ppd1 & SPR_PPD_TOPO_MASK) == SPR_PPD_TOPO_B2B_DSD) {
+ /* NTB Port is configured as DSD/USP */
+ ntb->conn_type = NTB_CONN_B2B;
+ ntb->dev_type = NTB_DEV_DSD;
+ } else {
+ device_printf(ntb->device, "Unsupported connection type: %u\n",
+ (ppd1 & SPR_PPD_CONN_MASK));
+ return (ENXIO);
+ }
+ }
+
+ device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x,"
+ "features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features);
+
+ return (0);
+}
+
static int
intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
{
@@ -1840,6 +2034,39 @@ intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb)
return (rc);
}
+static int
+intel_ntb_xeon_gen4_init_dev(struct ntb_softc *ntb)
+{
+ int rc;
+ uint16_t lnkctl;
+
+ ntb->spad_count = XEON_GEN4_SPAD_COUNT;
+ ntb->db_count = XEON_GEN4_DB_COUNT;
+ ntb->db_link_mask = XEON_GEN4_DB_LINK_BIT;
+ ntb->db_vec_count = XEON_GEN4_DB_MSIX_VECTOR_COUNT;
+ ntb->db_vec_shift = XEON_GEN4_DB_MSIX_VECTOR_SHIFT;
+
+ if (intel_ntb_detect_xeon_gen4_cfg(ntb) != 0)
+ return (ENXIO);
+
+ ntb->reg = &xeon_gen4_reg;
+ ntb->self_reg = &xeon_gen4_pri_reg;
+ ntb->peer_reg = &xeon_gen4_b2b_reg;
+ ntb->xlat_reg = &xeon_gen4_sec_xlat;
+
+ ntb->db_valid_mask = (1ULL << ntb->db_count) - 1;
+ xeon_gen4_setup_b2b_mw(ntb);
+
+ /* init link setup */
+ lnkctl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL);
+ lnkctl |= GEN4_LINK_CTRL_LINK_DISABLE;
+ intel_ntb_reg_write(2, XEON_GEN4_REG_LINK_CTRL, lnkctl);
+
+ /* Setup Interrupt */
+ rc = intel_ntb_xeon_gen4_init_isr(ntb);
+ return (rc);
+}
+
static int
intel_ntb_atom_init_dev(struct ntb_softc *ntb)
{
@@ -2181,6 +2408,47 @@ xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb)
return (0);
}
+static int
+xeon_gen4_setup_b2b_mw(struct ntb_softc *ntb)
+{
+ uint32_t embarsz, imbarsz;
+
+ /* IMBAR23SZ should be equal to EMBAR23SZ */
+ imbarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_IMBAR1SZ, 1);
+ embarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_EMBAR1SZ, 1);
+ if (embarsz != imbarsz) {
+ device_printf(ntb->device,
+ "IMBAR23SZ (%u) should be equal to EMBAR23SZ (%u)\n",
+ imbarsz, embarsz);
+ return (EIO);
+ }
+ /* IMBAR45SZ should be equal to EMBAR45SZ */
+ imbarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_IMBAR2SZ, 1);
+ embarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_EMBAR2SZ, 1);
+ if (embarsz != imbarsz) {
+ device_printf(ntb->device,
+ "IMBAR45SZ (%u) should be equal to EMBAR45SZ (%u)\n",
+ imbarsz, embarsz);
+ return (EIO);
+ }
+
+ /* Client will provide the incoming IMBARXBASE, zero it for now */
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR1XBASE, 0);
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR2XBASE, 0);
+
+ /*
+ * If the value in IMBARXLIMIT is set equal to the value in IMBARXBASE,
+ * the local memory window exposure from EMBAR is disabled.
+ * Note: It is needed to avoid malicious access.
+ */
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR1XLIMIT, 0);
+ intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR2XLIMIT, 0);
+
+ /* EMBARXLIMIT & EMBARXBASE are gone for gen4, noop here */
+
+ return (0);
+}
+
static inline bool
_xeon_link_is_up(struct ntb_softc *ntb)
{
@@ -2194,7 +2462,9 @@ static inline bool
link_is_up(struct ntb_softc *ntb)
{
- if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3)
+ if (ntb->type == NTB_XEON_GEN1 ||
+ ntb->type == NTB_XEON_GEN3 ||
+ ntb->type == NTB_XEON_GEN4)
return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good ||
!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));
@@ -2326,6 +2596,52 @@ intel_ntb_peer_port_idx(device_t dev, int port)
return (0);
}
+static int
+intel_ntb4_link_enable(device_t dev, enum ntb_speed speed __unused,
+ enum ntb_width width __unused)
+{
+ struct ntb_softc *ntb = device_get_softc(dev);
+ uint32_t cntl, ppd0, ltr;
+ uint16_t lnkctl;
+
+ if (!HAS_FEATURE(ntb, NTB_LTR_BAD)) {
+ /* Setup active snoop LTR values */
+ ltr = NTB_LTR_ACTIVE_REQMNT | NTB_LTR_ACTIVE_VAL | NTB_LTR_ACTIVE_LATSCALE;
+ /* Setup active non-snoop values */
+ ltr = (ltr << NTB_LTR_NS_SHIFT) | ltr;
+ intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_ACTIVE, ltr);
+
+ /* Setup idle snoop LTR values */
+ ltr = NTB_LTR_IDLE_VAL | NTB_LTR_IDLE_LATSCALE | NTB_LTR_IDLE_REQMNT;
+ /* Setup idle non-snoop values */
+ ltr = (ltr << NTB_LTR_NS_SHIFT) | ltr;
+ intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_IDLE, ltr);
+
+ /* setup PCIe LTR to active */
+ intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_SWSEL, NTB_LTR_SWSEL_ACTIVE);
+ }
+
+ cntl = NTB_CTL_E2I_BAR23_SNOOP | NTB_CTL_I2E_BAR23_SNOOP;
+ cntl |= NTB_CTL_E2I_BAR45_SNOOP | NTB_CTL_I2E_BAR45_SNOOP;
+ intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
+
+ lnkctl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL);
+ lnkctl &= ~GEN4_LINK_CTRL_LINK_DISABLE;
+ intel_ntb_reg_write(2, XEON_GEN4_REG_LINK_CTRL, lnkctl);
+
+ /* start link training in PPD0 */
+ ppd0 = intel_ntb_reg_read(4, XEON_GEN4_REG_PPD0);
+ ppd0 |= GEN4_PPD_LINKTRN;
+ intel_ntb_reg_write(4, XEON_GEN4_REG_PPD0, ppd0);
+
+ /* make sure link training has started */
+ ppd0 = intel_ntb_reg_read(4, XEON_GEN4_REG_PPD0);
+ if (!(ppd0 & GEN4_PPD_LINKTRN))
+ intel_ntb_printf(2, "Link is not training\n");
+
+ return (0);
+}
+
static int
intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused,
enum ntb_width width __unused)
@@ -2335,6 +2651,9 @@ intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused,
intel_ntb_printf(2, "%s\n", __func__);
+ if (ntb->type == NTB_XEON_GEN4)
+ return (intel_ntb4_link_enable(dev, speed, width));
+
if (ntb->type == NTB_ATOM) {
pci_write_config(ntb->device, NTB_PPD_OFFSET,
ntb->ppd | ATOM_PPD_INIT_LINK, 4);
@@ -2356,6 +2675,30 @@ intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused,
return (0);
}
+static int
+intel_ntb4_link_disable(device_t dev)
+{
+ struct ntb_softc *ntb = device_get_softc(dev);
+ uint32_t cntl;
+ uint16_t lnkctl;
+
+ /* clear the snoop bits */
+ cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
+ cntl &= ~(NTB_CTL_E2I_BAR23_SNOOP | NTB_CTL_I2E_BAR23_SNOOP);
+ cntl &= ~(NTB_CTL_E2I_BAR45_SNOOP | NTB_CTL_I2E_BAR45_SNOOP);
+ intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
+
+ lnkctl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL);
+ lnkctl |= GEN4_LINK_CTRL_LINK_DISABLE;
+ intel_ntb_reg_write(2, XEON_GEN4_REG_LINK_CTRL, lnkctl);
+
+ /* set LTR to idle */
+ if (!HAS_FEATURE(ntb, NTB_LTR_BAD))
+ intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_SWSEL, NTB_LTR_SWSEL_IDLE);
+
+ return (0);
+}
+
static int
intel_ntb_link_disable(device_t dev)
{
@@ -2364,6 +2707,9 @@ intel_ntb_link_disable(device_t dev)
intel_ntb_printf(2, "%s\n", __func__);
+ if (ntb->type == NTB_XEON_GEN4)
+ return (intel_ntb4_link_disable(dev));
+
if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
ntb_link_event(dev);
return (0);
@@ -2393,6 +2739,11 @@ intel_ntb_link_enabled(device_t dev)
if (ntb->conn_type == NTB_CONN_TRANSPARENT)
return (true);
+ if (ntb->type == NTB_XEON_GEN4) {
+ cntl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL);
+ return ((cntl & GEN4_LINK_CTRL_LINK_DISABLE) == 0);
+ }
+
cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
return ((cntl & NTB_CNTL_LINK_DISABLE) == 0);
}
@@ -2445,45 +2796,88 @@ retry:
* Polls the HW link status register(s); returns true if something has changed.
*/
static bool
-intel_ntb_poll_link(struct ntb_softc *ntb)
+intel_ntb_atom_poll_link(struct ntb_softc *ntb)
{
uint32_t ntb_cntl;
- uint16_t reg_val;
- if (ntb->type == NTB_ATOM) {
- ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
- if (ntb_cntl == ntb->ntb_ctl)
- return (false);
+ ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
+ if (ntb_cntl == ntb->ntb_ctl)
+ return (false);
- ntb->ntb_ctl = ntb_cntl;
- ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
- } else {
- if (ntb->type == NTB_XEON_GEN1)
- db_iowrite_raw(ntb, ntb->self_reg->db_bell,
- ntb->db_link_mask);
+ ntb->ntb_ctl = ntb_cntl;
+ ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
+ return (true);
+}
+
+static bool
+intel_ntb_xeon_gen1_poll_link(struct ntb_softc *ntb)
+{
+ uint16_t reg_val;
- reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
- if (reg_val == ntb->lnk_sta)
- return (false);
+ if (ntb->type == NTB_XEON_GEN1)
+ db_iowrite_raw(ntb, ntb->self_reg->db_bell,
+ ntb->db_link_mask);
- ntb->lnk_sta = reg_val;
+ reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
+ if (reg_val == ntb->lnk_sta)
+ return (false);
- if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
- if (_xeon_link_is_up(ntb)) {
- if (!ntb->peer_msix_good) {
- callout_reset(&ntb->peer_msix_work, 0,
- intel_ntb_exchange_msix, ntb);
- return (false);
- }
- } else {
- ntb->peer_msix_good = false;
- ntb->peer_msix_done = false;
+ ntb->lnk_sta = reg_val;
+
+ if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
+ if (_xeon_link_is_up(ntb)) {
+ if (!ntb->peer_msix_good) {
+ callout_reset(&ntb->peer_msix_work, 0,
+ intel_ntb_exchange_msix, ntb);
+ return (false);
}
+ } else {
+ ntb->peer_msix_good = false;
+ ntb->peer_msix_done = false;
}
}
return (true);
}
+static bool
+intel_ntb_xeon_gen4_poll_link(struct ntb_softc *ntb)
+{
+ uint16_t reg_val;
+
+ /*
+ * We need to write to DLLSCS bit in the SLOTSTS before we
+ * can clear the hardware link interrupt on ICX NTB.
+ */
+ intel_ntb_reg_write(2, XEON_GEN4_REG_SLOTSTS, GEN4_SLOTSTS_DLLSCS);
+ db_iowrite_raw(ntb, ntb->self_reg->db_clear, ntb->db_link_mask);
+
+ reg_val = intel_ntb_reg_read(2, ntb->reg->lnk_sta);
+ if (reg_val == ntb->lnk_sta)
+ return (false);
+
+ ntb->lnk_sta = reg_val;
+ return (true);
+}
+
+static bool
+intel_ntb_poll_link(struct ntb_softc *ntb)
+{
+ bool val;
+
+ switch(ntb->type) {
+ case NTB_ATOM:
+ val = intel_ntb_atom_poll_link(ntb);
+ break;
+ case NTB_XEON_GEN4:
+ val = intel_ntb_xeon_gen4_poll_link(ntb);
+ break;
+ default:
+ val = intel_ntb_xeon_gen1_poll_link(ntb);
+ break;
+ }
+ return (val);
+}
+
static inline enum ntb_speed
intel_ntb_link_sta_speed(struct ntb_softc *ntb)
{
@@ -3231,7 +3625,7 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
limit = 0;
if (bar_is_64bit(ntb, bar_num)) {
- if (ntb->type == NTB_XEON_GEN3)
+ if (ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4)
base = addr;
else
base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK;
@@ -3259,7 +3653,7 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
}
} else {
/* Configure 32-bit (split) BAR MW */
- if (ntb->type == NTB_XEON_GEN3)
+ if (ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4)
return (EIO);
if ((addr & UINT32_MAX) != addr)
@@ -3376,7 +3770,7 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bits)
return;
}
- if (ntb->type == NTB_XEON_GEN3) {
+ if (ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4) {
while (bits != 0) {
db = ffsll(bits);
diff --git a/sys/dev/ntb/ntb_hw/ntb_hw_intel.h b/sys/dev/ntb/ntb_hw/ntb_hw_intel.h
index c861b77743e3..9f9cadaa4d47 100644
--- a/sys/dev/ntb/ntb_hw/ntb_hw_intel.h
+++ b/sys/dev/ntb/ntb_hw/ntb_hw_intel.h
@@ -250,4 +250,119 @@
#define XEON_GEN3_REG_PPD_ONE_MSIX_F(X) M8_F(X, 5, 1)
#define XEON_GEN3_REG_PPD_BAR45_SPL_F(X) M8_F(X, 6, 1)
+/* Xeon ICX/SPR NTB register definitions */
+
+/* CFG Space */
+#define XEON_GEN4_CFG_REG_BAR0BASE 0x0010
+#define XEON_GEN4_CFG_REG_BAR1BASE 0x0018
+#define XEON_GEN4_CFG_REG_BAR2BASE 0x0020
+#define XEON_GEN4_CFG_REG_IMBAR1SZ 0x00c4
+#define XEON_GEN4_CFG_REG_IMBAR2SZ 0x00c5
+#define XEON_GEN4_CFG_REG_EMBAR1SZ 0x00c6
+#define XEON_GEN4_CFG_REG_EMBAR2SZ 0x00c7
+#define XEON_GEN4_CFG_REG_DEVCTRL 0x0048
+#define XEON_GEN4_CFG_REG_DEVSTS 0x004a
+#define XEON_GEN4_CFG_REG_UNCERRSTS 0x0104
+#define XEON_GEN4_CFG_REG_CORERRSTS 0x0110
+
+/* BAR0 MMIO */
+#define XEON_GEN4_REG_IMNTB_CTL 0x0000
+#define XEON_GEN4_REG_IMBAR1XBASE 0x0010
+#define XEON_GEN4_REG_IMBAR1XLIMIT 0x0018
+#define XEON_GEN4_REG_IMBAR2XBASE 0x0020
+#define XEON_GEN4_REG_IMBAR2XLIMIT 0x0028
+#define XEON_GEN4_REG_IMINT_STATUS 0x0040
+#define XEON_GEN4_REG_IMINT_DISABLE 0x0048
+#define XEON_GEN4_REG_INTVEC 0x0050 /* 0-32 vecs */
+#define XEON_GEN4_REG_IMSPAD 0x0080 /* 0-15 SPADs */
+#define XEON_GEN4_REG_IMDOORBELL 0x0100 /* 0-31 doorbells */
+
+/*
+ * External EndPoint Configuration Registers
+ * These are located within BAR0 of the internal endpoint.
+ */
+#define XEON_GEN4_REG_EXT_BAR1BASE 0x3018
+#define XEON_GEN4_REG_EXT_BAR2BASE 0x3020
+#define XEON_GEN4_REG_EXT_LTR_SWSEL 0x30ec
+#define XEON_GEN4_REG_EXT_LTR_ACTIVE 0x30f0
+#define XEON_GEN4_REG_EXT_LTR_IDLE 0x30f4
+
+#define XEON_GEN4_REG_EMSPAD 0x8080 /* 32K + SPAD_offset */
+
+/* note, link status is now in MMIO and not config space for NTB */
+#define XEON_GEN4_REG_LINK_CTRL 0xb050
+#define XEON_GEN4_REG_LINK_STATUS 0xb052
+#define XEON_GEN4_REG_SLOTSTS 0xb05a
+#define XEON_GEN4_REG_PPD0 0xb0d4
+#define XEON_GEN4_REG_PPD1 0xb4c0
+#define XEON_GEN4_REG_LTSSMSTATEJMP 0xf040
+
+/* Supported PCI device revision range for ICX */
+#define PCI_DEV_REV_ICX_MIN 0x2
+#define PCI_DEV_REV_ICX_MAX 0xF
+
+#define XEON_GEN4_DB_COUNT 32
+#define XEON_GEN4_DB_LINK 32
+#define XEON_GEN4_DB_LINK_BIT (1ULL << XEON_GEN4_DB_LINK)
+#define XEON_GEN4_DB_MSIX_VECTOR_COUNT 33
+#define XEON_GEN4_DB_MSIX_VECTOR_SHIFT 1
+#define XEON_GEN4_DB_TOTAL_SHIFT 33
+#define XEON_GEN4_SPAD_COUNT 16
+
+/* NTBCTL field */
+#define NTB_CTL_E2I_BAR23_SNOOP 0x000004
+#define NTB_CTL_E2I_BAR23_NOSNOOP 0x000008
+#define NTB_CTL_I2E_BAR23_SNOOP 0x000010
+#define NTB_CTL_I2E_BAR23_NOSNOOP 0x000020
+#define NTB_CTL_E2I_BAR45_SNOOP 0x000040
+#define NTB_CTL_E2I_BAR45_NOSNOO 0x000080
+#define NTB_CTL_I2E_BAR45_SNOOP 0x000100
+#define NTB_CTL_I2E_BAR45_NOSNOOP 0x000200
+#define NTB_CTL_BUSNO_DIS_INC 0x000400
+#define NTB_CTL_LINK_DOWN 0x010000
+
+#define NTB_SJC_FORCEDETECT 0x000004
+
+/* PPD field */
+#define GEN4_PPD_CLEAR_TRN 0x0001
+#define GEN4_PPD_LINKTRN 0x0008
+#define GEN4_PPD_CONN_MASK 0x0300
+#define SPR_PPD_CONN_MASK 0x0700
+#define GEN4_PPD_CONN_B2B 0x0200
+#define GEN4_PPD_DEV_MASK 0x1000
+#define GEN4_PPD_DEV_DSD 0x1000
+#define GEN4_PPD_DEV_USD 0x0000
+#define SPR_PPD_DEV_MASK 0x4000
+#define SPR_PPD_DEV_DSD 0x4000
+#define SPR_PPD_DEV_USD 0x0000
+
+#define GEN4_LINK_CTRL_LINK_DISABLE 0x0010
+#define GEN4_SLOTSTS_DLLSCS 0x100
+
+#define GEN4_PPD_TOPO_MASK \
+ (GEN4_PPD_CONN_MASK | GEN4_PPD_DEV_MASK)
+#define GEN4_PPD_TOPO_B2B_USD \
+ (GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_USD)
+#define GEN4_PPD_TOPO_B2B_DSD \
+ (GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_DSD)
+
+#define SPR_PPD_TOPO_MASK \
+ (SPR_PPD_CONN_MASK | SPR_PPD_DEV_MASK)
+#define SPR_PPD_TOPO_B2B_USD \
+ (GEN4_PPD_CONN_B2B | SPR_PPD_DEV_USD)
+#define SPR_PPD_TOPO_B2B_DSD \
+ (GEN4_PPD_CONN_B2B | SPR_PPD_DEV_DSD)
+
+/* LTR field */
+#define NTB_LTR_SWSEL_ACTIVE 0x0
+#define NTB_LTR_SWSEL_IDLE 0x1
+
+#define NTB_LTR_NS_SHIFT 16
+#define NTB_LTR_ACTIVE_VAL 0x0000 /* 0 us */
+#define NTB_LTR_ACTIVE_LATSCALE 0x0800 /* 1us scale */
+#define NTB_LTR_ACTIVE_REQMNT 0x8000 /* snoop req enable */
+
+#define NTB_LTR_IDLE_VAL 0x0258 /* 600 us */
+#define NTB_LTR_IDLE_LATSCALE 0x0800 /* 1us scale */
+#define NTB_LTR_IDLE_REQMNT 0x8000 /* snoop req enable */
#endif /* _NTB_REGS_H_ */