cxl: Export library to support IBM XSL

This patch exports a in-kernel 'library' API which can be called by
other drivers to help interacting with an IBM XSL on a POWER9 system.

The XSL (Translation Service Layer) is a stripped down version of the
PSL (Power Service Layer) used in some cards such as the Mellanox CX5.
Like the PSL, it implements the CAIA architecture, but has a number
of differences, mostly in it's implementation dependent registers.

The XSL also uses a special DMA cxl mode, which uses a slightly
different init sequence for the CAPP and PHB.

Signed-off-by: Andrew Donnellan <[e-mail]>
Signed-off-by: Christophe Lombard <[e-mail]>
Acked-by: Frederic Barrat <[e-mail]>
Signed-off-by: Michael Ellerman <[e-mail]>

arch/powerpc/include/asm/opal-api.h

···
 	OPAL_PHB_CAPI_MODE_SNOOP_OFF    = 2,
 	OPAL_PHB_CAPI_MODE_SNOOP_ON	= 3,
 	OPAL_PHB_CAPI_MODE_DMA		= 4,
+	OPAL_PHB_CAPI_MODE_DMA_TVT1	= 5,
 };
 
 /* OPAL I2C request */

drivers/misc/cxl/Kconfig

···
 	bool
 	default n
 
+config CXL_LIB
+	bool
+	default n
+
 config CXL
 	tristate "Support for IBM Coherent Accelerators (CXL)"
 	depends on PPC_POWERNV && PCI_MSI && EEH
 	select CXL_BASE
 	select CXL_AFU_DRIVER_OPS
+	select CXL_LIB
 	default m
 	help
 	  Select this option to enable driver support for IBM Coherent

drivers/misc/cxl/Makefile

···
 
 cxl-y				+= main.o file.o irq.o fault.o native.o
 cxl-y				+= context.o sysfs.o pci.o trace.o
-cxl-y				+= vphb.o phb.o api.o
+cxl-y				+= vphb.o phb.o api.o cxllib.o
 cxl-$(CONFIG_PPC_PSERIES)	+= flash.o guest.o of.o hcalls.o
 cxl-$(CONFIG_DEBUG_FS)		+= debugfs.o
 obj-$(CONFIG_CXL)		+= cxl.o

drivers/misc/cxl/cxl.h

···
 
 void cxl_handle_fault(struct work_struct *work);
 void cxl_prefault(struct cxl_context *ctx, u64 wed);
+int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar);
 
 struct cxl *get_cxl_adapter(int num);
 int cxl_alloc_sst(struct cxl_context *ctx);
···
 int cxl_data_cache_flush(struct cxl *adapter);
 int cxl_afu_disable(struct cxl_afu *afu);
 int cxl_psl_purge(struct cxl_afu *afu);
+int cxl_calc_capp_routing(struct pci_dev *dev, u64 *chipid,
+			  u32 *phb_index, u64 *capp_unit_id);
+int cxl_slot_is_switched(struct pci_dev *dev);
+int cxl_get_xsl9_dsnctl(u64 capp_unit_id, u64 *reg);
+u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9);
 
 void cxl_native_irq_dump_regs_psl9(struct cxl_context *ctx);
 void cxl_native_irq_dump_regs_psl8(struct cxl_context *ctx);

drivers/misc/cxl/cxllib.c

···
+/*
+ * Copyright 2017 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/hugetlb.h>
+#include <linux/sched/mm.h>
+#include <asm/pnv-pci.h>
+#include <misc/cxllib.h>
+
+#include "cxl.h"
+
+#define CXL_INVALID_DRA                 ~0ull
+#define CXL_DUMMY_READ_SIZE             128
+#define CXL_DUMMY_READ_ALIGN            8
+#define CXL_CAPI_WINDOW_START           0x2000000000000ull
+#define CXL_CAPI_WINDOW_LOG_SIZE        48
+#define CXL_XSL_CONFIG_CURRENT_VERSION  CXL_XSL_CONFIG_VERSION1
+
+
+bool cxllib_slot_is_supported(struct pci_dev *dev, unsigned long flags)
+{
+	int rc;
+	u32 phb_index;
+	u64 chip_id, capp_unit_id;
+
+	/* No flags currently supported */
+	if (flags)
+		return false;
+
+	if (!cpu_has_feature(CPU_FTR_HVMODE))
+		return false;
+
+	if (!cxl_is_power9())
+		return false;
+
+	if (cxl_slot_is_switched(dev))
+		return false;
+
+	/* on p9, some pci slots are not connected to a CAPP unit */
+	rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
+	if (rc)
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL_GPL(cxllib_slot_is_supported);
+
+static DEFINE_MUTEX(dra_mutex);
+static u64 dummy_read_addr = CXL_INVALID_DRA;
+
+static int allocate_dummy_read_buf(void)
+{
+	u64 buf, vaddr;
+	size_t buf_size;
+
+	/*
+	 * Dummy read buffer is 128-byte long, aligned on a
+	 * 256-byte boundary and we need the physical address.
+	 */
+	buf_size = CXL_DUMMY_READ_SIZE + (1ull << CXL_DUMMY_READ_ALIGN);
+	buf = (u64) kzalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	vaddr = (buf + (1ull << CXL_DUMMY_READ_ALIGN) - 1) &
+					(~0ull << CXL_DUMMY_READ_ALIGN);
+
+	WARN((vaddr + CXL_DUMMY_READ_SIZE) > (buf + buf_size),
+		"Dummy read buffer alignment issue");
+	dummy_read_addr = virt_to_phys((void *) vaddr);
+	return 0;
+}
+
+int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg)
+{
+	int rc;
+	u32 phb_index;
+	u64 chip_id, capp_unit_id;
+
+	if (!cpu_has_feature(CPU_FTR_HVMODE))
+		return -EINVAL;
+
+	mutex_lock(&dra_mutex);
+	if (dummy_read_addr == CXL_INVALID_DRA) {
+		rc = allocate_dummy_read_buf();
+		if (rc) {
+			mutex_unlock(&dra_mutex);
+			return rc;
+		}
+	}
+	mutex_unlock(&dra_mutex);
+
+	rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
+	if (rc)
+		return rc;
+
+	rc = cxl_get_xsl9_dsnctl(capp_unit_id, &cfg->dsnctl);
+	if (rc)
+		return rc;
+	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
+		/* workaround for DD1 - nbwind = capiind */
+		cfg->dsnctl |= ((u64)0x02 << (63-47));
+	}
+
+	cfg->version  = CXL_XSL_CONFIG_CURRENT_VERSION;
+	cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE;
+	cfg->bar_addr = CXL_CAPI_WINDOW_START;
+	cfg->dra = dummy_read_addr;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cxllib_get_xsl_config);
+
+int cxllib_switch_phb_mode(struct pci_dev *dev, enum cxllib_mode mode,
+			unsigned long flags)
+{
+	int rc = 0;
+
+	if (!cpu_has_feature(CPU_FTR_HVMODE))
+		return -EINVAL;
+
+	switch (mode) {
+	case CXL_MODE_PCI:
+		/*
+		 * We currently don't support going back to PCI mode
+		 * However, we'll turn the invalidations off, so that
+		 * the firmware doesn't have to ack them and can do
+		 * things like reset, etc.. with no worries.
+		 * So always return EPERM (can't go back to PCI) or
+		 * EBUSY if we couldn't even turn off snooping
+		 */
+		rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_OFF);
+		if (rc)
+			rc = -EBUSY;
+		else
+			rc = -EPERM;
+		break;
+	case CXL_MODE_CXL:
+		/* DMA only supported on TVT1 for the time being */
+		if (flags != CXL_MODE_DMA_TVT1)
+			return -EINVAL;
+		rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_DMA_TVT1);
+		if (rc)
+			return rc;
+		rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON);
+		break;
+	default:
+		rc = -EINVAL;
+	}
+	return rc;
+}
+EXPORT_SYMBOL_GPL(cxllib_switch_phb_mode);
+
+/*
+ * When switching the PHB to capi mode, the TVT#1 entry for
+ * the Partitionable Endpoint is set in bypass mode, like
+ * in PCI mode.
+ * Configure the device dma to use TVT#1, which is done
+ * by calling dma_set_mask() with a mask large enough.
+ */
+int cxllib_set_device_dma(struct pci_dev *dev, unsigned long flags)
+{
+	int rc;
+
+	if (flags)
+		return -EINVAL;
+
+	rc = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
+	return rc;
+}
+EXPORT_SYMBOL_GPL(cxllib_set_device_dma);
+
+int cxllib_get_PE_attributes(struct task_struct *task,
+			     unsigned long translation_mode,
+			     struct cxllib_pe_attributes *attr)
+{
+	struct mm_struct *mm = NULL;
+
+	if (translation_mode != CXL_TRANSLATED_MODE &&
+		translation_mode != CXL_REAL_MODE)
+		return -EINVAL;
+
+	attr->sr = cxl_calculate_sr(false,
+				task == NULL,
+				translation_mode == CXL_REAL_MODE,
+				true);
+	attr->lpid = mfspr(SPRN_LPID);
+	if (task) {
+		mm = get_task_mm(task);
+		if (mm == NULL)
+			return -EINVAL;
+		/*
+		 * Caller is keeping a reference on mm_users for as long
+		 * as XSL uses the memory context
+		 */
+		attr->pid = mm->context.id;
+		mmput(mm);
+	} else {
+		attr->pid = 0;
+	}
+	attr->tid = 0;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cxllib_get_PE_attributes);
+
+int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags)
+{
+	int rc;
+	u64 dar;
+	struct vm_area_struct *vma = NULL;
+	unsigned long page_size;
+
+	if (mm == NULL)
+		return -EFAULT;
+
+	down_read(&mm->mmap_sem);
+
+	for (dar = addr; dar < addr + size; dar += page_size) {
+		if (!vma || dar < vma->vm_start || dar > vma->vm_end) {
+			vma = find_vma(mm, addr);
+			if (!vma) {
+				pr_err("Can't find vma for addr %016llx\n", addr);
+				rc = -EFAULT;
+				goto out;
+			}
+			/* get the size of the pages allocated */
+			page_size = vma_kernel_pagesize(vma);
+		}
+
+		rc = cxl_handle_mm_fault(mm, flags, dar);
+		if (rc) {
+			pr_err("cxl_handle_mm_fault failed %d", rc);
+			rc = -EFAULT;
+			goto out;
+		}
+	}
+	rc = 0;
+out:
+	up_read(&mm->mmap_sem);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(cxllib_handle_fault);

drivers/misc/cxl/fault.c

···
 	return IRQ_HANDLED;
 }
 
-static void cxl_handle_page_fault(struct cxl_context *ctx,
-				  struct mm_struct *mm, u64 dsisr, u64 dar)
+int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar)
 {
 	unsigned flt = 0;
 	int result;
 	unsigned long access, flags, inv_flags = 0;
 
-	trace_cxl_pte_miss(ctx, dsisr, dar);
-
 	if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) {
 		pr_devel("copro_handle_mm_fault failed: %#x\n", result);
-		return cxl_ack_ae(ctx);
+		return result;
 	}
 
 	if (!radix_enabled()) {
···
 		if (dsisr & CXL_PSL_DSISR_An_S)
 			access |= _PAGE_WRITE;
 
-		access |= _PAGE_PRIVILEGED;
-		if ((!ctx->kernel) || (REGION_ID(dar) == USER_REGION_ID))
-			access &= ~_PAGE_PRIVILEGED;
+		if (!mm && (REGION_ID(dar) != USER_REGION_ID))
+			access |= _PAGE_PRIVILEGED;
 
 		if (dsisr & DSISR_NOHPTE)
 			inv_flags |= HPTE_NOHPTE_UPDATE;
···
 		hash_page_mm(mm, dar, access, 0x300, inv_flags);
 		local_irq_restore(flags);
 	}
-	pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);
-	cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
+	return 0;
+}
+
+static void cxl_handle_page_fault(struct cxl_context *ctx,
+				  struct mm_struct *mm,
+				  u64 dsisr, u64 dar)
+{
+	trace_cxl_pte_miss(ctx, dsisr, dar);
+
+	if (cxl_handle_mm_fault(mm, dsisr, dar)) {
+		cxl_ack_ae(ctx);
+	} else {
+		pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);
+		cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
+	}
 }
 
 /*

drivers/misc/cxl/native.c

···
 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
 #endif
 
-static u64 calculate_sr(struct cxl_context *ctx)
+u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9)
 {
 	u64 sr = 0;
 
 	set_endian(sr);
-	if (ctx->master)
+	if (master)
 		sr |= CXL_PSL_SR_An_MP;
 	if (mfspr(SPRN_LPCR) & LPCR_TC)
 		sr |= CXL_PSL_SR_An_TC;
-	if (ctx->kernel) {
-		if (!ctx->real_mode)
+	if (kernel) {
+		if (!real_mode)
 			sr |= CXL_PSL_SR_An_R;
 		sr |= (mfmsr() & MSR_SF) | CXL_PSL_SR_An_HV;
 	} else {
···
 		if (!test_tsk_thread_flag(current, TIF_32BIT))
 			sr |= CXL_PSL_SR_An_SF;
 	}
-	if (cxl_is_power9()) {
+	if (p9) {
 		if (radix_enabled())
 			sr |= CXL_PSL_SR_An_XLAT_ror;
 		else
 			sr |= CXL_PSL_SR_An_XLAT_hpt;
 	}
 	return sr;
+}
+
+static u64 calculate_sr(struct cxl_context *ctx)
+{
+	return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode,
+				cxl_is_power9());
 }
 
 static void update_ivtes_directed(struct cxl_context *ctx)

drivers/misc/cxl/pci.c

···
 	return 0;
 }
 
-static int calc_capp_routing(struct pci_dev *dev, u64 *chipid,
+int cxl_calc_capp_routing(struct pci_dev *dev, u64 *chipid,
 			     u32 *phb_index, u64 *capp_unit_id)
 {
 	int rc;
···
 	return 0;
 }
 
-static int init_implementation_adapter_regs_psl9(struct cxl *adapter, struct pci_dev *dev)
+int cxl_get_xsl9_dsnctl(u64 capp_unit_id, u64 *reg)
 {
-	u64 xsl_dsnctl, psl_fircntl;
-	u64 chipid;
-	u32 phb_index;
-	u64 capp_unit_id;
-	int rc;
-
-	rc = calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
-	if (rc)
-		return rc;
+	u64 xsl_dsnctl;
 
 	/*
 	 * CAPI Identifier bits [0:7]
···
 		 */
 		xsl_dsnctl |= ((u64)0x04 << (63-55));
 	}
+
+	*reg = xsl_dsnctl;
+	return 0;
+}
+
+static int init_implementation_adapter_regs_psl9(struct cxl *adapter,
+						 struct pci_dev *dev)
+{
+	u64 xsl_dsnctl, psl_fircntl;
+	u64 chipid;
+	u32 phb_index;
+	u64 capp_unit_id;
+	int rc;
+
+	rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
+	if (rc)
+		return rc;
+
+	rc = cxl_get_xsl9_dsnctl(capp_unit_id, &xsl_dsnctl);
+	if (rc)
+		return rc;
 
 	cxl_p1_write(adapter, CXL_XSL9_DSNCTL, xsl_dsnctl);
 
···
 	u64 capp_unit_id;
 	int rc;
 
-	rc = calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
+	rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
 	if (rc)
 		return rc;
 
···
 	u64 capp_unit_id;
 	int rc;
 
-	rc = calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
+	rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
 	if (rc)
 		return rc;
 
···
 
 #define CXL_MAX_PCIEX_PARENT 2
 
-static int cxl_slot_is_switched(struct pci_dev *dev)
+int cxl_slot_is_switched(struct pci_dev *dev)
 {
 	struct device_node *np;
 	int depth = 0;

include/misc/cxllib.h

···
+/*
+ * Copyright 2017 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef _MISC_CXLLIB_H
+#define _MISC_CXLLIB_H
+
+#include <linux/pci.h>
+#include <asm/reg.h>
+
+/*
+ * cxl driver exports a in-kernel 'library' API which can be called by
+ * other drivers to help interacting with an IBM XSL.
+ */
+
+/*
+ * tells whether capi is supported on the PCIe slot where the
+ * device is seated
+ *
+ * Input:
+ *	dev: device whose slot needs to be checked
+ *	flags: 0 for the time being
+ */
+bool cxllib_slot_is_supported(struct pci_dev *dev, unsigned long flags);
+
+
+/*
+ * Returns the configuration parameters to be used by the XSL or device
+ *
+ * Input:
+ *	dev: device, used to find PHB
+ * Output:
+ *	struct cxllib_xsl_config:
+ *		version
+ *		capi BAR address, i.e. 0x2000000000000-0x2FFFFFFFFFFFF
+ *		capi BAR size
+ *		data send control (XSL_DSNCTL)
+ *		dummy read address (XSL_DRA)
+ */
+#define CXL_XSL_CONFIG_VERSION1		1
+struct cxllib_xsl_config {
+	u32	version;     /* format version for register encoding */
+	u32	log_bar_size;/* log size of the capi_window */
+	u64	bar_addr;    /* address of the start of capi window */
+	u64	dsnctl;      /* matches definition of XSL_DSNCTL */
+	u64	dra;         /* real address that can be used for dummy read */
+};
+
+int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg);
+
+
+/*
+ * Activate capi for the pci host bridge associated with the device.
+ * Can be extended to deactivate once we know how to do it.
+ * Device must be ready to accept messages from the CAPP unit and
+ * respond accordingly (TLB invalidates, ...)
+ *
+ * PHB is switched to capi mode through calls to skiboot.
+ * CAPP snooping is activated
+ *
+ * Input:
+ *	dev: device whose PHB should switch mode
+ *	mode: mode to switch to i.e. CAPI or PCI
+ *	flags: options related to the mode
+ */
+enum cxllib_mode {
+	CXL_MODE_CXL,
+	CXL_MODE_PCI,
+};
+
+#define CXL_MODE_NO_DMA       0
+#define CXL_MODE_DMA_TVT0     1
+#define CXL_MODE_DMA_TVT1     2
+
+int cxllib_switch_phb_mode(struct pci_dev *dev, enum cxllib_mode mode,
+			unsigned long flags);
+
+
+/*
+ * Set the device for capi DMA.
+ * Define its dma_ops and dma offset so that allocations will be using TVT#1
+ *
+ * Input:
+ *	dev: device to set
+ *	flags: options. CXL_MODE_DMA_TVT1 should be used
+ */
+int cxllib_set_device_dma(struct pci_dev *dev, unsigned long flags);
+
+
+/*
+ * Get the Process Element structure for the given thread
+ *
+ * Input:
+ *    task: task_struct for the context of the translation
+ *    translation_mode: whether addresses should be translated
+ * Output:
+ *    attr: attributes to fill up the Process Element structure from CAIA
+ */
+struct cxllib_pe_attributes {
+	u64 sr;
+	u32 lpid;
+	u32 tid;
+	u32 pid;
+};
+#define CXL_TRANSLATED_MODE 0
+#define CXL_REAL_MODE 1
+
+int cxllib_get_PE_attributes(struct task_struct *task,
+	     unsigned long translation_mode, struct cxllib_pe_attributes *attr);
+
+
+/*
+ * Handle memory fault.
+ * Fault in all the pages of the specified buffer for the permissions
+ * provided in ‘flags’
+ *
+ * Shouldn't be called from interrupt context
+ *
+ * Input:
+ *	mm: struct mm for the thread faulting the pages
+ *	addr: base address of the buffer to page in
+ *	size: size of the buffer to page in
+ *	flags: permission requested (DSISR_ISSTORE...)
+ */
+int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags);
+
+
+#endif /* _MISC_CXLLIB_H */