Merge tag 'usb-for-v5.5' of git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb into usb-next

Felipe writes:

USB: changes for v5.5

We have TI's glue layer for the Cadence USB3 controller going
upstream. Tegra's XUDC driver is also going upstream with this pull
request.

Apart from these two big features, we have a bunch of patches switching
over to devm_platform_ioremap_resource() in order to simplify code a
little; and a non-critical fix for DWC3 usage via kexec.

* tag 'usb-for-v5.5' of git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb: (44 commits)
usb: dwc3: of-simple: add a shutdown
usb: cdns3: Add TI specific wrapper driver
dt-bindings: usb: Add binding for the TI wrapper for Cadence USB3 controller
usb: mtu3: fix race condition about delayed_status
usb: gadget: Add UDC driver for tegra XUSB device mode controller
usb: dwc3: debug: Remove newline printout
usb: dwc2: use a longer core rest timeout in dwc2_core_reset()
usb: gadget: udc: lpc32xx: Use devm_platform_ioremap_resource() in lpc32xx_udc_probe()
USB: gadget: udc: clean up an indentation issue
usb: gadget: Quieten gadget config message
phy: renesas: rcar-gen3-usb2: Use platform_get_irq_optional() for optional irq
usb: gadget: Remove set but not used variable 'opts' in msg_do_config
usb: gadget: Remove set but not used variable 'opts' in acm_ms_do_config
usb: mtu3: add a new function to do status stage
usb: gadget: configfs: fix concurrent issue between composite APIs
usb: gadget: f_tcm: Provide support to get alternate setting in tcm function
usb: gadget: Correct NULL pointer checking in fsl gadget
usb: fsl: Remove unused variable
USB: dummy-hcd: use usb_urb_dir_in instead of usb_pipein
USB: dummy-hcd: increase max number of devices to 32
...

Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml

···
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: "http://devicetree.org/schemas/usb/ti,j721e-usb.yaml#"
+$schema: "http://devicetree.org/meta-schemas/core.yaml#"
+
+title: Bindings for the TI wrapper module for the Cadence USBSS-DRD controller
+
+maintainers:
+  - Roger Quadros <[e-mail]>
+
+properties:
+  compatible:
+    items:
+      - const: ti,j721e-usb
+
+  reg:
+    description: module registers
+
+  power-domains:
+    description:
+       PM domain provider node and an args specifier containing
+       the USB device id value. See,
+       Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt
+
+  clocks:
+    description: Clock phandles to usb2_refclk and lpm_clk
+    minItems: 2
+    maxItems: 2
+
+  clock-names:
+    items:
+      - const: ref
+      - const: lpm
+
+  ti,usb2-only:
+    description:
+      If present, it restricts the controller to USB2.0 mode of
+      operation. Must be present if USB3 PHY is not available
+      for USB.
+    type: boolean
+
+  ti,vbus-divider:
+    description:
+      Should be present if USB VBUS line is connected to the
+      VBUS pin of the SoC via a 1/3 voltage divider.
+    type: boolean
+
+required:
+  - compatible
+  - reg
+  - power-domains
+  - clocks
+  - clock-names
+
+examples:
+  - |
+    #include <dt-bindings/soc/ti,sci_pm_domain.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    cdns_usb@4104000 {
+          compatible = "ti,j721e-usb";
+          reg = <0x00 0x4104000 0x00 0x100>;
+          power-domains = <&k3_pds 288 TI_SCI_PD_EXCLUSIVE>;
+          clocks = <&k3_clks 288 15>, <&k3_clks 288 3>;
+          clock-names = "ref", "lpm";
+          assigned-clocks = <&k3_clks 288 15>;	/* USB2_REFCLK */
+          assigned-clock-parents = <&k3_clks 288 16>; /* HFOSC0 */
+          #address-cells = <2>;
+          #size-cells = <2>;
+
+          usb@6000000 {
+                compatible = "cdns,usb3";
+                reg = <0x00 0x6000000 0x00 0x10000>,
+                      <0x00 0x6010000 0x00 0x10000>,
+                      <0x00 0x6020000 0x00 0x10000>;
+                reg-names = "otg", "xhci", "dev";
+                interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,	/* irq.0 */
+                             <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,	/* irq.6 */
+                             <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;	/* otgirq.0 */
+                interrupt-names = "host",
+                                  "peripheral",
+                                  "otg";
+                maximum-speed = "super-speed";
+                dr_mode = "otg";
+        };
+    };

drivers/phy/renesas/phy-rcar-gen3-usb2.c

···
 		return PTR_ERR(channel->base);
 
 	/* call request_irq for OTG */
-	irq = platform_get_irq(pdev, 0);
+	irq = platform_get_irq_optional(pdev, 0);
 	if (irq >= 0) {
 		INIT_WORK(&channel->work, rcar_gen3_phy_usb2_work);
 		irq = devm_request_irq(dev, irq, rcar_gen3_phy_usb2_irq,

drivers/usb/cdns3/Kconfig

···
 	  If you choose to build this driver as module it will
 	  be dynamically linked and module will be called cdns3-pci.ko
 
+config USB_CDNS3_TI
+	tristate "Cadence USB3 support on TI platforms"
+	depends on ARCH_K3 || COMPILE_TEST
+	default USB_CDNS3
+	help
+	  Say 'Y' or 'M' here if you are building for Texas Instruments
+	  platforms that contain Cadence USB3 controller core.
+
+	  e.g. J721e.
+
 endif

drivers/usb/cdns3/Makefile

···
 cdns3-$(CONFIG_USB_CDNS3_HOST)		+= host.o
 
 obj-$(CONFIG_USB_CDNS3_PCI_WRAP)	+= cdns3-pci-wrap.o
+obj-$(CONFIG_USB_CDNS3_TI)		+= cdns3-ti.o

drivers/usb/cdns3/cdns3-ti.c

···
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * cdns3-ti.c - TI specific Glue layer for Cadence USB Controller
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+
+/* USB Wrapper register offsets */
+#define USBSS_PID		0x0
+#define	USBSS_W1		0x4
+#define USBSS_STATIC_CONFIG	0x8
+#define USBSS_PHY_TEST		0xc
+#define	USBSS_DEBUG_CTRL	0x10
+#define	USBSS_DEBUG_INFO	0x14
+#define	USBSS_DEBUG_LINK_STATE	0x18
+#define	USBSS_DEVICE_CTRL	0x1c
+
+/* Wrapper 1 register bits */
+#define USBSS_W1_PWRUP_RST		BIT(0)
+#define USBSS_W1_OVERCURRENT_SEL	BIT(8)
+#define USBSS_W1_MODESTRAP_SEL		BIT(9)
+#define USBSS_W1_OVERCURRENT		BIT(16)
+#define USBSS_W1_MODESTRAP_MASK		GENMASK(18, 17)
+#define USBSS_W1_MODESTRAP_SHIFT	17
+#define USBSS_W1_USB2_ONLY		BIT(19)
+
+/* Static config register bits */
+#define USBSS1_STATIC_PLL_REF_SEL_MASK	GENMASK(8, 5)
+#define USBSS1_STATIC_PLL_REF_SEL_SHIFT	5
+#define USBSS1_STATIC_LOOPBACK_MODE_MASK	GENMASK(4, 3)
+#define USBSS1_STATIC_LOOPBACK_MODE_SHIFT	3
+#define USBSS1_STATIC_VBUS_SEL_MASK	GENMASK(2, 1)
+#define USBSS1_STATIC_VBUS_SEL_SHIFT	1
+#define USBSS1_STATIC_LANE_REVERSE	BIT(0)
+
+/* Modestrap modes */
+enum modestrap_mode { USBSS_MODESTRAP_MODE_NONE,
+		      USBSS_MODESTRAP_MODE_HOST,
+		      USBSS_MODESTRAP_MODE_PERIPHERAL};
+
+struct cdns_ti {
+	struct device *dev;
+	void __iomem *usbss;
+	int usb2_only:1;
+	int vbus_divider:1;
+	struct clk *usb2_refclk;
+	struct clk *lpm_clk;
+};
+
+static const int cdns_ti_rate_table[] = {	/* in KHZ */
+	9600,
+	10000,
+	12000,
+	19200,
+	20000,
+	24000,
+	25000,
+	26000,
+	38400,
+	40000,
+	58000,
+	50000,
+	52000,
+};
+
+static inline u32 cdns_ti_readl(struct cdns_ti *data, u32 offset)
+{
+	return readl(data->usbss + offset);
+}
+
+static inline void cdns_ti_writel(struct cdns_ti *data, u32 offset, u32 value)
+{
+	writel(value, data->usbss + offset);
+}
+
+static int cdns_ti_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = pdev->dev.of_node;
+	struct cdns_ti *data;
+	int error;
+	u32 reg;
+	int rate_code, i;
+	unsigned long rate;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, data);
+
+	data->dev = dev;
+
+	data->usbss = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(data->usbss)) {
+		dev_err(dev, "can't map IOMEM resource\n");
+		return PTR_ERR(data->usbss);
+	}
+
+	data->usb2_refclk = devm_clk_get(dev, "ref");
+	if (IS_ERR(data->usb2_refclk)) {
+		dev_err(dev, "can't get usb2_refclk\n");
+		return PTR_ERR(data->usb2_refclk);
+	}
+
+	data->lpm_clk = devm_clk_get(dev, "lpm");
+	if (IS_ERR(data->lpm_clk)) {
+		dev_err(dev, "can't get lpm_clk\n");
+		return PTR_ERR(data->lpm_clk);
+	}
+
+	rate = clk_get_rate(data->usb2_refclk);
+	rate /= 1000;	/* To KHz */
+	for (i = 0; i < ARRAY_SIZE(cdns_ti_rate_table); i++) {
+		if (cdns_ti_rate_table[i] == rate)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(cdns_ti_rate_table)) {
+		dev_err(dev, "unsupported usb2_refclk rate: %lu KHz\n", rate);
+		return -EINVAL;
+	}
+
+	rate_code = i;
+
+	pm_runtime_enable(dev);
+	error = pm_runtime_get_sync(dev);
+	if (error < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed: %d\n", error);
+		goto err_get;
+	}
+
+	/* assert RESET */
+	reg = cdns_ti_readl(data, USBSS_W1);
+	reg &= ~USBSS_W1_PWRUP_RST;
+	cdns_ti_writel(data, USBSS_W1, reg);
+
+	/* set static config */
+	reg = cdns_ti_readl(data, USBSS_STATIC_CONFIG);
+	reg &= ~USBSS1_STATIC_PLL_REF_SEL_MASK;
+	reg |= rate_code << USBSS1_STATIC_PLL_REF_SEL_SHIFT;
+
+	reg &= ~USBSS1_STATIC_VBUS_SEL_MASK;
+	data->vbus_divider = device_property_read_bool(dev, "ti,vbus-divider");
+	if (data->vbus_divider)
+		reg |= 1 << USBSS1_STATIC_VBUS_SEL_SHIFT;
+
+	cdns_ti_writel(data, USBSS_STATIC_CONFIG, reg);
+	reg = cdns_ti_readl(data, USBSS_STATIC_CONFIG);
+
+	/* set USB2_ONLY mode if requested */
+	reg = cdns_ti_readl(data, USBSS_W1);
+	data->usb2_only = device_property_read_bool(dev, "ti,usb2-only");
+	if (data->usb2_only)
+		reg |= USBSS_W1_USB2_ONLY;
+
+	/* set default modestrap */
+	reg |= USBSS_W1_MODESTRAP_SEL;
+	reg &= ~USBSS_W1_MODESTRAP_MASK;
+	reg |= USBSS_MODESTRAP_MODE_NONE << USBSS_W1_MODESTRAP_SHIFT;
+	cdns_ti_writel(data, USBSS_W1, reg);
+
+	/* de-assert RESET */
+	reg |= USBSS_W1_PWRUP_RST;
+	cdns_ti_writel(data, USBSS_W1, reg);
+
+	error = of_platform_populate(node, NULL, NULL, dev);
+	if (error) {
+		dev_err(dev, "failed to create children: %d\n", error);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	pm_runtime_put_sync(data->dev);
+err_get:
+	pm_runtime_disable(data->dev);
+
+	return error;
+}
+
+static int cdns_ti_remove_core(struct device *dev, void *c)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	platform_device_unregister(pdev);
+
+	return 0;
+}
+
+static int cdns_ti_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	device_for_each_child(dev, NULL, cdns_ti_remove_core);
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+static const struct of_device_id cdns_ti_of_match[] = {
+	{ .compatible = "ti,j721e-usb", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, cdns_ti_of_match);
+
+static struct platform_driver cdns_ti_driver = {
+	.probe		= cdns_ti_probe,
+	.remove		= cdns_ti_remove,
+	.driver		= {
+		.name	= "cdns3-ti",
+		.of_match_table	= cdns_ti_of_match,
+	},
+};
+
+module_platform_driver(cdns_ti_driver);
+
+MODULE_ALIAS("platform:cdns3-ti");
+MODULE_AUTHOR("Roger Quadros <[e-mail]>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Cadence USB3 TI Glue Layer");

drivers/usb/dwc2/core.c

···
 	greset |= GRSTCTL_CSFTRST;
 	dwc2_writel(hsotg, greset, GRSTCTL);
 
-	if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL, GRSTCTL_CSFTRST, 50)) {
+	if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL, GRSTCTL_CSFTRST, 10000)) {
 		dev_warn(hsotg->dev, "%s: HANG! Soft Reset timeout GRSTCTL GRSTCTL_CSFTRST\n",
 			 __func__);
 		return -EBUSY;

drivers/usb/dwc3/core.c

···
  */
 static int dwc3_phy_setup(struct dwc3 *dwc)
 {
+	unsigned int hw_mode;
 	u32 reg;
+
+	hw_mode = DWC3_GHWPARAMS0_MODE(dwc->hwparams.hwparams0);
 
 	reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
 
···
 	 */
 	if (dwc->revision > DWC3_REVISION_194A)
 		reg |= DWC3_GUSB3PIPECTL_SUSPHY;
+
+	/*
+	 * For DRD controllers, GUSB3PIPECTL.SUSPENDENABLE must be cleared after
+	 * power-on reset, and it can be set after core initialization, which is
+	 * after device soft-reset during initialization.
+	 */
+	if (hw_mode == DWC3_GHWPARAMS0_MODE_DRD)
+		reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
 
 	if (dwc->u2ss_inp3_quirk)
 		reg |= DWC3_GUSB3PIPECTL_U2SSINP3OK;
···
 	 */
 	if (dwc->revision > DWC3_REVISION_194A)
 		reg |= DWC3_GUSB2PHYCFG_SUSPHY;
+
+	/*
+	 * For DRD controllers, GUSB2PHYCFG.SUSPHY must be cleared after
+	 * power-on reset, and it can be set after core initialization, which is
+	 * after device soft-reset during initialization.
+	 */
+	if (hw_mode == DWC3_GHWPARAMS0_MODE_DRD)
+		reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 
 	if (dwc->dis_u2_susphy_quirk)
 		reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
···
  */
 static int dwc3_core_init(struct dwc3 *dwc)
 {
+	unsigned int		hw_mode;
 	u32			reg;
 	int			ret;
+
+	hw_mode = DWC3_GHWPARAMS0_MODE(dwc->hwparams.hwparams0);
 
 	/*
 	 * Write Linux Version Code to our GUID register so it's easy to figure
···
 	ret = dwc3_core_soft_reset(dwc);
 	if (ret)
 		goto err0a;
+
+	if (hw_mode == DWC3_GHWPARAMS0_MODE_DRD &&
+	    dwc->revision > DWC3_REVISION_194A) {
+		if (!dwc->dis_u3_susphy_quirk) {
+			reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
+			reg |= DWC3_GUSB3PIPECTL_SUSPHY;
+			dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
+		}
+
+		if (!dwc->dis_u2_susphy_quirk) {
+			reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
+			reg |= DWC3_GUSB2PHYCFG_SUSPHY;
+			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
+		}
+	}
 
 	dwc3_core_setup_global_control(dwc);
 	dwc3_core_num_eps(dwc);

drivers/usb/dwc3/debug.h

···
 	case DWC3_LINK_STATE_RESUME:
 		return "Resume";
 	default:
-		return "UNKNOWN link state\n";
+		return "UNKNOWN link state";
 	}
 }
 
···
 	case DWC3_LINK_STATE_RESUME:
 		return "Resume";
 	default:
-		return "UNKNOWN link state\n";
+		return "UNKNOWN link state";
 	}
 }
 

drivers/usb/dwc3/dwc3-of-simple.c

···
 	return ret;
 }
 
-static int dwc3_of_simple_remove(struct platform_device *pdev)
+static void __dwc3_of_simple_teardown(struct dwc3_of_simple *simple)
 {
-	struct dwc3_of_simple	*simple = platform_get_drvdata(pdev);
-	struct device		*dev = &pdev->dev;
-
-	of_platform_depopulate(dev);
+	of_platform_depopulate(simple->dev);
 
 	clk_bulk_disable_unprepare(simple->num_clocks, simple->clks);
 	clk_bulk_put_all(simple->num_clocks, simple->clks);
···
 
 	reset_control_put(simple->resets);
 
-	pm_runtime_disable(dev);
-	pm_runtime_put_noidle(dev);
-	pm_runtime_set_suspended(dev);
+	pm_runtime_disable(simple->dev);
+	pm_runtime_put_noidle(simple->dev);
+	pm_runtime_set_suspended(simple->dev);
+}
+
+static int dwc3_of_simple_remove(struct platform_device *pdev)
+{
+	struct dwc3_of_simple	*simple = platform_get_drvdata(pdev);
+
+	__dwc3_of_simple_teardown(simple);
 
 	return 0;
+}
+
+static void dwc3_of_simple_shutdown(struct platform_device *pdev)
+{
+	struct dwc3_of_simple	*simple = platform_get_drvdata(pdev);
+
+	__dwc3_of_simple_teardown(simple);
 }
 
 static int __maybe_unused dwc3_of_simple_runtime_suspend(struct device *dev)
···
 static struct platform_driver dwc3_of_simple_driver = {
 	.probe		= dwc3_of_simple_probe,
 	.remove		= dwc3_of_simple_remove,
+	.shutdown	= dwc3_of_simple_shutdown,
 	.driver		= {
 		.name	= "dwc3-of-simple",
 		.of_match_table = of_dwc3_simple_match,

drivers/usb/gadget/composite.c

···
 		result = 0;
 	}
 
-	INFO(cdev, "%s config #%d: %s\n",
-	     usb_speed_string(gadget->speed),
-	     number, c ? c->label : "unconfigured");
+	DBG(cdev, "%s config #%d: %s\n",
+	    usb_speed_string(gadget->speed),
+	    number, c ? c->label : "unconfigured");
 
 	if (!c)
 		goto done;

drivers/usb/gadget/function/f_acm.c

···
 	.release                = acm_attr_release,
 };
 
+#ifdef CONFIG_U_SERIAL_CONSOLE
+
+static ssize_t f_acm_console_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	return gserial_set_console(to_f_serial_opts(item)->port_num,
+				   page, count);
+}
+
+static ssize_t f_acm_console_show(struct config_item *item, char *page)
+{
+	return gserial_get_console(to_f_serial_opts(item)->port_num, page);
+}
+
+CONFIGFS_ATTR(f_acm_, console);
+
+#endif /* CONFIG_U_SERIAL_CONSOLE */
+
 static ssize_t f_acm_port_num_show(struct config_item *item, char *page)
 {
 	return sprintf(page, "%u\n", to_f_serial_opts(item)->port_num);
···
 CONFIGFS_ATTR_RO(f_acm_, port_num);
 
 static struct configfs_attribute *acm_attrs[] = {
+#ifdef CONFIG_U_SERIAL_CONSOLE
+	&f_acm_attr_console,
+#endif
 	&f_acm_attr_port_num,
 	NULL,
 };

drivers/usb/gadget/function/f_obex.c

···
 		return ERR_PTR(-ENOMEM);
 
 	opts->func_inst.free_func_inst = obex_free_inst;
-	ret = gserial_alloc_line(&opts->port_num);
+	ret = gserial_alloc_line_no_console(&opts->port_num);
 	if (ret) {
 		kfree(opts);
 		return ERR_PTR(ret);

drivers/usb/gadget/function/f_serial.c

···
 	.release	= serial_attr_release,
 };
 
+#ifdef CONFIG_U_SERIAL_CONSOLE
+
+static ssize_t f_serial_console_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	return gserial_set_console(to_f_serial_opts(item)->port_num,
+				   page, count);
+}
+
+static ssize_t f_serial_console_show(struct config_item *item, char *page)
+{
+	return gserial_get_console(to_f_serial_opts(item)->port_num, page);
+}
+
+CONFIGFS_ATTR(f_serial_, console);
+
+#endif /* CONFIG_U_SERIAL_CONSOLE */
+
 static ssize_t f_serial_port_num_show(struct config_item *item, char *page)
 {
 	return sprintf(page, "%u\n", to_f_serial_opts(item)->port_num);
···
 CONFIGFS_ATTR_RO(f_serial_, port_num);
 
 static struct configfs_attribute *acm_attrs[] = {
+#ifdef CONFIG_U_SERIAL_CONSOLE
+	&f_serial_attr_console,
+#endif
 	&f_serial_attr_port_num,
 	NULL,
 };

drivers/usb/gadget/function/f_tcm.c

···
 
 	fu->flags = USBG_IS_UAS;
 
-	if (gadget->speed == USB_SPEED_SUPER)
+	if (gadget->speed >= USB_SPEED_SUPER)
 		fu->flags |= USBG_USE_STREAMS;
 
 	config_ep_by_speed(gadget, f, fu->ep_in);
···
 	usb_composite_setup_continue(fu->function.config->cdev);
 }
 
+static int tcm_get_alt(struct usb_function *f, unsigned intf)
+{
+	if (intf == bot_intf_desc.bInterfaceNumber)
+		return USB_G_ALT_INT_BBB;
+	if (intf == uasp_intf_desc.bInterfaceNumber)
+		return USB_G_ALT_INT_UAS;
+
+	return -EOPNOTSUPP;
+}
+
 static int tcm_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
 {
 	struct f_uas *fu = to_f_uas(f);
···
 	fu->function.bind = tcm_bind;
 	fu->function.unbind = tcm_unbind;
 	fu->function.set_alt = tcm_set_alt;
+	fu->function.get_alt = tcm_get_alt;
 	fu->function.setup = tcm_setup;
 	fu->function.disable = tcm_disable;
 	fu->function.free_func = tcm_free;

drivers/usb/gadget/function/u_serial.c

···
 #define GS_CONSOLE_BUF_SIZE	8192
 
 /* console info */
-struct gscons_info {
-	struct gs_port		*port;
-	struct task_struct	*console_thread;
-	struct kfifo		con_buf;
-	/* protect the buf and busy flag */
-	spinlock_t		con_lock;
-	int			req_busy;
-	struct usb_request	*console_req;
+struct gs_console {
+	struct console		console;
+	struct work_struct	work;
+	spinlock_t		lock;
+	struct usb_request	*req;
+	struct kfifo		buf;
+	size_t			missed;
 };
 
 /*
···
 	spinlock_t		port_lock;	/* guard port_* access */
 
 	struct gserial		*port_usb;
+#ifdef CONFIG_U_SERIAL_CONSOLE
+	struct gs_console	*console;
+#endif
 
-	bool			openclose;	/* open/close in progress */
 	u8			port_num;
 
 	struct list_head	read_pool;
···
 {
 	int		port_num = tty->index;
 	struct gs_port	*port;
-	int		status;
+	int		status = 0;
 
-	do {
-		mutex_lock(&ports[port_num].lock);
-		port = ports[port_num].port;
-		if (!port)
-			status = -ENODEV;
-		else {
-			spin_lock_irq(&port->port_lock);
+	mutex_lock(&ports[port_num].lock);
+	port = ports[port_num].port;
+	if (!port) {
+		status = -ENODEV;
+		goto out;
+	}
 
-			/* already open?  Great. */
-			if (port->port.count) {
-				status = 0;
-				port->port.count++;
-
-			/* currently opening/closing? wait ... */
-			} else if (port->openclose) {
-				status = -EBUSY;
-
-			/* ... else we do the work */
-			} else {
-				status = -EAGAIN;
-				port->openclose = true;
-			}
-			spin_unlock_irq(&port->port_lock);
-		}
-		mutex_unlock(&ports[port_num].lock);
-
-		switch (status) {
-		default:
-			/* fully handled */
-			return status;
-		case -EAGAIN:
-			/* must do the work */
-			break;
-		case -EBUSY:
-			/* wait for EAGAIN task to finish */
-			msleep(1);
-			/* REVISIT could have a waitchannel here, if
-			 * concurrent open performance is important
-			 */
-			break;
-		}
-	} while (status != -EAGAIN);
-
-	/* Do the "real open" */
 	spin_lock_irq(&port->port_lock);
 
 	/* allocate circular buffer on first open */
 	if (!kfifo_initialized(&port->port_write_buf)) {
 
 		spin_unlock_irq(&port->port_lock);
+
+		/*
+		 * portmaster's mutex still protects from simultaneous open(),
+		 * and close() can't happen, yet.
+		 */
+
 		status = kfifo_alloc(&port->port_write_buf,
 				     WRITE_BUF_SIZE, GFP_KERNEL);
-		spin_lock_irq(&port->port_lock);
-
 		if (status) {
 			pr_debug("gs_open: ttyGS%d (%p,%p) no buffer\n",
-				port->port_num, tty, file);
-			port->openclose = false;
-			goto exit_unlock_port;
+				 port_num, tty, file);
+			goto out;
 		}
+
+		spin_lock_irq(&port->port_lock);
 	}
 
-	/* REVISIT if REMOVED (ports[].port NULL), abort the open
-	 * to let rmmod work faster (but this way isn't wrong).
-	 */
-
-	/* REVISIT maybe wait for "carrier detect" */
+	/* already open?  Great. */
+	if (port->port.count++)
+		goto exit_unlock_port;
 
 	tty->driver_data = port;
 	port->port.tty = tty;
-
-	port->port.count = 1;
-	port->openclose = false;
 
 	/* if connected, start the I/O stream */
 	if (port->port_usb) {
···
 
 	pr_debug("gs_open: ttyGS%d (%p,%p)\n", port->port_num, tty, file);
 
-	status = 0;
-
 exit_unlock_port:
 	spin_unlock_irq(&port->port_lock);
+out:
+	mutex_unlock(&ports[port_num].lock);
 	return status;
 }
 
-static int gs_writes_finished(struct gs_port *p)
+static int gs_close_flush_done(struct gs_port *p)
 {
 	int cond;
 
-	/* return true on disconnect or empty buffer */
+	/* return true on disconnect or empty buffer or if raced with open() */
 	spin_lock_irq(&p->port_lock);
-	cond = (p->port_usb == NULL) || !kfifo_len(&p->port_write_buf);
+	cond = p->port_usb == NULL || !kfifo_len(&p->port_write_buf) ||
+		p->port.count > 1;
 	spin_unlock_irq(&p->port_lock);
 
 	return cond;
···
 	spin_lock_irq(&port->port_lock);
 
 	if (port->port.count != 1) {
+raced_with_open:
 		if (port->port.count == 0)
 			WARN_ON(1);
 		else
···
 	}
 
 	pr_debug("gs_close: ttyGS%d (%p,%p) ...\n", port->port_num, tty, file);
-
-	/* mark port as closing but in use; we can drop port lock
-	 * and sleep if necessary
-	 */
-	port->openclose = true;
-	port->port.count = 0;
 
 	gser = port->port_usb;
 	if (gser && gser->disconnect)
···
 	if (kfifo_len(&port->port_write_buf) > 0 && gser) {
 		spin_unlock_irq(&port->port_lock);
 		wait_event_interruptible_timeout(port->drain_wait,
-					gs_writes_finished(port),
+					gs_close_flush_done(port),
 					GS_CLOSE_TIMEOUT * HZ);
 		spin_lock_irq(&port->port_lock);
+
+		if (port->port.count != 1)
+			goto raced_with_open;
+
 		gser = port->port_usb;
 	}
 
···
 	else
 		kfifo_reset(&port->port_write_buf);
 
+	port->port.count = 0;
 	port->port.tty = NULL;
-
-	port->openclose = false;
 
 	pr_debug("gs_close: ttyGS%d (%p,%p) done!\n",
 			port->port_num, tty, file);
···
 
 #ifdef CONFIG_U_SERIAL_CONSOLE
 
-static struct gscons_info gscons_info;
-static struct console gserial_cons;
-
-static struct usb_request *gs_request_new(struct usb_ep *ep)
+static void gs_console_complete_out(struct usb_ep *ep, struct usb_request *req)
 {
-	struct usb_request *req = usb_ep_alloc_request(ep, GFP_ATOMIC);
-	if (!req)
-		return NULL;
-
-	req->buf = kmalloc(ep->maxpacket, GFP_ATOMIC);
-	if (!req->buf) {
-		usb_ep_free_request(ep, req);
-		return NULL;
-	}
-
-	return req;
-}
-
-static void gs_request_free(struct usb_request *req, struct usb_ep *ep)
-{
-	if (!req)
-		return;
-
-	kfree(req->buf);
-	usb_ep_free_request(ep, req);
-}
-
-static void gs_complete_out(struct usb_ep *ep, struct usb_request *req)
-{
-	struct gscons_info *info = &gscons_info;
+	struct gs_console *cons = req->context;
 
 	switch (req->status) {
 	default:
···
 		/* fall through */
 	case 0:
 		/* normal completion */
-		spin_lock(&info->con_lock);
-		info->req_busy = 0;
-		spin_unlock(&info->con_lock);
-
-		wake_up_process(info->console_thread);
+		spin_lock(&cons->lock);
+		req->length = 0;
+		schedule_work(&cons->work);
+		spin_unlock(&cons->lock);
 		break;
+	case -ECONNRESET:
 	case -ESHUTDOWN:
 		/* disconnect */
 		pr_vdebug("%s: %s shutdown\n", __func__, ep->name);
···
 	}
 }
 
-static int gs_console_connect(int port_num)
+static void __gs_console_push(struct gs_console *cons)
 {
-	struct gscons_info *info = &gscons_info;
-	struct gs_port *port;
+	struct usb_request *req = cons->req;
 	struct usb_ep *ep;
+	size_t size;
 
-	if (port_num != gserial_cons.index) {
-		pr_err("%s: port num [%d] is not support console\n",
-		       __func__, port_num);
-		return -ENXIO;
+	if (!req)
+		return;	/* disconnected */
+
+	if (req->length)
+		return;	/* busy */
+
+	ep = cons->console.data;
+	size = kfifo_out(&cons->buf, req->buf, ep->maxpacket);
+	if (!size)
+		return;
+
+	if (cons->missed && ep->maxpacket >= 64) {
+		char buf[64];
+		size_t len;
+
+		len = sprintf(buf, "\n[missed %zu bytes]\n", cons->missed);
+		kfifo_in(&cons->buf, buf, len);
+		cons->missed = 0;
 	}
 
-	port = ports[port_num].port;
-	ep = port->port_usb->in;
-	if (!info->console_req) {
-		info->console_req = gs_request_new(ep);
-		if (!info->console_req)
-			return -ENOMEM;
-		info->console_req->complete = gs_complete_out;
-	}
-
-	info->port = port;
-	spin_lock(&info->con_lock);
-	info->req_busy = 0;
-	spin_unlock(&info->con_lock);
-	pr_vdebug("port[%d] console connect!\n", port_num);
-	return 0;
+	req->length = size;
+	if (usb_ep_queue(ep, req, GFP_ATOMIC))
+		req->length = 0;
 }
 
-static void gs_console_disconnect(struct usb_ep *ep)
+static void gs_console_work(struct work_struct *work)
 {
-	struct gscons_info *info = &gscons_info;
-	struct usb_request *req = info->console_req;
+	struct gs_console *cons = container_of(work, struct gs_console, work);
 
-	gs_request_free(req, ep);
-	info->console_req = NULL;
-}
+	spin_lock_irq(&cons->lock);
 
-static int gs_console_thread(void *data)
-{
-	struct gscons_info *info = &gscons_info;
-	struct gs_port *port;
-	struct usb_request *req;
-	struct usb_ep *ep;
-	int xfer, ret, count, size;
+	__gs_console_push(cons);
 
-	do {
-		port = info->port;
-		set_current_state(TASK_INTERRUPTIBLE);
-		if (!port || !port->port_usb
-		    || !port->port_usb->in || !info->console_req)
-			goto sched;
-
-		req = info->console_req;
-		ep = port->port_usb->in;
-
-		spin_lock_irq(&info->con_lock);
-		count = kfifo_len(&info->con_buf);
-		size = ep->maxpacket;
-
-		if (count > 0 && !info->req_busy) {
-			set_current_state(TASK_RUNNING);
-			if (count < size)
-				size = count;
-
-			xfer = kfifo_out(&info->con_buf, req->buf, size);
-			req->length = xfer;
-
-			spin_unlock(&info->con_lock);
-			ret = usb_ep_queue(ep, req, GFP_ATOMIC);
-			spin_lock(&info->con_lock);
-			if (ret < 0)
-				info->req_busy = 0;
-			else
-				info->req_busy = 1;
-
-			spin_unlock_irq(&info->con_lock);
-		} else {
-			spin_unlock_irq(&info->con_lock);
-sched:
-			if (kthread_should_stop()) {
-				set_current_state(TASK_RUNNING);
-				break;
-			}
-			schedule();
-		}
-	} while (1);
-
-	return 0;
-}
-
-static int gs_console_setup(struct console *co, char *options)
-{
-	struct gscons_info *info = &gscons_info;
-	int status;
-
-	info->port = NULL;
-	info->console_req = NULL;
-	info->req_busy = 0;
-	spin_lock_init(&info->con_lock);
-
-	status = kfifo_alloc(&info->con_buf, GS_CONSOLE_BUF_SIZE, GFP_KERNEL);
-	if (status) {
-		pr_err("%s: allocate console buffer failed\n", __func__);
-		return status;
-	}
-
-	info->console_thread = kthread_create(gs_console_thread,
-					      co, "gs_console");
-	if (IS_ERR(info->console_thread)) {
-		pr_err("%s: cannot create console thread\n", __func__);
-		kfifo_free(&info->con_buf);
-		return PTR_ERR(info->console_thread);
-	}
-	wake_up_process(info->console_thread);
-
-	return 0;
+	spin_unlock_irq(&cons->lock);
 }
 
 static void gs_console_write(struct console *co,
 			     const char *buf, unsigned count)
 {
-	struct gscons_info *info = &gscons_info;
+	struct gs_console *cons = container_of(co, struct gs_console, console);
 	unsigned long flags;
+	size_t n;
 
-	spin_lock_irqsave(&info->con_lock, flags);
-	kfifo_in(&info->con_buf, buf, count);
-	spin_unlock_irqrestore(&info->con_lock, flags);
+	spin_lock_irqsave(&cons->lock, flags);
 
-	wake_up_process(info->console_thread);
+	n = kfifo_in(&cons->buf, buf, count);
+	if (n < count)
+		cons->missed += count - n;
+
+	if (cons->req && !cons->req->length)
+		schedule_work(&cons->work);
+
+	spin_unlock_irqrestore(&cons->lock, flags);
 }
 
 static struct tty_driver *gs_console_device(struct console *co, int *index)
 {
-	struct tty_driver **p = (struct tty_driver **)co->data;
-
-	if (!*p)
-		return NULL;
-
 	*index = co->index;
-	return *p;
+	return gs_tty_driver;
 }
 
-static struct console gserial_cons = {
-	.name =		"ttyGS",
-	.write =	gs_console_write,
-	.device =	gs_console_device,
-	.setup =	gs_console_setup,
-	.flags =	CON_PRINTBUFFER,
-	.index =	-1,
-	.data =		&gs_tty_driver,
-};
-
-static void gserial_console_init(void)
+static int gs_console_connect(struct gs_port *port)
 {
-	register_console(&gserial_cons);
+	struct gs_console *cons = port->console;
+	struct usb_request *req;
+	struct usb_ep *ep;
+
+	if (!cons)
+		return 0;
+
+	ep = port->port_usb->in;
+	req = gs_alloc_req(ep, ep->maxpacket, GFP_ATOMIC);
+	if (!req)
+		return -ENOMEM;
+	req->complete = gs_console_complete_out;
+	req->context = cons;
+	req->length = 0;
+
+	spin_lock(&cons->lock);
+	cons->req = req;
+	cons->console.data = ep;
+	spin_unlock(&cons->lock);
+
+	pr_debug("ttyGS%d: console connected!\n", port->port_num);
+
+	schedule_work(&cons->work);
+
+	return 0;
 }
 
-static void gserial_console_exit(void)
+static void gs_console_disconnect(struct gs_port *port)
 {
-	struct gscons_info *info = &gscons_info;
+	struct gs_console *cons = port->console;
+	struct usb_request *req;
+	struct usb_ep *ep;
 
-	unregister_console(&gserial_cons);
-	if (!IS_ERR_OR_NULL(info->console_thread))
-		kthread_stop(info->console_thread);
-	kfifo_free(&info->con_buf);
+	if (!cons)
+		return;
+
+	spin_lock(&cons->lock);
+
+	req = cons->req;
+	ep = cons->console.data;
+	cons->req = NULL;
+
+	spin_unlock(&cons->lock);
+
+	if (!req)
+		return;
+
+	usb_ep_dequeue(ep, req);
+	gs_free_req(ep, req);
 }
+
+static int gs_console_init(struct gs_port *port)
+{
+	struct gs_console *cons;
+	int err;
+
+	if (port->console)
+		return 0;
+
+	cons = kzalloc(sizeof(*port->console), GFP_KERNEL);
+	if (!cons)
+		return -ENOMEM;
+
+	strcpy(cons->console.name, "ttyGS");
+	cons->console.write = gs_console_write;
+	cons->console.device = gs_console_device;
+	cons->console.flags = CON_PRINTBUFFER;
+	cons->console.index = port->port_num;
+
+	INIT_WORK(&cons->work, gs_console_work);
+	spin_lock_init(&cons->lock);
+
+	err = kfifo_alloc(&cons->buf, GS_CONSOLE_BUF_SIZE, GFP_KERNEL);
+	if (err) {
+		pr_err("ttyGS%d: allocate console buffer failed\n", port->port_num);
+		kfree(cons);
+		return err;
+	}
+
+	port->console = cons;
+	register_console(&cons->console);
+
+	spin_lock_irq(&port->port_lock);
+	if (port->port_usb)
+		gs_console_connect(port);
+	spin_unlock_irq(&port->port_lock);
+
+	return 0;
+}
+
+static void gs_console_exit(struct gs_port *port)
+{
+	struct gs_console *cons = port->console;
+
+	if (!cons)
+		return;
+
+	unregister_console(&cons->console);
+
+	spin_lock_irq(&port->port_lock);
+	if (cons->req)
+		gs_console_disconnect(port);
+	spin_unlock_irq(&port->port_lock);
+
+	cancel_work_sync(&cons->work);
+	kfifo_free(&cons->buf);
+	kfree(cons);
+	port->console = NULL;
+}
+
+ssize_t gserial_set_console(unsigned char port_num, const char *page, size_t count)
+{
+	struct gs_port *port;
+	bool enable;
+	int ret;
+
+	ret = strtobool(page, &enable);
+	if (ret)
+		return ret;
+
+	mutex_lock(&ports[port_num].lock);
+	port = ports[port_num].port;
+
+	if (WARN_ON(port == NULL)) {
+		ret = -ENXIO;
+		goto out;
+	}
+
+	if (enable)
+		ret = gs_console_init(port);
+	else
+		gs_console_exit(port);
+out:
+	mutex_unlock(&ports[port_num].lock);
+
+	return ret < 0 ? ret : count;
+}
+EXPORT_SYMBOL_GPL(gserial_set_console);
+
+ssize_t gserial_get_console(unsigned char port_num, char *page)
+{
+	struct gs_port *port;
+	ssize_t ret;
+
+	mutex_lock(&ports[port_num].lock);
+	port = ports[port_num].port;
+
+	if (WARN_ON(port == NULL))
+		ret = -ENXIO;
+	else
+		ret = sprintf(page, "%u\n", !!port->console);
+
+	mutex_unlock(&ports[port_num].lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gserial_get_console);
 
 #else
 
-static int gs_console_connect(int port_num)
+static int gs_console_connect(struct gs_port *port)
 {
 	return 0;
 }
 
-static void gs_console_disconnect(struct usb_ep *ep)
+static void gs_console_disconnect(struct gs_port *port)
 {
 }
 
-static void gserial_console_init(void)
+static int gs_console_init(struct gs_port *port)
 {
+	return -ENOSYS;
 }
 
-static void gserial_console_exit(void)
+static void gs_console_exit(struct gs_port *port)
 {
 }
 
···
 	int cond;
 
 	spin_lock_irq(&port->port_lock);
-	cond = (port->port.count == 0) && !port->openclose;
+	cond = port->port.count == 0;
 	spin_unlock_irq(&port->port_lock);
+
 	return cond;
 }
 
···
 		return;
 	}
 	port = ports[port_num].port;
+	gs_console_exit(port);
 	ports[port_num].port = NULL;
 	mutex_unlock(&ports[port_num].lock);
 
 	gserial_free_port(port);
 	tty_unregister_device(gs_tty_driver, port_num);
-	gserial_console_exit();
 }
 EXPORT_SYMBOL_GPL(gserial_free_line);
 
-int gserial_alloc_line(unsigned char *line_num)
+int gserial_alloc_line_no_console(unsigned char *line_num)
 {
 	struct usb_cdc_line_coding	coding;
+	struct gs_port			*port;
 	struct device			*tty_dev;
 	int				ret;
 	int				port_num;
···
 
 	/* ... and sysfs class devices, so mdev/udev make /dev/ttyGS* */
 
-	tty_dev = tty_port_register_device(&ports[port_num].port->port,
+	port = ports[port_num].port;
+	tty_dev = tty_port_register_device(&port->port,
 			gs_tty_driver, port_num, NULL);
 	if (IS_ERR(tty_dev)) {
-		struct gs_port	*port;
 		pr_err("%s: failed to register tty for port %d, err %ld\n",
 				__func__, port_num, PTR_ERR(tty_dev));
 
 		ret = PTR_ERR(tty_dev);
-		port = ports[port_num].port;
+		mutex_lock(&ports[port_num].lock);
 		ports[port_num].port = NULL;
+		mutex_unlock(&ports[port_num].lock);
 		gserial_free_port(port);
 		goto err;
 	}
 	*line_num = port_num;
-	gserial_console_init();
 err:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gserial_alloc_line_no_console);
+
+int gserial_alloc_line(unsigned char *line_num)
+{
+	int ret = gserial_alloc_line_no_console(line_num);
+
+	if (!ret && !*line_num)
+		gs_console_init(ports[*line_num].port);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(gserial_alloc_line);
···
 			gser->disconnect(gser);
 	}
 
-	status = gs_console_connect(port_num);
+	status = gs_console_connect(port);
 	spin_unlock_irqrestore(&port->port_lock, flags);
 
 	return status;
···
 	/* tell the TTY glue not to do I/O here any more */
 	spin_lock_irqsave(&port->port_lock, flags);
 
+	gs_console_disconnect(port);
+
 	/* REVISIT as above: how best to track this? */
 	port->port_line_coding = gser->port_line_coding;
 
 	port->port_usb = NULL;
 	gser->ioport = NULL;
-	if (port->port.count > 0 || port->openclose) {
+	if (port->port.count > 0) {
 		wake_up_interruptible(&port->drain_wait);
 		if (port->port.tty)
 			tty_hangup(port->port.tty);
···
 
 	/* finally, free any unused/unusable I/O buffers */
 	spin_lock_irqsave(&port->port_lock, flags);
-	if (port->port.count == 0 && !port->openclose)
+	if (port->port.count == 0)
 		kfifo_free(&port->port_write_buf);
 	gs_free_requests(gser->out, &port->read_pool, NULL);
 	gs_free_requests(gser->out, &port->read_queue, NULL);
···
 	port->read_allocated = port->read_started =
 		port->write_allocated = port->write_started = 0;
 
-	gs_console_disconnect(gser->in);
 	spin_unlock_irqrestore(&port->port_lock, flags);
 }
 EXPORT_SYMBOL_GPL(gserial_disconnect);

drivers/usb/gadget/function/u_serial.h

···
 void gs_free_req(struct usb_ep *, struct usb_request *req);
 
 /* management of individual TTY ports */
+int gserial_alloc_line_no_console(unsigned char *port_line);
 int gserial_alloc_line(unsigned char *port_line);
 void gserial_free_line(unsigned char port_line);
+
+#ifdef CONFIG_U_SERIAL_CONSOLE
+
+ssize_t gserial_set_console(unsigned char port_num, const char *page, size_t count);
+ssize_t gserial_get_console(unsigned char port_num, char *page);
+
+#endif /* CONFIG_U_SERIAL_CONSOLE */
 
 /* connect/disconnect is handled by individual functions */
 int gserial_connect(struct gserial *, u8 port_num);

drivers/usb/gadget/legacy/serial.c

···
 module_param(n_ports, uint, 0);
 MODULE_PARM_DESC(n_ports, "number of ports to create, default=1");
 
+static bool enable = true;
+
+static int switch_gserial_enable(bool do_enable);
+
+static int enable_set(const char *s, const struct kernel_param *kp)
+{
+	bool do_enable;
+	int ret;
+
+	if (!s)	/* called for no-arg enable == default */
+		return 0;
+
+	ret = strtobool(s, &do_enable);
+	if (ret || enable == do_enable)
+		return ret;
+
+	ret = switch_gserial_enable(do_enable);
+	if (!ret)
+		enable = do_enable;
+
+	return ret;
+}
+
+static const struct kernel_param_ops enable_ops = {
+	.set = enable_set,
+	.get = param_get_bool,
+};
+
+module_param_cb(enable, &enable_ops, &enable, 0644);
+
 /*-------------------------------------------------------------------------*/
 
 static struct usb_configuration serial_config_driver = {
···
 	.unbind		= gs_unbind,
 };
 
+static int switch_gserial_enable(bool do_enable)
+{
+	if (!serial_config_driver.label)
+		/* init() was not called, yet */
+		return 0;
+
+	if (do_enable)
+		return usb_composite_probe(&gserial_driver);
+
+	usb_composite_unregister(&gserial_driver);
+	return 0;
+}
+
 static int __init init(void)
 {
 	/* We *could* export two configs; that'd be much cleaner...
···
 	}
 	strings_dev[STRING_DESCRIPTION_IDX].s = serial_config_driver.label;
 
+	if (!enable)
+		return 0;
+
 	return usb_composite_probe(&gserial_driver);
 }
 module_init(init);
 
 static void __exit cleanup(void)
 {
-	usb_composite_unregister(&gserial_driver);
+	if (enable)
+		usb_composite_unregister(&gserial_driver);
 }
 module_exit(cleanup);

drivers/usb/gadget/udc/Kconfig

···
 	  dynamically linked module called "udc-xilinx" and force all
 	  gadget drivers to also be dynamically linked.
 
+config USB_TEGRA_XUDC
+	tristate "NVIDIA Tegra Superspeed USB 3.0 Device Controller"
+	depends on ARCH_TEGRA || COMPILE_TEST
+	depends on PHY_TEGRA_XUSB
+	help
+	 Enables NVIDIA Tegra USB 3.0 device mode controller driver.
+
+	 Say "y" to link the driver statically, or "m" to build a
+	 dynamically linked module called "tegra_xudc" and force all
+	 gadget drivers to also be dynamically linked.
+
 source "drivers/usb/gadget/udc/aspeed-vhub/Kconfig"
 
 #

drivers/usb/gadget/udc/Makefile

···
 obj-$(CONFIG_USB_FSL_USB2)	+= fsl_usb2_udc.o
 fsl_usb2_udc-y			:= fsl_udc_core.o
 fsl_usb2_udc-$(CONFIG_ARCH_MXC)	+= fsl_mxc_udc.o
+obj-$(CONFIG_USB_TEGRA_XUDC)	+= tegra-xudc.o
 obj-$(CONFIG_USB_M66592)	+= m66592-udc.o
 obj-$(CONFIG_USB_R8A66597)	+= r8a66597-udc.o
 obj-$(CONFIG_USB_RENESAS_USB3)	+= renesas_usb3.o

drivers/usb/gadget/udc/fsl_udc_core.c

···
 	u32 bitmask;
 	struct ep_queue_head *qh;
 
-	ep = container_of(_ep, struct fsl_ep, ep);
-	if (!_ep || (!ep->ep.desc && ep_index(ep) != 0))
+	if (!_ep || _ep->desc || !(_ep->desc->bEndpointAddress&0xF))
 		return -ENODEV;
+
+	ep = container_of(_ep, struct fsl_ep, ep);
 
 	udc = (struct fsl_udc *)ep->udc;
 
···
 		struct fsl_req *curr_req)
 {
 	struct ep_td_struct *curr_td;
-	int	td_complete, actual, remaining_length, j, tmp;
+	int	actual, remaining_length, j, tmp;
 	int	status = 0;
 	int	errors = 0;
 	struct  ep_queue_head *curr_qh = &udc->ep_qh[pipe];
 	int direction = pipe % 2;
 
 	curr_td = curr_req->head;
-	td_complete = 0;
 	actual = curr_req->req.length;
 
 	for (j = 0; j < curr_req->dtd_count; j++) {
···
 				status = -EPROTO;
 				break;
 			} else {
-				td_complete++;
 				break;
 			}
 		} else {
-			td_complete++;
 			VDBG("dTD transmitted successful");
 		}
 

drivers/usb/gadget/udc/lpc32xx_udc.c

···
 	struct device *dev = &pdev->dev;
 	struct lpc32xx_udc *udc;
 	int retval, i;
-	struct resource *res;
 	dma_addr_t dma_handle;
 	struct device_node *isp1301_node;
 
···
 	 *  IORESOURCE_IRQ, USB device interrupt number
 	 *  IORESOURCE_IRQ, USB transceiver interrupt number
 	 */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENXIO;
 
 	spin_lock_init(&udc->lock);
 
···
 			return udc->udp_irq[i];
 	}
 
-	udc->udp_baseaddr = devm_ioremap_resource(dev, res);
+	udc->udp_baseaddr = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(udc->udp_baseaddr)) {
 		dev_err(udc->dev, "IO map failure\n");
 		return PTR_ERR(udc->udp_baseaddr);

drivers/usb/gadget/udc/renesas_usb3.c

···
 	usb3_transition_to_default_state(usb3, false);
 }
 
+static void usb3_irq_epc_int_1_suspend(struct renesas_usb3 *usb3)
+{
+	usb3_disable_irq_1(usb3, USB_INT_1_B2_SPND);
+
+	if (usb3->gadget.speed != USB_SPEED_UNKNOWN &&
+	    usb3->gadget.state != USB_STATE_NOTATTACHED) {
+		if (usb3->driver && usb3->driver->suspend)
+			usb3->driver->suspend(&usb3->gadget);
+		usb_gadget_set_state(&usb3->gadget, USB_STATE_SUSPENDED);
+	}
+}
+
 static void usb3_irq_epc_int_1_disable(struct renesas_usb3 *usb3)
 {
 	usb3_stop_usb3_connection(usb3);
···
 
 	if (int_sta_1 & USB_INT_1_B2_RSUM)
 		usb3_irq_epc_int_1_resume(usb3);
+
+	if (int_sta_1 & USB_INT_1_B2_SPND)
+		usb3_irq_epc_int_1_suspend(usb3);
 
 	if (int_sta_1 & USB_INT_1_SPEED)
 		usb3_irq_epc_int_1_speed(usb3);

drivers/usb/gadget/udc/tegra-xudc.c

···
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * NVIDIA Tegra XUSB device mode controller
+ *
+ * Copyright (c) 2013-2019, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2015, Google Inc.
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/phy/tegra/xusb.h>
+#include <linux/pm_domain.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/usb/ch9.h>
+#include <linux/usb/gadget.h>
+#include <linux/usb/role.h>
+#include <linux/workqueue.h>
+
+/* XUSB_DEV registers */
+#define SPARAM 0x000
+#define  SPARAM_ERSTMAX_MASK GENMASK(20, 16)
+#define  SPARAM_ERSTMAX(x) (((x) << 16) & SPARAM_ERSTMAX_MASK)
+#define DB 0x004
+#define  DB_TARGET_MASK GENMASK(15, 8)
+#define  DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
+#define  DB_STREAMID_MASK GENMASK(31, 16)
+#define  DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
+#define ERSTSZ 0x008
+#define  ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
+#define  ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
+#define ERSTXBALO(x) (0x010 + 8 * (x))
+#define ERSTXBAHI(x) (0x014 + 8 * (x))
+#define ERDPLO 0x020
+#define  ERDPLO_EHB BIT(3)
+#define ERDPHI 0x024
+#define EREPLO 0x028
+#define  EREPLO_ECS BIT(0)
+#define  EREPLO_SEGI BIT(1)
+#define EREPHI 0x02c
+#define CTRL 0x030
+#define  CTRL_RUN BIT(0)
+#define  CTRL_LSE BIT(1)
+#define  CTRL_IE BIT(4)
+#define  CTRL_SMI_EVT BIT(5)
+#define  CTRL_SMI_DSE BIT(6)
+#define  CTRL_EWE BIT(7)
+#define  CTRL_DEVADDR_MASK GENMASK(30, 24)
+#define  CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
+#define  CTRL_ENABLE BIT(31)
+#define ST 0x034
+#define  ST_RC BIT(0)
+#define  ST_IP BIT(4)
+#define RT_IMOD	0x038
+#define  RT_IMOD_IMODI_MASK GENMASK(15, 0)
+#define  RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
+#define  RT_IMOD_IMODC_MASK GENMASK(31, 16)
+#define  RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
+#define PORTSC 0x03c
+#define  PORTSC_CCS BIT(0)
+#define  PORTSC_PED BIT(1)
+#define  PORTSC_PR BIT(4)
+#define  PORTSC_PLS_SHIFT 5
+#define  PORTSC_PLS_MASK GENMASK(8, 5)
+#define  PORTSC_PLS_U0 0x0
+#define  PORTSC_PLS_U2 0x2
+#define  PORTSC_PLS_U3 0x3
+#define  PORTSC_PLS_DISABLED 0x4
+#define  PORTSC_PLS_RXDETECT 0x5
+#define  PORTSC_PLS_INACTIVE 0x6
+#define  PORTSC_PLS_RESUME 0xf
+#define  PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
+#define  PORTSC_PS_SHIFT 10
+#define  PORTSC_PS_MASK GENMASK(13, 10)
+#define  PORTSC_PS_UNDEFINED 0x0
+#define  PORTSC_PS_FS 0x1
+#define  PORTSC_PS_LS 0x2
+#define  PORTSC_PS_HS 0x3
+#define  PORTSC_PS_SS 0x4
+#define  PORTSC_LWS BIT(16)
+#define  PORTSC_CSC BIT(17)
+#define  PORTSC_WRC BIT(19)
+#define  PORTSC_PRC BIT(21)
+#define  PORTSC_PLC BIT(22)
+#define  PORTSC_CEC BIT(23)
+#define  PORTSC_WPR BIT(30)
+#define  PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
+			     PORTSC_PLC | PORTSC_CEC)
+#define ECPLO 0x040
+#define ECPHI 0x044
+#define MFINDEX 0x048
+#define  MFINDEX_FRAME_SHIFT 3
+#define  MFINDEX_FRAME_MASK GENMASK(13, 3)
+#define PORTPM 0x04c
+#define  PORTPM_L1S_MASK GENMASK(1, 0)
+#define  PORTPM_L1S_DROP 0x0
+#define  PORTPM_L1S_ACCEPT 0x1
+#define  PORTPM_L1S_NYET 0x2
+#define  PORTPM_L1S_STALL 0x3
+#define  PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
+#define  PORTPM_RWE BIT(3)
+#define  PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
+#define  PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
+#define  PORTPM_FLA BIT(24)
+#define  PORTPM_VBA BIT(25)
+#define  PORTPM_WOC BIT(26)
+#define  PORTPM_WOD BIT(27)
+#define  PORTPM_U1E BIT(28)
+#define  PORTPM_U2E BIT(29)
+#define  PORTPM_FRWE BIT(30)
+#define  PORTPM_PNG_CYA BIT(31)
+#define EP_HALT 0x050
+#define EP_PAUSE 0x054
+#define EP_RELOAD 0x058
+#define EP_STCHG 0x05c
+#define DEVNOTIF_LO 0x064
+#define  DEVNOTIF_LO_TRIG BIT(0)
+#define  DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
+#define  DEVNOTIF_LO_TYPE(x) (((x) << 4)  & DEVNOTIF_LO_TYPE_MASK)
+#define  DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
+#define DEVNOTIF_HI 0x068
+#define PORTHALT 0x06c
+#define  PORTHALT_HALT_LTSSM BIT(0)
+#define  PORTHALT_HALT_REJECT BIT(1)
+#define  PORTHALT_STCHG_REQ BIT(20)
+#define  PORTHALT_STCHG_INTR_EN BIT(24)
+#define PORT_TM	0x070
+#define EP_THREAD_ACTIVE 0x074
+#define EP_STOPPED 0x078
+#define HSFSPI_COUNT0 0x100
+#define HSFSPI_COUNT13 0x134
+#define  HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
+#define  HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
+				HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
+#define BLCG 0x840
+#define SSPX_CORE_CNT0 0x610
+#define  SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
+#define  SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
+#define SSPX_CORE_CNT30 0x688
+#define  SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
+					SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
+#define SSPX_CORE_CNT32 0x690
+#define  SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
+#define  SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
+					SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
+#define SSPX_CORE_PADCTL4 0x750
+#define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
+#define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
+				SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
+#define  BLCG_DFPCI BIT(0)
+#define  BLCG_UFPCI BIT(1)
+#define  BLCG_FE BIT(2)
+#define  BLCG_COREPLL_PWRDN BIT(8)
+#define  BLCG_IOPLL_0_PWRDN BIT(9)
+#define  BLCG_IOPLL_1_PWRDN BIT(10)
+#define  BLCG_IOPLL_2_PWRDN BIT(11)
+#define  BLCG_ALL 0x1ff
+#define CFG_DEV_SSPI_XFER 0x858
+#define  CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
+#define  CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
+					CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
+#define CFG_DEV_FE 0x85c
+#define  CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
+#define  CFG_DEV_FE_PORTREGSEL_SS_PI 1
+#define  CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
+#define  CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
+#define  CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
+
+/* FPCI registers */
+#define XUSB_DEV_CFG_1 0x004
+#define  XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
+#define  XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
+#define  XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
+#define XUSB_DEV_CFG_4 0x010
+#define  XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
+#define XUSB_DEV_CFG_5 0x014
+
+/* IPFS registers */
+#define XUSB_DEV_CONFIGURATION_0 0x180
+#define  XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
+#define XUSB_DEV_INTR_MASK_0 0x188
+#define  XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
+
+struct tegra_xudc_ep_context {
+	__le32 info0;
+	__le32 info1;
+	__le32 deq_lo;
+	__le32 deq_hi;
+	__le32 tx_info;
+	__le32 rsvd[11];
+};
+
+#define EP_STATE_DISABLED 0
+#define EP_STATE_RUNNING 1
+#define EP_STATE_HALTED 2
+#define EP_STATE_STOPPED 3
+#define EP_STATE_ERROR 4
+
+#define EP_TYPE_INVALID 0
+#define EP_TYPE_ISOCH_OUT 1
+#define EP_TYPE_BULK_OUT 2
+#define EP_TYPE_INTERRUPT_OUT 3
+#define EP_TYPE_CONTROL 4
+#define EP_TYPE_ISCOH_IN 5
+#define EP_TYPE_BULK_IN 6
+#define EP_TYPE_INTERRUPT_IN 7
+
+#define BUILD_EP_CONTEXT_RW(name, member, shift, mask)			\
+static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx)	\
+{									\
+	return (le32_to_cpu(ctx->member) >> (shift)) & (mask);		\
+}									\
+static inline void							\
+ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val)		\
+{									\
+	u32 tmp;							\
+									\
+	tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift));		\
+	tmp |= (val & (mask)) << (shift);				\
+	ctx->member = cpu_to_le32(tmp);					\
+}
+
+BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
+BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
+BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
+BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
+BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
+BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
+BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
+BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
+BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
+BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
+BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
+BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
+BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
+BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
+BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
+BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
+BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 24, 0xff)
+BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
+BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
+BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
+BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
+BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
+
+static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
+{
+	return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
+		(ep_ctx_read_deq_lo(ctx) << 4);
+}
+
+static inline void
+ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
+{
+	ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
+	ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
+}
+
+struct tegra_xudc_trb {
+	__le32 data_lo;
+	__le32 data_hi;
+	__le32 status;
+	__le32 control;
+};
+
+#define TRB_TYPE_RSVD 0
+#define TRB_TYPE_NORMAL 1
+#define TRB_TYPE_SETUP_STAGE 2
+#define TRB_TYPE_DATA_STAGE 3
+#define TRB_TYPE_STATUS_STAGE 4
+#define TRB_TYPE_ISOCH 5
+#define TRB_TYPE_LINK 6
+#define TRB_TYPE_TRANSFER_EVENT 32
+#define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
+#define TRB_TYPE_STREAM 48
+#define TRB_TYPE_SETUP_PACKET_EVENT 63
+
+#define TRB_CMPL_CODE_INVALID 0
+#define TRB_CMPL_CODE_SUCCESS 1
+#define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
+#define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
+#define TRB_CMPL_CODE_USB_TRANS_ERR 4
+#define TRB_CMPL_CODE_TRB_ERR 5
+#define TRB_CMPL_CODE_STALL 6
+#define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
+#define TRB_CMPL_CODE_SHORT_PACKET 13
+#define TRB_CMPL_CODE_RING_UNDERRUN 14
+#define TRB_CMPL_CODE_RING_OVERRUN 15
+#define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
+#define TRB_CMPL_CODE_STOPPED 26
+#define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
+#define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
+#define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
+#define TRB_CMPL_CODE_HOST_REJECTED 221
+#define TRB_CMPL_CODE_CTRL_DIR_ERR 222
+#define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
+
+#define BUILD_TRB_RW(name, member, shift, mask)				\
+static inline u32 trb_read_##name(struct tegra_xudc_trb *trb)		\
+{									\
+	return (le32_to_cpu(trb->member) >> (shift)) & (mask);		\
+}									\
+static inline void							\
+trb_write_##name(struct tegra_xudc_trb *trb, u32 val)			\
+{									\
+	u32 tmp;							\
+									\
+	tmp = le32_to_cpu(trb->member) & ~((mask) << (shift));		\
+	tmp |= (val & (mask)) << (shift);				\
+	trb->member = cpu_to_le32(tmp);					\
+}
+
+BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
+BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
+BUILD_TRB_RW(seq_num, status, 0, 0xffff)
+BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
+BUILD_TRB_RW(td_size, status, 17, 0x1f)
+BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
+BUILD_TRB_RW(cycle, control, 0, 0x1)
+BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
+BUILD_TRB_RW(isp, control, 2, 0x1)
+BUILD_TRB_RW(chain, control, 4, 0x1)
+BUILD_TRB_RW(ioc, control, 5, 0x1)
+BUILD_TRB_RW(type, control, 10, 0x3f)
+BUILD_TRB_RW(stream_id, control, 16, 0xffff)
+BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
+BUILD_TRB_RW(tlbpc, control, 16, 0xf)
+BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
+BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
+BUILD_TRB_RW(sia, control, 31, 0x1)
+
+static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
+{
+	return ((u64)trb_read_data_hi(trb) << 32) |
+		trb_read_data_lo(trb);
+}
+
+static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
+{
+	trb_write_data_lo(trb, lower_32_bits(addr));
+	trb_write_data_hi(trb, upper_32_bits(addr));
+}
+
+struct tegra_xudc_request {
+	struct usb_request usb_req;
+
+	size_t buf_queued;
+	unsigned int trbs_queued;
+	unsigned int trbs_needed;
+	bool need_zlp;
+
+	struct tegra_xudc_trb *first_trb;
+	struct tegra_xudc_trb *last_trb;
+
+	struct list_head list;
+};
+
+struct tegra_xudc_ep {
+	struct tegra_xudc *xudc;
+	struct usb_ep usb_ep;
+	unsigned int index;
+	char name[8];
+
+	struct tegra_xudc_ep_context *context;
+
+#define XUDC_TRANSFER_RING_SIZE 64
+	struct tegra_xudc_trb *transfer_ring;
+	dma_addr_t transfer_ring_phys;
+
+	unsigned int enq_ptr;
+	unsigned int deq_ptr;
+	bool pcs;
+	bool ring_full;
+	bool stream_rejected;
+
+	struct list_head queue;
+	const struct usb_endpoint_descriptor *desc;
+	const struct usb_ss_ep_comp_descriptor *comp_desc;
+};
+
+struct tegra_xudc_sel_timing {
+	__u8 u1sel;
+	__u8 u1pel;
+	__le16 u2sel;
+	__le16 u2pel;
+};
+
+enum tegra_xudc_setup_state {
+	WAIT_FOR_SETUP,
+	DATA_STAGE_XFER,
+	DATA_STAGE_RECV,
+	STATUS_STAGE_XFER,
+	STATUS_STAGE_RECV,
+};
+
+struct tegra_xudc_setup_packet {
+	struct usb_ctrlrequest ctrl_req;
+	unsigned int seq_num;
+};
+
+struct tegra_xudc_save_regs {
+	u32 ctrl;
+	u32 portpm;
+};
+
+struct tegra_xudc {
+	struct device *dev;
+	const struct tegra_xudc_soc *soc;
+	struct tegra_xusb_padctl *padctl;
+
+	spinlock_t lock;
+
+	struct usb_gadget gadget;
+	struct usb_gadget_driver *driver;
+
+#define XUDC_NR_EVENT_RINGS 2
+#define XUDC_EVENT_RING_SIZE 4096
+	struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
+	dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
+	unsigned int event_ring_index;
+	unsigned int event_ring_deq_ptr;
+	bool ccs;
+
+#define XUDC_NR_EPS 32
+	struct tegra_xudc_ep ep[XUDC_NR_EPS];
+	struct tegra_xudc_ep_context *ep_context;
+	dma_addr_t ep_context_phys;
+
+	struct device *genpd_dev_device;
+	struct device *genpd_dev_ss;
+	struct device_link *genpd_dl_device;
+	struct device_link *genpd_dl_ss;
+
+	struct dma_pool *transfer_ring_pool;
+
+	bool queued_setup_packet;
+	struct tegra_xudc_setup_packet setup_packet;
+	enum tegra_xudc_setup_state setup_state;
+	u16 setup_seq_num;
+
+	u16 dev_addr;
+	u16 isoch_delay;
+	struct tegra_xudc_sel_timing sel_timing;
+	u8 test_mode_pattern;
+	u16 status_buf;
+	struct tegra_xudc_request *ep0_req;
+
+	bool pullup;
+
+	unsigned int nr_enabled_eps;
+	unsigned int nr_isoch_eps;
+
+	unsigned int device_state;
+	unsigned int resume_state;
+
+	int irq;
+
+	void __iomem *base;
+	resource_size_t phys_base;
+	void __iomem *ipfs;
+	void __iomem *fpci;
+
+	struct regulator_bulk_data *supplies;
+
+	struct clk_bulk_data *clks;
+
+	enum usb_role device_mode;
+	struct usb_role_switch *usb_role_sw;
+	struct work_struct usb_role_sw_work;
+
+	struct phy *usb3_phy;
+	struct phy *utmi_phy;
+
+	struct tegra_xudc_save_regs saved_regs;
+	bool suspended;
+	bool powergated;
+
+	struct completion disconnect_complete;
+
+	bool selfpowered;
+
+#define TOGGLE_VBUS_WAIT_MS 100
+	struct delayed_work plc_reset_work;
+	bool wait_csc;
+
+	struct delayed_work port_reset_war_work;
+	bool wait_for_sec_prc;
+};
+
+#define XUDC_TRB_MAX_BUFFER_SIZE 65536
+#define XUDC_MAX_ISOCH_EPS 4
+#define XUDC_INTERRUPT_MODERATION_US 0
+
+static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
+	.bLength = USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType = USB_DT_ENDPOINT,
+	.bEndpointAddress = 0,
+	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
+	.wMaxPacketSize = cpu_to_le16(64),
+};
+
+struct tegra_xudc_soc {
+	const char * const *supply_names;
+	unsigned int num_supplies;
+	const char * const *clock_names;
+	unsigned int num_clks;
+	bool u1_enable;
+	bool u2_enable;
+	bool lpm_enable;
+	bool invalid_seq_num;
+	bool pls_quirk;
+	bool port_reset_quirk;
+	bool has_ipfs;
+};
+
+static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
+{
+	return readl(xudc->fpci + offset);
+}
+
+static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
+			       unsigned int offset)
+{
+	writel(val, xudc->fpci + offset);
+}
+
+static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
+{
+	return readl(xudc->ipfs + offset);
+}
+
+static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
+			       unsigned int offset)
+{
+	writel(val, xudc->ipfs + offset);
+}
+
+static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
+{
+	return readl(xudc->base + offset);
+}
+
+static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
+			       unsigned int offset)
+{
+	writel(val, xudc->base + offset);
+}
+
+static inline int xudc_readl_poll(struct tegra_xudc *xudc,
+				  unsigned int offset, u32 mask, u32 val)
+{
+	u32 regval;
+
+	return readl_poll_timeout_atomic(xudc->base + offset, regval,
+					 (regval & mask) == val, 1, 100);
+}
+
+static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
+{
+	return container_of(gadget, struct tegra_xudc, gadget);
+}
+
+static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
+{
+	return container_of(ep, struct tegra_xudc_ep, usb_ep);
+}
+
+static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
+{
+	return container_of(req, struct tegra_xudc_request, usb_req);
+}
+
+static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
+			    struct tegra_xudc_trb *trb)
+{
+	dev_dbg(xudc->dev,
+		"%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
+		type, trb, trb->data_lo, trb->data_hi, trb->status,
+		trb->control);
+}
+
+static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
+{
+	int err;
+
+	pm_runtime_get_sync(xudc->dev);
+
+	err = phy_power_on(xudc->utmi_phy);
+	if (err < 0)
+		dev_err(xudc->dev, "utmi power on failed %d\n", err);
+
+	err = phy_power_on(xudc->usb3_phy);
+	if (err < 0)
+		dev_err(xudc->dev, "usb3 phy power on failed %d\n", err);
+
+	dev_dbg(xudc->dev, "device mode on\n");
+
+	tegra_xusb_padctl_set_vbus_override(xudc->padctl, true);
+
+	xudc->device_mode = USB_ROLE_DEVICE;
+}
+
+static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
+{
+	bool connected = false;
+	u32 pls, val;
+	int err;
+
+	dev_dbg(xudc->dev, "device mode off\n");
+
+	connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
+
+	reinit_completion(&xudc->disconnect_complete);
+
+	tegra_xusb_padctl_set_vbus_override(xudc->padctl, false);
+
+	pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
+		PORTSC_PLS_SHIFT;
+
+	/* Direct link to U0 if disconnected in RESUME or U2. */
+	if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
+	    (pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
+		val = xudc_readl(xudc, PORTPM);
+		val |= PORTPM_FRWE;
+		xudc_writel(xudc, val, PORTPM);
+
+		val = xudc_readl(xudc, PORTSC);
+		val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
+		val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
+		xudc_writel(xudc, val, PORTSC);
+	}
+
+	xudc->device_mode = USB_ROLE_NONE;
+
+	/* Wait for disconnect event. */
+	if (connected)
+		wait_for_completion(&xudc->disconnect_complete);
+
+	/* Make sure interrupt handler has completed before powergating. */
+	synchronize_irq(xudc->irq);
+
+	err = phy_power_off(xudc->utmi_phy);
+	if (err < 0)
+		dev_err(xudc->dev, "utmi_phy power off failed %d\n", err);
+
+	err = phy_power_off(xudc->usb3_phy);
+	if (err < 0)
+		dev_err(xudc->dev, "usb3_phy power off failed %d\n", err);
+
+	pm_runtime_put(xudc->dev);
+}
+
+static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
+{
+	struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
+					       usb_role_sw_work);
+
+	if (!xudc->usb_role_sw ||
+		usb_role_switch_get_role(xudc->usb_role_sw) == USB_ROLE_DEVICE)
+		tegra_xudc_device_mode_on(xudc);
+	else
+		tegra_xudc_device_mode_off(xudc);
+
+}
+
+static int tegra_xudc_usb_role_sw_set(struct device *dev, enum usb_role role)
+{
+	struct tegra_xudc *xudc = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	dev_dbg(dev, "%s role is %d\n", __func__, role);
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if (!xudc->suspended)
+		schedule_work(&xudc->usb_role_sw_work);
+
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return 0;
+}
+
+static void tegra_xudc_plc_reset_work(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
+					       plc_reset_work);
+	unsigned long flags;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if (xudc->wait_csc) {
+		u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
+			PORTSC_PLS_SHIFT;
+
+		if (pls == PORTSC_PLS_INACTIVE) {
+			dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
+			tegra_xusb_padctl_set_vbus_override(xudc->padctl,
+							      false);
+			tegra_xusb_padctl_set_vbus_override(xudc->padctl, true);
+			xudc->wait_csc = false;
+		}
+	}
+
+	spin_unlock_irqrestore(&xudc->lock, flags);
+}
+
+static void tegra_xudc_port_reset_war_work(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tegra_xudc *xudc =
+		container_of(dwork, struct tegra_xudc, port_reset_war_work);
+	unsigned long flags;
+	u32 pls;
+	int ret;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if ((xudc->device_mode == USB_ROLE_DEVICE)
+			      && xudc->wait_for_sec_prc) {
+		pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
+			PORTSC_PLS_SHIFT;
+		dev_dbg(xudc->dev, "pls = %x\n", pls);
+
+		if (pls == PORTSC_PLS_DISABLED) {
+			dev_dbg(xudc->dev, "toggle vbus\n");
+			/* PRC doesn't complete in 100ms, toggle the vbus */
+			ret = tegra_phy_xusb_utmi_port_reset(xudc->utmi_phy);
+			if (ret == 1)
+				xudc->wait_for_sec_prc = 0;
+		}
+	}
+
+	spin_unlock_irqrestore(&xudc->lock, flags);
+}
+
+static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
+				   struct tegra_xudc_trb *trb)
+{
+	unsigned int index;
+
+	index = trb - ep->transfer_ring;
+
+	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
+		return 0;
+
+	return (ep->transfer_ring_phys + index * sizeof(*trb));
+}
+
+static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
+					       dma_addr_t addr)
+{
+	struct tegra_xudc_trb *trb;
+	unsigned int index;
+
+	index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
+
+	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
+		return NULL;
+
+	trb = &ep->transfer_ring[index];
+
+	return trb;
+}
+
+static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
+{
+	xudc_writel(xudc, BIT(ep), EP_RELOAD);
+	xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), 0);
+}
+
+static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, EP_PAUSE);
+	if (val & BIT(ep))
+		return;
+	val |= BIT(ep);
+
+	xudc_writel(xudc, val, EP_PAUSE);
+
+	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
+
+	xudc_writel(xudc, BIT(ep), EP_STCHG);
+}
+
+static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, EP_PAUSE);
+	if (!(val & BIT(ep)))
+		return;
+	val &= ~BIT(ep);
+
+	xudc_writel(xudc, val, EP_PAUSE);
+
+	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
+
+	xudc_writel(xudc, BIT(ep), EP_STCHG);
+}
+
+static void ep_unpause_all(struct tegra_xudc *xudc)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, EP_PAUSE);
+
+	xudc_writel(xudc, 0, EP_PAUSE);
+
+	xudc_readl_poll(xudc, EP_STCHG, val, val);
+
+	xudc_writel(xudc, val, EP_STCHG);
+}
+
+static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, EP_HALT);
+	if (val & BIT(ep))
+		return;
+	val |= BIT(ep);
+	xudc_writel(xudc, val, EP_HALT);
+
+	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
+
+	xudc_writel(xudc, BIT(ep), EP_STCHG);
+}
+
+static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, EP_HALT);
+	if (!(val & BIT(ep)))
+		return;
+	val &= ~BIT(ep);
+	xudc_writel(xudc, val, EP_HALT);
+
+	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
+
+	xudc_writel(xudc, BIT(ep), EP_STCHG);
+}
+
+static void ep_unhalt_all(struct tegra_xudc *xudc)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, EP_HALT);
+	if (!val)
+		return;
+	xudc_writel(xudc, 0, EP_HALT);
+
+	xudc_readl_poll(xudc, EP_STCHG, val, val);
+
+	xudc_writel(xudc, val, EP_STCHG);
+}
+
+static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
+{
+	xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
+	xudc_writel(xudc, BIT(ep), EP_STOPPED);
+}
+
+static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
+{
+	xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), 0);
+}
+
+static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
+				struct tegra_xudc_request *req, int status)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+
+	dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
+		 req, ep->index, status);
+
+	if (likely(req->usb_req.status == -EINPROGRESS))
+		req->usb_req.status = status;
+
+	list_del_init(&req->list);
+
+	if (usb_endpoint_xfer_control(ep->desc)) {
+		usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
+					 (xudc->setup_state ==
+					  DATA_STAGE_XFER));
+	} else {
+		usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
+					 usb_endpoint_dir_in(ep->desc));
+	}
+
+	spin_unlock(&xudc->lock);
+	usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
+	spin_lock(&xudc->lock);
+}
+
+static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
+{
+	struct tegra_xudc_request *req;
+
+	while (!list_empty(&ep->queue)) {
+		req = list_first_entry(&ep->queue, struct tegra_xudc_request,
+				       list);
+		tegra_xudc_req_done(ep, req, status);
+	}
+}
+
+static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
+{
+	if (ep->ring_full)
+		return 0;
+
+	if (ep->deq_ptr > ep->enq_ptr)
+		return ep->deq_ptr - ep->enq_ptr - 1;
+
+	return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
+}
+
+static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
+				     struct tegra_xudc_request *req,
+				     struct tegra_xudc_trb *trb,
+				     bool ioc)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+	dma_addr_t buf_addr;
+	size_t len;
+
+	len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
+		    req->buf_queued);
+	if (len > 0)
+		buf_addr = req->usb_req.dma + req->buf_queued;
+	else
+		buf_addr = 0;
+
+	trb_write_data_ptr(trb, buf_addr);
+
+	trb_write_transfer_len(trb, len);
+	trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);
+
+	if (req->trbs_queued == req->trbs_needed - 1 ||
+		(req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
+		trb_write_chain(trb, 0);
+	else
+		trb_write_chain(trb, 1);
+
+	trb_write_ioc(trb, ioc);
+
+	if (usb_endpoint_dir_out(ep->desc) ||
+	    (usb_endpoint_xfer_control(ep->desc) &&
+	     (xudc->setup_state == DATA_STAGE_RECV)))
+		trb_write_isp(trb, 1);
+	else
+		trb_write_isp(trb, 0);
+
+	if (usb_endpoint_xfer_control(ep->desc)) {
+		if (xudc->setup_state == DATA_STAGE_XFER ||
+		    xudc->setup_state == DATA_STAGE_RECV)
+			trb_write_type(trb, TRB_TYPE_DATA_STAGE);
+		else
+			trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
+
+		if (xudc->setup_state == DATA_STAGE_XFER ||
+		    xudc->setup_state == STATUS_STAGE_XFER)
+			trb_write_data_stage_dir(trb, 1);
+		else
+			trb_write_data_stage_dir(trb, 0);
+	} else if (usb_endpoint_xfer_isoc(ep->desc)) {
+		trb_write_type(trb, TRB_TYPE_ISOCH);
+		trb_write_sia(trb, 1);
+		trb_write_frame_id(trb, 0);
+		trb_write_tlbpc(trb, 0);
+	} else if (usb_ss_max_streams(ep->comp_desc)) {
+		trb_write_type(trb, TRB_TYPE_STREAM);
+		trb_write_stream_id(trb, req->usb_req.stream_id);
+	} else {
+		trb_write_type(trb, TRB_TYPE_NORMAL);
+		trb_write_stream_id(trb, 0);
+	}
+
+	trb_write_cycle(trb, ep->pcs);
+
+	req->trbs_queued++;
+	req->buf_queued += len;
+
+	dump_trb(xudc, "TRANSFER", trb);
+}
+
+static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
+					  struct tegra_xudc_request *req)
+{
+	unsigned int i, count, available;
+	bool wait_td = false;
+
+	available = ep_available_trbs(ep);
+	count = req->trbs_needed - req->trbs_queued;
+	if (available < count) {
+		count = available;
+		ep->ring_full = true;
+	}
+
+	/*
+	 * To generate zero-length packet on USB bus, SW needs schedule a
+	 * standalone zero-length TD. According to HW's behavior, SW needs
+	 * to schedule TDs in different ways for different endpoint types.
+	 *
+	 * For control endpoint:
+	 * - Data stage TD (IOC = 1, CH = 0)
+	 * - Ring doorbell and wait transfer event
+	 * - Data stage TD for ZLP (IOC = 1, CH = 0)
+	 * - Ring doorbell
+	 *
+	 * For bulk and interrupt endpoints:
+	 * - Normal transfer TD (IOC = 0, CH = 0)
+	 * - Normal transfer TD for ZLP (IOC = 1, CH = 0)
+	 * - Ring doorbell
+	 */
+
+	if (req->need_zlp && usb_endpoint_xfer_control(ep->desc) && count > 1)
+		wait_td = true;
+
+	if (!req->first_trb)
+		req->first_trb = &ep->transfer_ring[ep->enq_ptr];
+
+	for (i = 0; i < count; i++) {
+		struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
+		bool ioc = false;
+
+		if ((i == count - 1) || (wait_td && i == count - 2))
+			ioc = true;
+
+		tegra_xudc_queue_one_trb(ep, req, trb, ioc);
+		req->last_trb = trb;
+
+		ep->enq_ptr++;
+		if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
+			trb = &ep->transfer_ring[ep->enq_ptr];
+			trb_write_cycle(trb, ep->pcs);
+			ep->pcs = !ep->pcs;
+			ep->enq_ptr = 0;
+		}
+
+		if (ioc)
+			break;
+	}
+
+	return count;
+}
+
+static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+	u32 val;
+
+	if (list_empty(&ep->queue))
+		return;
+
+	val = DB_TARGET(ep->index);
+	if (usb_endpoint_xfer_control(ep->desc)) {
+		val |= DB_STREAMID(xudc->setup_seq_num);
+	} else if (usb_ss_max_streams(ep->comp_desc) > 0) {
+		struct tegra_xudc_request *req;
+
+		/* Don't ring doorbell if the stream has been rejected. */
+		if (ep->stream_rejected)
+			return;
+
+		req = list_first_entry(&ep->queue, struct tegra_xudc_request,
+				       list);
+		val |= DB_STREAMID(req->usb_req.stream_id);
+	}
+
+	dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
+	xudc_writel(xudc, val, DB);
+}
+
+static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
+{
+	struct tegra_xudc_request *req;
+	bool trbs_queued = false;
+
+	list_for_each_entry(req, &ep->queue, list) {
+		if (ep->ring_full)
+			break;
+
+		if (tegra_xudc_queue_trbs(ep, req) > 0)
+			trbs_queued = true;
+	}
+
+	if (trbs_queued)
+		tegra_xudc_ep_ring_doorbell(ep);
+}
+
+static int
+__tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+	int err;
+
+	if (usb_endpoint_xfer_control(ep->desc) && !list_empty(&ep->queue)) {
+		dev_err(xudc->dev, "control EP has pending transfers\n");
+		return -EINVAL;
+	}
+
+	if (usb_endpoint_xfer_control(ep->desc)) {
+		err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
+					     (xudc->setup_state ==
+					      DATA_STAGE_XFER));
+	} else {
+		err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
+					     usb_endpoint_dir_in(ep->desc));
+	}
+
+	if (err < 0) {
+		dev_err(xudc->dev, "failed to map request: %d\n", err);
+		return err;
+	}
+
+	req->first_trb = NULL;
+	req->last_trb = NULL;
+	req->buf_queued = 0;
+	req->trbs_queued = 0;
+	req->need_zlp = false;
+	req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
+					XUDC_TRB_MAX_BUFFER_SIZE);
+	if (req->usb_req.length == 0)
+		req->trbs_needed++;
+
+	if (!usb_endpoint_xfer_isoc(ep->desc) &&
+	    req->usb_req.zero && req->usb_req.length &&
+	    ((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
+		req->trbs_needed++;
+		req->need_zlp = true;
+	}
+
+	req->usb_req.status = -EINPROGRESS;
+	req->usb_req.actual = 0;
+
+	list_add_tail(&req->list, &ep->queue);
+
+	tegra_xudc_ep_kick_queue(ep);
+
+	return 0;
+}
+
+static int
+tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
+		    gfp_t gfp)
+{
+	struct tegra_xudc_request *req;
+	struct tegra_xudc_ep *ep;
+	struct tegra_xudc *xudc;
+	unsigned long flags;
+	int ret;
+
+	if (!usb_ep || !usb_req)
+		return -EINVAL;
+
+	ep = to_xudc_ep(usb_ep);
+	req = to_xudc_req(usb_req);
+	xudc = ep->xudc;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	if (xudc->powergated || !ep->desc) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+
+	ret = __tegra_xudc_ep_queue(ep, req);
+unlock:
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
+				  struct tegra_xudc_request *req)
+{
+	struct tegra_xudc_trb *trb = req->first_trb;
+	bool pcs_enq = trb_read_cycle(trb);
+	bool pcs;
+
+	/*
+	 * Clear out all the TRBs part of or after the cancelled request,
+	 * and must correct trb cycle bit to the last un-enqueued state.
+	 */
+	while (trb != &ep->transfer_ring[ep->enq_ptr]) {
+		pcs = trb_read_cycle(trb);
+		memset(trb, 0, sizeof(*trb));
+		trb_write_cycle(trb, !pcs);
+		trb++;
+
+		if (trb_read_type(trb) == TRB_TYPE_LINK)
+			trb = ep->transfer_ring;
+	}
+
+	/* Requests will be re-queued at the start of the cancelled request. */
+	ep->enq_ptr = req->first_trb - ep->transfer_ring;
+	/*
+	 * Retrieve the correct cycle bit state from the first trb of
+	 * the cancelled request.
+	 */
+	ep->pcs = pcs_enq;
+	ep->ring_full = false;
+	list_for_each_entry_continue(req, &ep->queue, list) {
+		req->usb_req.status = -EINPROGRESS;
+		req->usb_req.actual = 0;
+
+		req->first_trb = NULL;
+		req->last_trb = NULL;
+		req->buf_queued = 0;
+		req->trbs_queued = 0;
+	}
+}
+
+/*
+ * Determine if the given TRB is in the range [first trb, last trb] for the
+ * given request.
+ */
+static bool trb_in_request(struct tegra_xudc_ep *ep,
+			   struct tegra_xudc_request *req,
+			   struct tegra_xudc_trb *trb)
+{
+	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
+		req->first_trb, req->last_trb, trb);
+
+	if (trb >= req->first_trb && (trb <= req->last_trb ||
+				      req->last_trb < req->first_trb))
+		return true;
+
+	if (trb < req->first_trb && trb <= req->last_trb &&
+	    req->last_trb < req->first_trb)
+		return true;
+
+	return false;
+}
+
+/*
+ * Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
+ * for the given endpoint and request.
+ */
+static bool trb_before_request(struct tegra_xudc_ep *ep,
+			       struct tegra_xudc_request *req,
+			       struct tegra_xudc_trb *trb)
+{
+	struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
+
+	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
+		__func__, req->first_trb, req->last_trb, enq_trb, trb);
+
+	if (trb < req->first_trb && (enq_trb <= trb ||
+				     req->first_trb < enq_trb))
+		return true;
+
+	if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
+		return true;
+
+	return false;
+}
+
+static int
+__tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
+			struct tegra_xudc_request *req)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+	struct tegra_xudc_request *r;
+	struct tegra_xudc_trb *deq_trb;
+	bool busy, kick_queue = false;
+	int ret = 0;
+
+	/* Make sure the request is actually queued to this endpoint. */
+	list_for_each_entry(r, &ep->queue, list) {
+		if (r == req)
+			break;
+	}
+
+	if (r != req)
+		return -EINVAL;
+
+	/* Request hasn't been queued in the transfer ring yet. */
+	if (!req->trbs_queued) {
+		tegra_xudc_req_done(ep, req, -ECONNRESET);
+		return 0;
+	}
+
+	/* Halt DMA for this endpiont. */
+	if (ep_ctx_read_state(ep->context) == EP_STATE_RUNNING) {
+		ep_pause(xudc, ep->index);
+		ep_wait_for_inactive(xudc, ep->index);
+	}
+
+	deq_trb = trb_phys_to_virt(ep, ep_ctx_read_deq_ptr(ep->context));
+	/* Is the hardware processing the TRB at the dequeue pointer? */
+	busy = (trb_read_cycle(deq_trb) == ep_ctx_read_dcs(ep->context));
+
+	if (trb_in_request(ep, req, deq_trb) && busy) {
+		/*
+		 * Request has been partially completed or it hasn't
+		 * started processing yet.
+		 */
+		dma_addr_t deq_ptr;
+
+		squeeze_transfer_ring(ep, req);
+
+		req->usb_req.actual = ep_ctx_read_edtla(ep->context);
+		tegra_xudc_req_done(ep, req, -ECONNRESET);
+		kick_queue = true;
+
+		/* EDTLA is > 0: request has been partially completed */
+		if (req->usb_req.actual > 0) {
+			/*
+			 * Abort the pending transfer and update the dequeue
+			 * pointer
+			 */
+			ep_ctx_write_edtla(ep->context, 0);
+			ep_ctx_write_partial_td(ep->context, 0);
+			ep_ctx_write_data_offset(ep->context, 0);
+
+			deq_ptr = trb_virt_to_phys(ep,
+					&ep->transfer_ring[ep->enq_ptr]);
+
+			if (dma_mapping_error(xudc->dev, deq_ptr)) {
+				ret = -EINVAL;
+			} else {
+				ep_ctx_write_deq_ptr(ep->context, deq_ptr);
+				ep_ctx_write_dcs(ep->context, ep->pcs);
+				ep_reload(xudc, ep->index);
+			}
+		}
+	} else if (trb_before_request(ep, req, deq_trb) && busy) {
+		/* Request hasn't started processing yet. */
+		squeeze_transfer_ring(ep, req);
+
+		tegra_xudc_req_done(ep, req, -ECONNRESET);
+		kick_queue = true;
+	} else {
+		/*
+		 * Request has completed, but we haven't processed the
+		 * completion event yet.
+		 */
+		tegra_xudc_req_done(ep, req, -ECONNRESET);
+		ret = -EINVAL;
+	}
+
+	/* Resume the endpoint. */
+	ep_unpause(xudc, ep->index);
+
+	if (kick_queue)
+		tegra_xudc_ep_kick_queue(ep);
+
+	return ret;
+}
+
+static int
+tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
+{
+	struct tegra_xudc_request *req;
+	struct tegra_xudc_ep *ep;
+	struct tegra_xudc *xudc;
+	unsigned long flags;
+	int ret;
+
+	if (!usb_ep || !usb_req)
+		return -EINVAL;
+
+	ep = to_xudc_ep(usb_ep);
+	req = to_xudc_req(usb_req);
+	xudc = ep->xudc;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if (xudc->powergated || !ep->desc) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+
+	ret = __tegra_xudc_ep_dequeue(ep, req);
+unlock:
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+
+	if (!ep->desc)
+		return -EINVAL;
+
+	if (usb_endpoint_xfer_isoc(ep->desc)) {
+		dev_err(xudc->dev, "can't halt isoc EP\n");
+		return -ENOTSUPP;
+	}
+
+	if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
+		dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
+			halt ? "halted" : "not halted");
+		return 0;
+	}
+
+	if (halt) {
+		ep_halt(xudc, ep->index);
+	} else {
+		ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
+
+		ep_reload(xudc, ep->index);
+
+		ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
+		ep_ctx_write_seq_num(ep->context, 0);
+
+		ep_reload(xudc, ep->index);
+		ep_unpause(xudc, ep->index);
+		ep_unhalt(xudc, ep->index);
+
+		tegra_xudc_ep_ring_doorbell(ep);
+	}
+
+	return 0;
+}
+
+static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
+{
+	struct tegra_xudc_ep *ep;
+	struct tegra_xudc *xudc;
+	unsigned long flags;
+	int ret;
+
+	if (!usb_ep)
+		return -EINVAL;
+
+	ep = to_xudc_ep(usb_ep);
+	xudc = ep->xudc;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	if (xudc->powergated) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+
+	if (value && usb_endpoint_dir_in(ep->desc) &&
+	    !list_empty(&ep->queue)) {
+		dev_err(xudc->dev, "can't halt EP with requests pending\n");
+		ret = -EAGAIN;
+		goto unlock;
+	}
+
+	ret = __tegra_xudc_ep_set_halt(ep, value);
+unlock:
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
+{
+	const struct usb_endpoint_descriptor *desc = ep->desc;
+	const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
+	struct tegra_xudc *xudc = ep->xudc;
+	u16 maxpacket, maxburst = 0, esit = 0;
+	u32 val;
+
+	maxpacket = usb_endpoint_maxp(desc) & 0x7ff;
+	if (xudc->gadget.speed == USB_SPEED_SUPER) {
+		if (!usb_endpoint_xfer_control(desc))
+			maxburst = comp_desc->bMaxBurst;
+
+		if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc))
+			esit = le16_to_cpu(comp_desc->wBytesPerInterval);
+	} else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
+		   (usb_endpoint_xfer_int(desc) ||
+		    usb_endpoint_xfer_isoc(desc))) {
+		if (xudc->gadget.speed == USB_SPEED_HIGH) {
+			maxburst = (usb_endpoint_maxp(desc) >> 11) & 0x3;
+			if (maxburst == 0x3) {
+				dev_warn(xudc->dev,
+					 "invalid endpoint maxburst\n");
+				maxburst = 0x2;
+			}
+		}
+		esit = maxpacket * (maxburst + 1);
+	}
+
+	memset(ep->context, 0, sizeof(*ep->context));
+
+	ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
+	ep_ctx_write_interval(ep->context, desc->bInterval);
+	if (xudc->gadget.speed == USB_SPEED_SUPER) {
+		if (usb_endpoint_xfer_isoc(desc)) {
+			ep_ctx_write_mult(ep->context,
+					  comp_desc->bmAttributes & 0x3);
+		}
+
+		if (usb_endpoint_xfer_bulk(desc)) {
+			ep_ctx_write_max_pstreams(ep->context,
+						  comp_desc->bmAttributes &
+						  0x1f);
+			ep_ctx_write_lsa(ep->context, 1);
+		}
+	}
+
+	if (!usb_endpoint_xfer_control(desc) && usb_endpoint_dir_out(desc))
+		val = usb_endpoint_type(desc);
+	else
+		val = usb_endpoint_type(desc) + EP_TYPE_CONTROL;
+
+	ep_ctx_write_type(ep->context, val);
+	ep_ctx_write_cerr(ep->context, 0x3);
+	ep_ctx_write_max_packet_size(ep->context, maxpacket);
+	ep_ctx_write_max_burst_size(ep->context, maxburst);
+
+	ep_ctx_write_deq_ptr(ep->context, ep->transfer_ring_phys);
+	ep_ctx_write_dcs(ep->context, ep->pcs);
+
+	/* Select a reasonable average TRB length based on endpoint type. */
+	switch (usb_endpoint_type(desc)) {
+	case USB_ENDPOINT_XFER_CONTROL:
+		val = 8;
+		break;
+	case USB_ENDPOINT_XFER_INT:
+		val = 1024;
+		break;
+	case USB_ENDPOINT_XFER_BULK:
+	case USB_ENDPOINT_XFER_ISOC:
+	default:
+		val = 3072;
+		break;
+	}
+
+	ep_ctx_write_avg_trb_len(ep->context, val);
+	ep_ctx_write_max_esit_payload(ep->context, esit);
+
+	ep_ctx_write_cerrcnt(ep->context, 0x3);
+}
+
+static void setup_link_trb(struct tegra_xudc_ep *ep,
+			   struct tegra_xudc_trb *trb)
+{
+	trb_write_data_ptr(trb, ep->transfer_ring_phys);
+	trb_write_type(trb, TRB_TYPE_LINK);
+	trb_write_toggle_cycle(trb, 1);
+}
+
+static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+
+	if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
+		dev_err(xudc->dev, "endpoint %u already disabled\n",
+			ep->index);
+		return -EINVAL;
+	}
+
+	ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
+
+	ep_reload(xudc, ep->index);
+
+	tegra_xudc_ep_nuke(ep, -ESHUTDOWN);
+
+	xudc->nr_enabled_eps--;
+	if (usb_endpoint_xfer_isoc(ep->desc))
+		xudc->nr_isoch_eps--;
+
+	ep->desc = NULL;
+	ep->comp_desc = NULL;
+
+	memset(ep->context, 0, sizeof(*ep->context));
+
+	ep_unpause(xudc, ep->index);
+	ep_unhalt(xudc, ep->index);
+	if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
+		xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
+
+	/*
+	 * If this is the last endpoint disabled in a de-configure request,
+	 * switch back to address state.
+	 */
+	if ((xudc->device_state == USB_STATE_CONFIGURED) &&
+	    (xudc->nr_enabled_eps == 1)) {
+		u32 val;
+
+		xudc->device_state = USB_STATE_ADDRESS;
+		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+
+		val = xudc_readl(xudc, CTRL);
+		val &= ~CTRL_RUN;
+		xudc_writel(xudc, val, CTRL);
+	}
+
+	dev_info(xudc->dev, "ep %u disabled\n", ep->index);
+
+	return 0;
+}
+
+static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
+{
+	struct tegra_xudc_ep *ep;
+	struct tegra_xudc *xudc;
+	unsigned long flags;
+	int ret;
+
+	if (!usb_ep)
+		return -EINVAL;
+
+	ep = to_xudc_ep(usb_ep);
+	xudc = ep->xudc;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	if (xudc->powergated) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+
+	ret = __tegra_xudc_ep_disable(ep);
+unlock:
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
+				  const struct usb_endpoint_descriptor *desc)
+{
+	struct tegra_xudc *xudc = ep->xudc;
+	unsigned int i;
+	u32 val;
+
+	if (xudc->gadget.speed == USB_SPEED_SUPER &&
+		!usb_endpoint_xfer_control(desc) && !ep->usb_ep.comp_desc)
+		return -EINVAL;
+
+	/* Disable the EP if it is not disabled */
+	if (ep_ctx_read_state(ep->context) != EP_STATE_DISABLED)
+		__tegra_xudc_ep_disable(ep);
+
+	ep->desc = desc;
+	ep->comp_desc = ep->usb_ep.comp_desc;
+
+	if (usb_endpoint_xfer_isoc(desc)) {
+		if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
+			dev_err(xudc->dev, "too many isoch endpoints\n");
+			return -EBUSY;
+		}
+		xudc->nr_isoch_eps++;
+	}
+
+	memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
+	       sizeof(*ep->transfer_ring));
+	setup_link_trb(ep, &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
+
+	ep->enq_ptr = 0;
+	ep->deq_ptr = 0;
+	ep->pcs = true;
+	ep->ring_full = false;
+	xudc->nr_enabled_eps++;
+
+	tegra_xudc_ep_context_setup(ep);
+
+	/*
+	 * No need to reload and un-halt EP0.  This will be done automatically
+	 * once a valid SETUP packet is received.
+	 */
+	if (usb_endpoint_xfer_control(desc))
+		goto out;
+
+	/*
+	 * Transition to configured state once the first non-control
+	 * endpoint is enabled.
+	 */
+	if (xudc->device_state == USB_STATE_ADDRESS) {
+		val = xudc_readl(xudc, CTRL);
+		val |= CTRL_RUN;
+		xudc_writel(xudc, val, CTRL);
+
+		xudc->device_state = USB_STATE_CONFIGURED;
+		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+	}
+
+	if (usb_endpoint_xfer_isoc(desc)) {
+		/*
+		 * Pause all bulk endpoints when enabling an isoch endpoint
+		 * to ensure the isoch endpoint is allocated enough bandwidth.
+		 */
+		for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
+			if (xudc->ep[i].desc &&
+			    usb_endpoint_xfer_bulk(xudc->ep[i].desc))
+				ep_pause(xudc, i);
+		}
+	}
+
+	ep_reload(xudc, ep->index);
+	ep_unpause(xudc, ep->index);
+	ep_unhalt(xudc, ep->index);
+
+	if (usb_endpoint_xfer_isoc(desc)) {
+		for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
+			if (xudc->ep[i].desc &&
+			    usb_endpoint_xfer_bulk(xudc->ep[i].desc))
+				ep_unpause(xudc, i);
+		}
+	}
+
+out:
+	dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
+		 usb_ep_type_string(usb_endpoint_type(ep->desc)),
+		 usb_endpoint_dir_in(ep->desc) ? "in" : "out");
+
+	return 0;
+}
+
+static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
+				const struct usb_endpoint_descriptor *desc)
+{
+	struct tegra_xudc_ep *ep;
+	struct tegra_xudc *xudc;
+	unsigned long flags;
+	int ret;
+
+	if  (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
+		return -EINVAL;
+
+	ep = to_xudc_ep(usb_ep);
+	xudc = ep->xudc;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	if (xudc->powergated) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+
+	ret = __tegra_xudc_ep_enable(ep, desc);
+unlock:
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static struct usb_request *
+tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
+{
+	struct tegra_xudc_request *req;
+
+	req = kzalloc(sizeof(*req), gfp);
+	if (!req)
+		return NULL;
+
+	INIT_LIST_HEAD(&req->list);
+
+	return &req->usb_req;
+}
+
+static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
+				       struct usb_request *usb_req)
+{
+	struct tegra_xudc_request *req = to_xudc_req(usb_req);
+
+	kfree(req);
+}
+
+static struct usb_ep_ops tegra_xudc_ep_ops = {
+	.enable = tegra_xudc_ep_enable,
+	.disable = tegra_xudc_ep_disable,
+	.alloc_request = tegra_xudc_ep_alloc_request,
+	.free_request = tegra_xudc_ep_free_request,
+	.queue = tegra_xudc_ep_queue,
+	.dequeue = tegra_xudc_ep_dequeue,
+	.set_halt = tegra_xudc_ep_set_halt,
+};
+
+static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
+				 const struct usb_endpoint_descriptor *desc)
+{
+	return -EBUSY;
+}
+
+static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
+{
+	return -EBUSY;
+}
+
+static struct usb_ep_ops tegra_xudc_ep0_ops = {
+	.enable = tegra_xudc_ep0_enable,
+	.disable = tegra_xudc_ep0_disable,
+	.alloc_request = tegra_xudc_ep_alloc_request,
+	.free_request = tegra_xudc_ep_free_request,
+	.queue = tegra_xudc_ep_queue,
+	.dequeue = tegra_xudc_ep_dequeue,
+	.set_halt = tegra_xudc_ep_set_halt,
+};
+
+static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
+{
+	struct tegra_xudc *xudc = to_xudc(gadget);
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	if (xudc->powergated) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+
+	ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
+		MFINDEX_FRAME_SHIFT;
+unlock:
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
+{
+	unsigned int i;
+	u32 val;
+
+	ep_unpause_all(xudc);
+
+	/* Direct link to U0. */
+	val = xudc_readl(xudc, PORTSC);
+	if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
+		val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
+		val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
+		xudc_writel(xudc, val, PORTSC);
+	}
+
+	if (xudc->device_state == USB_STATE_SUSPENDED) {
+		xudc->device_state = xudc->resume_state;
+		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+		xudc->resume_state = 0;
+	}
+
+	/*
+	 * Doorbells may be dropped if they are sent too soon (< ~200ns)
+	 * after unpausing the endpoint.  Wait for 500ns just to be safe.
+	 */
+	ndelay(500);
+	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
+		tegra_xudc_ep_ring_doorbell(&xudc->ep[i]);
+}
+
+static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
+{
+	struct tegra_xudc *xudc = to_xudc(gadget);
+	unsigned long flags;
+	int ret = 0;
+	u32 val;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if (xudc->powergated) {
+		ret = -ESHUTDOWN;
+		goto unlock;
+	}
+	val = xudc_readl(xudc, PORTPM);
+	dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
+			val, gadget->speed);
+
+	if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
+	     (val & PORTPM_RWE)) ||
+	    ((xudc->gadget.speed == USB_SPEED_SUPER) &&
+	     (val & PORTPM_FRWE))) {
+		tegra_xudc_resume_device_state(xudc);
+
+		/* Send Device Notification packet. */
+		if (xudc->gadget.speed == USB_SPEED_SUPER) {
+			val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
+					     | DEVNOTIF_LO_TRIG;
+			xudc_writel(xudc, 0, DEVNOTIF_HI);
+			xudc_writel(xudc, val, DEVNOTIF_LO);
+		}
+	}
+
+unlock:
+	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return ret;
+}
+
+static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
+{
+	struct tegra_xudc *xudc = to_xudc(gadget);
+	unsigned long flags;
+	u32 val;
+
+	pm_runtime_get_sync(xudc->dev);
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if (is_on != xudc->pullup) {
+		val = xudc_readl(xudc, CTRL);
+		if (is_on)
+			val |= CTRL_ENABLE;
+		else
+			val &= ~CTRL_ENABLE;
+		xudc_writel(xudc, val, CTRL);
+	}
+
+	xudc->pullup = is_on;
+	dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
+
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	pm_runtime_put(xudc->dev);
+
+	return 0;
+}
+
+static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
+				   struct usb_gadget_driver *driver)
+{
+	struct tegra_xudc *xudc = to_xudc(gadget);
+	unsigned long flags;
+	u32 val;
+	int ret;
+
+	if (!driver)
+		return -EINVAL;
+
+	pm_runtime_get_sync(xudc->dev);
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	if (xudc->driver) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	xudc->setup_state = WAIT_FOR_SETUP;
+	xudc->device_state = USB_STATE_DEFAULT;
+	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+
+	ret = __tegra_xudc_ep_enable(&xudc->ep[0], &tegra_xudc_ep0_desc);
+	if (ret < 0)
+		goto unlock;
+
+	val = xudc_readl(xudc, CTRL);
+	val |= CTRL_IE | CTRL_LSE;
+	xudc_writel(xudc, val, CTRL);
+
+	val = xudc_readl(xudc, PORTHALT);
+	val |= PORTHALT_STCHG_INTR_EN;
+	xudc_writel(xudc, val, PORTHALT);
+
+	if (xudc->pullup) {
+		val = xudc_readl(xudc, CTRL);
+		val |= CTRL_ENABLE;
+		xudc_writel(xudc, val, CTRL);
+	}
+
+	xudc->driver = driver;
+unlock:
+	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	pm_runtime_put(xudc->dev);
+
+	return ret;
+}
+
+static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
+{
+	struct tegra_xudc *xudc = to_xudc(gadget);
+	unsigned long flags;
+	u32 val;
+
+	pm_runtime_get_sync(xudc->dev);
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	val = xudc_readl(xudc, CTRL);
+	val &= ~(CTRL_IE | CTRL_ENABLE);
+	xudc_writel(xudc, val, CTRL);
+
+	__tegra_xudc_ep_disable(&xudc->ep[0]);
+
+	xudc->driver = NULL;
+	dev_dbg(xudc->dev, "Gadget stopped");
+
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	pm_runtime_put(xudc->dev);
+
+	return 0;
+}
+
+static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
+{
+	struct tegra_xudc *xudc = to_xudc(gadget);
+
+	dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
+	xudc->selfpowered = !!is_on;
+
+	return 0;
+}
+
+static struct usb_gadget_ops tegra_xudc_gadget_ops = {
+	.get_frame = tegra_xudc_gadget_get_frame,
+	.wakeup = tegra_xudc_gadget_wakeup,
+	.pullup = tegra_xudc_gadget_pullup,
+	.udc_start = tegra_xudc_gadget_start,
+	.udc_stop = tegra_xudc_gadget_stop,
+	.set_selfpowered = tegra_xudc_set_selfpowered,
+};
+
+static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
+{
+}
+
+static int
+tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
+		void (*cmpl)(struct usb_ep *, struct usb_request *))
+{
+	xudc->ep0_req->usb_req.buf = NULL;
+	xudc->ep0_req->usb_req.dma = 0;
+	xudc->ep0_req->usb_req.length = 0;
+	xudc->ep0_req->usb_req.complete = cmpl;
+	xudc->ep0_req->usb_req.context = xudc;
+
+	return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
+}
+
+static int
+tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
+		void (*cmpl)(struct usb_ep *, struct usb_request *))
+{
+	xudc->ep0_req->usb_req.buf = buf;
+	xudc->ep0_req->usb_req.length = len;
+	xudc->ep0_req->usb_req.complete = cmpl;
+	xudc->ep0_req->usb_req.context = xudc;
+
+	return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
+}
+
+static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
+{
+	switch (xudc->setup_state) {
+	case DATA_STAGE_XFER:
+		xudc->setup_state = STATUS_STAGE_RECV;
+		tegra_xudc_ep0_queue_status(xudc, no_op_complete);
+		break;
+	case DATA_STAGE_RECV:
+		xudc->setup_state = STATUS_STAGE_XFER;
+		tegra_xudc_ep0_queue_status(xudc, no_op_complete);
+		break;
+	default:
+		xudc->setup_state = WAIT_FOR_SETUP;
+		break;
+	}
+}
+
+static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
+				       struct usb_ctrlrequest *ctrl)
+{
+	int ret;
+
+	spin_unlock(&xudc->lock);
+	ret = xudc->driver->setup(&xudc->gadget, ctrl);
+	spin_lock(&xudc->lock);
+
+	return ret;
+}
+
+static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct tegra_xudc *xudc = req->context;
+
+	if (xudc->test_mode_pattern) {
+		xudc_writel(xudc, xudc->test_mode_pattern, PORT_TM);
+		xudc->test_mode_pattern = 0;
+	}
+}
+
+static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
+				      struct usb_ctrlrequest *ctrl)
+{
+	bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
+	u32 feature = le16_to_cpu(ctrl->wValue);
+	u32 index = le16_to_cpu(ctrl->wIndex);
+	u32 val, ep;
+	int ret;
+
+	if (le16_to_cpu(ctrl->wLength) != 0)
+		return -EINVAL;
+
+	switch (ctrl->bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		switch (feature) {
+		case USB_DEVICE_REMOTE_WAKEUP:
+			if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
+			    (xudc->device_state == USB_STATE_DEFAULT))
+				return -EINVAL;
+
+			val = xudc_readl(xudc, PORTPM);
+			if (set)
+				val |= PORTPM_RWE;
+			else
+				val &= ~PORTPM_RWE;
+
+			xudc_writel(xudc, val, PORTPM);
+			break;
+		case USB_DEVICE_U1_ENABLE:
+		case USB_DEVICE_U2_ENABLE:
+			if ((xudc->device_state != USB_STATE_CONFIGURED) ||
+			    (xudc->gadget.speed != USB_SPEED_SUPER))
+				return -EINVAL;
+
+			val = xudc_readl(xudc, PORTPM);
+			if ((feature == USB_DEVICE_U1_ENABLE) &&
+			     xudc->soc->u1_enable) {
+				if (set)
+					val |= PORTPM_U1E;
+				else
+					val &= ~PORTPM_U1E;
+			}
+
+			if ((feature == USB_DEVICE_U2_ENABLE) &&
+			     xudc->soc->u2_enable) {
+				if (set)
+					val |= PORTPM_U2E;
+				else
+					val &= ~PORTPM_U2E;
+			}
+
+			xudc_writel(xudc, val, PORTPM);
+			break;
+		case USB_DEVICE_TEST_MODE:
+			if (xudc->gadget.speed != USB_SPEED_HIGH)
+				return -EINVAL;
+
+			if (!set)
+				return -EINVAL;
+
+			xudc->test_mode_pattern = index >> 8;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		break;
+	case USB_RECIP_INTERFACE:
+		if (xudc->device_state != USB_STATE_CONFIGURED)
+			return -EINVAL;
+
+		switch (feature) {
+		case USB_INTRF_FUNC_SUSPEND:
+			if (set) {
+				val = xudc_readl(xudc, PORTPM);
+
+				if (index & USB_INTRF_FUNC_SUSPEND_RW)
+					val |= PORTPM_FRWE;
+				else
+					val &= ~PORTPM_FRWE;
+
+				xudc_writel(xudc, val, PORTPM);
+			}
+
+			return tegra_xudc_ep0_delegate_req(xudc, ctrl);
+		default:
+			return -EINVAL;
+		}
+
+		break;
+	case USB_RECIP_ENDPOINT:
+		ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
+			((index & USB_DIR_IN) ? 1 : 0);
+
+		if ((xudc->device_state == USB_STATE_DEFAULT) ||
+		    ((xudc->device_state == USB_STATE_ADDRESS) &&
+		     (index != 0)))
+			return -EINVAL;
+
+		ret = __tegra_xudc_ep_set_halt(&xudc->ep[ep], set);
+		if (ret < 0)
+			return ret;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return tegra_xudc_ep0_queue_status(xudc, set_feature_complete);
+}
+
+static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
+				     struct usb_ctrlrequest *ctrl)
+{
+	struct tegra_xudc_ep_context *ep_ctx;
+	u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
+	u16 status = 0;
+
+	if (!(ctrl->bRequestType & USB_DIR_IN))
+		return -EINVAL;
+
+	if ((le16_to_cpu(ctrl->wValue) != 0) ||
+	    (le16_to_cpu(ctrl->wLength) != 2))
+		return -EINVAL;
+
+	switch (ctrl->bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		val = xudc_readl(xudc, PORTPM);
+
+		if (xudc->selfpowered)
+			status |= BIT(USB_DEVICE_SELF_POWERED);
+
+		if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
+		    (val & PORTPM_RWE))
+			status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
+
+		if (xudc->gadget.speed == USB_SPEED_SUPER) {
+			if (val & PORTPM_U1E)
+				status |= BIT(USB_DEV_STAT_U1_ENABLED);
+			if (val & PORTPM_U2E)
+				status |= BIT(USB_DEV_STAT_U2_ENABLED);
+		}
+		break;
+	case USB_RECIP_INTERFACE:
+		if (xudc->gadget.speed == USB_SPEED_SUPER) {
+			status |= USB_INTRF_STAT_FUNC_RW_CAP;
+			val = xudc_readl(xudc, PORTPM);
+			if (val & PORTPM_FRWE)
+				status |= USB_INTRF_STAT_FUNC_RW;
+		}
+		break;
+	case USB_RECIP_ENDPOINT:
+		ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
+			((index & USB_DIR_IN) ? 1 : 0);
+		ep_ctx = &xudc->ep_context[ep];
+
+		if ((xudc->device_state != USB_STATE_CONFIGURED) &&
+		    ((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
+			return -EINVAL;
+
+		if (ep_ctx_read_state(ep_ctx) == EP_STATE_DISABLED)
+			return -EINVAL;
+
+		if (xudc_readl(xudc, EP_HALT) & BIT(ep))
+			status |= BIT(USB_ENDPOINT_HALT);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	xudc->status_buf = cpu_to_le16(status);
+	return tegra_xudc_ep0_queue_data(xudc, &xudc->status_buf,
+					 sizeof(xudc->status_buf),
+					 no_op_complete);
+}
+
+static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	/* Nothing to do with SEL values */
+}
+
+static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
+				  struct usb_ctrlrequest *ctrl)
+{
+	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
+				     USB_TYPE_STANDARD))
+		return -EINVAL;
+
+	if (xudc->device_state == USB_STATE_DEFAULT)
+		return -EINVAL;
+
+	if ((le16_to_cpu(ctrl->wIndex) != 0) ||
+	    (le16_to_cpu(ctrl->wValue) != 0) ||
+	    (le16_to_cpu(ctrl->wLength) != 6))
+		return -EINVAL;
+
+	return tegra_xudc_ep0_queue_data(xudc, &xudc->sel_timing,
+					 sizeof(xudc->sel_timing),
+					 set_sel_complete);
+}
+
+static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	/* Nothing to do with isoch delay */
+}
+
+static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
+					  struct usb_ctrlrequest *ctrl)
+{
+	u32 delay = le16_to_cpu(ctrl->wValue);
+
+	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
+				   USB_TYPE_STANDARD))
+		return -EINVAL;
+
+	if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
+	    (le16_to_cpu(ctrl->wLength) != 0))
+		return -EINVAL;
+
+	xudc->isoch_delay = delay;
+
+	return tegra_xudc_ep0_queue_status(xudc, set_isoch_delay_complete);
+}
+
+static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct tegra_xudc *xudc = req->context;
+
+	if ((xudc->device_state == USB_STATE_DEFAULT) &&
+	    (xudc->dev_addr != 0)) {
+		xudc->device_state = USB_STATE_ADDRESS;
+		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+	} else if ((xudc->device_state == USB_STATE_ADDRESS) &&
+		   (xudc->dev_addr == 0)) {
+		xudc->device_state = USB_STATE_DEFAULT;
+		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+	}
+}
+
+static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
+				      struct usb_ctrlrequest *ctrl)
+{
+	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
+	u32 val, addr = le16_to_cpu(ctrl->wValue);
+
+	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
+				     USB_TYPE_STANDARD))
+		return -EINVAL;
+
+	if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
+	    (le16_to_cpu(ctrl->wLength) != 0))
+		return -EINVAL;
+
+	if (xudc->device_state == USB_STATE_CONFIGURED)
+		return -EINVAL;
+
+	dev_dbg(xudc->dev, "set address: %u\n", addr);
+
+	xudc->dev_addr = addr;
+	val = xudc_readl(xudc, CTRL);
+	val &= ~(CTRL_DEVADDR_MASK);
+	val |= CTRL_DEVADDR(addr);
+	xudc_writel(xudc, val, CTRL);
+
+	ep_ctx_write_devaddr(ep0->context, addr);
+
+	return tegra_xudc_ep0_queue_status(xudc, set_address_complete);
+}
+
+static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
+				      struct usb_ctrlrequest *ctrl)
+{
+	int ret;
+
+	switch (ctrl->bRequest) {
+	case USB_REQ_GET_STATUS:
+		dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
+		ret = tegra_xudc_ep0_get_status(xudc, ctrl);
+		break;
+	case USB_REQ_SET_ADDRESS:
+		dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
+		ret = tegra_xudc_ep0_set_address(xudc, ctrl);
+		break;
+	case USB_REQ_SET_SEL:
+		dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
+		ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
+		break;
+	case USB_REQ_SET_ISOCH_DELAY:
+		dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
+		ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
+		break;
+	case USB_REQ_CLEAR_FEATURE:
+	case USB_REQ_SET_FEATURE:
+		dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
+		ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
+		break;
+	case USB_REQ_SET_CONFIGURATION:
+		dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
+		/*
+		 * In theory we need to clear RUN bit before status stage of
+		 * deconfig request sent, but this seems to be causing problems.
+		 * Clear RUN once all endpoints are disabled instead.
+		 */
+		fallthrough;
+	default:
+		ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
+		break;
+	}
+
+	return ret;
+}
+
+static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
+					       struct usb_ctrlrequest *ctrl,
+					       u16 seq_num)
+{
+	int ret;
+
+	xudc->setup_seq_num = seq_num;
+
+	/* Ensure EP0 is unhalted. */
+	ep_unhalt(xudc, 0);
+
+	/*
+	 * On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
+	 * are invalid.  Halt EP0 until we get a valid packet.
+	 */
+	if (xudc->soc->invalid_seq_num &&
+	    (seq_num == 0xfffe || seq_num == 0xffff)) {
+		dev_warn(xudc->dev, "invalid sequence number detected\n");
+		ep_halt(xudc, 0);
+		return;
+	}
+
+	if (ctrl->wLength)
+		xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
+			DATA_STAGE_XFER :  DATA_STAGE_RECV;
+	else
+		xudc->setup_state = STATUS_STAGE_XFER;
+
+	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
+		ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
+	else
+		ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
+
+	if (ret < 0) {
+		dev_warn(xudc->dev, "setup request failed: %d\n", ret);
+		xudc->setup_state = WAIT_FOR_SETUP;
+		ep_halt(xudc, 0);
+	}
+}
+
+static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
+					struct tegra_xudc_trb *event)
+{
+	struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
+	u16 seq_num = trb_read_seq_num(event);
+
+	if (xudc->setup_state != WAIT_FOR_SETUP) {
+		/*
+		 * The controller is in the process of handling another
+		 * setup request.  Queue subsequent requests and handle
+		 * the last one once the controller reports a sequence
+		 * number error.
+		 */
+		memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
+		xudc->setup_packet.seq_num = seq_num;
+		xudc->queued_setup_packet = true;
+	} else {
+		tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
+	}
+}
+
+static struct tegra_xudc_request *
+trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
+{
+	struct tegra_xudc_request *req;
+
+	list_for_each_entry(req, &ep->queue, list) {
+		if (!req->trbs_queued)
+			break;
+
+		if (trb_in_request(ep, req, trb))
+			return req;
+	}
+
+	return NULL;
+}
+
+static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
+						  struct tegra_xudc_ep *ep,
+						  struct tegra_xudc_trb *event)
+{
+	struct tegra_xudc_request *req;
+	struct tegra_xudc_trb *trb;
+	bool short_packet;
+
+	short_packet = (trb_read_cmpl_code(event) ==
+			TRB_CMPL_CODE_SHORT_PACKET);
+
+	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
+	req = trb_to_request(ep, trb);
+
+	/*
+	 * TDs are complete on short packet or when the completed TRB is the
+	 * last TRB in the TD (the CHAIN bit is unset).
+	 */
+	if (req && (short_packet || (!trb_read_chain(trb) &&
+		(req->trbs_needed == req->trbs_queued)))) {
+		struct tegra_xudc_trb *last = req->last_trb;
+		unsigned int residual;
+
+		residual = trb_read_transfer_len(event);
+		req->usb_req.actual = req->usb_req.length - residual;
+
+		dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
+			req->usb_req.actual, req->usb_req.length);
+
+		tegra_xudc_req_done(ep, req, 0);
+
+		if (ep->desc && usb_endpoint_xfer_control(ep->desc))
+			tegra_xudc_ep0_req_done(xudc);
+
+		/*
+		 * Advance the dequeue pointer past the end of the current TD
+		 * on short packet completion.
+		 */
+		if (short_packet) {
+			ep->deq_ptr = (last - ep->transfer_ring) + 1;
+			if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
+				ep->deq_ptr = 0;
+		}
+	} else if (!req) {
+		dev_warn(xudc->dev, "transfer event on dequeued request\n");
+	}
+
+	if (ep->desc)
+		tegra_xudc_ep_kick_queue(ep);
+}
+
+static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
+					     struct tegra_xudc_trb *event)
+{
+	unsigned int ep_index = trb_read_endpoint_id(event);
+	struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
+	struct tegra_xudc_trb *trb;
+	u16 comp_code;
+
+	if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
+		dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
+			 ep_index);
+		return;
+	}
+
+	/* Update transfer ring dequeue pointer. */
+	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
+	comp_code = trb_read_cmpl_code(event);
+	if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
+		ep->deq_ptr = (trb - ep->transfer_ring) + 1;
+
+		if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
+			ep->deq_ptr = 0;
+		ep->ring_full = false;
+	}
+
+	switch (comp_code) {
+	case TRB_CMPL_CODE_SUCCESS:
+	case TRB_CMPL_CODE_SHORT_PACKET:
+		tegra_xudc_handle_transfer_completion(xudc, ep, event);
+		break;
+	case TRB_CMPL_CODE_HOST_REJECTED:
+		dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
+
+		ep->stream_rejected = true;
+		break;
+	case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
+		dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
+
+		if (ep->stream_rejected) {
+			ep->stream_rejected = false;
+			/*
+			 * An EP is stopped when a stream is rejected.  Wait
+			 * for the EP to report that it is stopped and then
+			 * un-stop it.
+			 */
+			ep_wait_for_stopped(xudc, ep_index);
+		}
+		tegra_xudc_ep_ring_doorbell(ep);
+		break;
+	case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
+		/*
+		 * Wait for the EP to be stopped so the controller stops
+		 * processing doorbells.
+		 */
+		ep_wait_for_stopped(xudc, ep_index);
+		ep->enq_ptr = ep->deq_ptr;
+		tegra_xudc_ep_nuke(ep, -EIO);
+		/* FALLTHROUGH */
+	case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
+	case TRB_CMPL_CODE_CTRL_DIR_ERR:
+	case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
+	case TRB_CMPL_CODE_RING_UNDERRUN:
+	case TRB_CMPL_CODE_RING_OVERRUN:
+	case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
+	case TRB_CMPL_CODE_USB_TRANS_ERR:
+	case TRB_CMPL_CODE_TRB_ERR:
+		dev_err(xudc->dev, "completion error %#x on EP %u\n",
+			comp_code, ep_index);
+
+		ep_halt(xudc, ep_index);
+		break;
+	case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
+		dev_info(xudc->dev, "sequence number error\n");
+
+		/*
+		 * Kill any queued control request and skip to the last
+		 * setup packet we received.
+		 */
+		tegra_xudc_ep_nuke(ep, -EINVAL);
+		xudc->setup_state = WAIT_FOR_SETUP;
+		if (!xudc->queued_setup_packet)
+			break;
+
+		tegra_xudc_handle_ep0_setup_packet(xudc,
+						   &xudc->setup_packet.ctrl_req,
+						   xudc->setup_packet.seq_num);
+		xudc->queued_setup_packet = false;
+		break;
+	case TRB_CMPL_CODE_STOPPED:
+		dev_dbg(xudc->dev, "stop completion code on EP %u\n",
+			ep_index);
+
+		/* Disconnected. */
+		tegra_xudc_ep_nuke(ep, -ECONNREFUSED);
+		break;
+	default:
+		dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
+			comp_code, ep_index);
+		break;
+	}
+}
+
+static void tegra_xudc_reset(struct tegra_xudc *xudc)
+{
+	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
+	dma_addr_t deq_ptr;
+	unsigned int i;
+
+	xudc->setup_state = WAIT_FOR_SETUP;
+	xudc->device_state = USB_STATE_DEFAULT;
+	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+
+	ep_unpause_all(xudc);
+
+	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
+		tegra_xudc_ep_nuke(&xudc->ep[i], -ESHUTDOWN);
+
+	/*
+	 * Reset sequence number and dequeue pointer to flush the transfer
+	 * ring.
+	 */
+	ep0->deq_ptr = ep0->enq_ptr;
+	ep0->ring_full = false;
+
+	xudc->setup_seq_num = 0;
+	xudc->queued_setup_packet = false;
+
+	ep_ctx_write_seq_num(ep0->context, xudc->setup_seq_num);
+
+	deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
+
+	if (!dma_mapping_error(xudc->dev, deq_ptr)) {
+		ep_ctx_write_deq_ptr(ep0->context, deq_ptr);
+		ep_ctx_write_dcs(ep0->context, ep0->pcs);
+	}
+
+	ep_unhalt_all(xudc);
+	ep_reload(xudc, 0);
+	ep_unpause(xudc, 0);
+}
+
+static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
+{
+	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
+	u16 maxpacket;
+	u32 val;
+
+	val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
+	switch (val) {
+	case PORTSC_PS_LS:
+		xudc->gadget.speed = USB_SPEED_LOW;
+		break;
+	case PORTSC_PS_FS:
+		xudc->gadget.speed = USB_SPEED_FULL;
+		break;
+	case PORTSC_PS_HS:
+		xudc->gadget.speed = USB_SPEED_HIGH;
+		break;
+	case PORTSC_PS_SS:
+		xudc->gadget.speed = USB_SPEED_SUPER;
+		break;
+	default:
+		xudc->gadget.speed = USB_SPEED_UNKNOWN;
+		break;
+	}
+
+	xudc->device_state = USB_STATE_DEFAULT;
+	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+
+	xudc->setup_state = WAIT_FOR_SETUP;
+
+	if (xudc->gadget.speed == USB_SPEED_SUPER)
+		maxpacket = 512;
+	else
+		maxpacket = 64;
+
+	ep_ctx_write_max_packet_size(ep0->context, maxpacket);
+	tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
+	usb_ep_set_maxpacket_limit(&ep0->usb_ep, maxpacket);
+
+	if (!xudc->soc->u1_enable) {
+		val = xudc_readl(xudc, PORTPM);
+		val &= ~(PORTPM_U1TIMEOUT_MASK);
+		xudc_writel(xudc, val, PORTPM);
+	}
+
+	if (!xudc->soc->u2_enable) {
+		val = xudc_readl(xudc, PORTPM);
+		val &= ~(PORTPM_U2TIMEOUT_MASK);
+		xudc_writel(xudc, val, PORTPM);
+	}
+
+	if (xudc->gadget.speed <= USB_SPEED_HIGH) {
+		val = xudc_readl(xudc, PORTPM);
+		val &= ~(PORTPM_L1S_MASK);
+		if (xudc->soc->lpm_enable)
+			val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
+		else
+			val |= PORTPM_L1S(PORTPM_L1S_NYET);
+		xudc_writel(xudc, val, PORTPM);
+	}
+
+	val = xudc_readl(xudc, ST);
+	if (val & ST_RC)
+		xudc_writel(xudc, ST_RC, ST);
+}
+
+static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
+{
+	tegra_xudc_reset(xudc);
+
+	if (xudc->driver && xudc->driver->disconnect) {
+		spin_unlock(&xudc->lock);
+		xudc->driver->disconnect(&xudc->gadget);
+		spin_lock(&xudc->lock);
+	}
+
+	xudc->device_state = USB_STATE_NOTATTACHED;
+	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+
+	complete(&xudc->disconnect_complete);
+}
+
+static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
+{
+	tegra_xudc_reset(xudc);
+
+	if (xudc->driver) {
+		spin_unlock(&xudc->lock);
+		usb_gadget_udc_reset(&xudc->gadget, xudc->driver);
+		spin_lock(&xudc->lock);
+	}
+
+	tegra_xudc_port_connect(xudc);
+}
+
+static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
+{
+	dev_dbg(xudc->dev, "port suspend\n");
+
+	xudc->resume_state = xudc->device_state;
+	xudc->device_state = USB_STATE_SUSPENDED;
+	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
+
+	if (xudc->driver->suspend) {
+		spin_unlock(&xudc->lock);
+		xudc->driver->suspend(&xudc->gadget);
+		spin_lock(&xudc->lock);
+	}
+}
+
+static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
+{
+	dev_dbg(xudc->dev, "port resume\n");
+
+	tegra_xudc_resume_device_state(xudc);
+
+	if (xudc->driver->resume) {
+		spin_unlock(&xudc->lock);
+		xudc->driver->resume(&xudc->gadget);
+		spin_lock(&xudc->lock);
+	}
+}
+
+static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
+{
+	u32 val;
+
+	val = xudc_readl(xudc, PORTSC);
+	val &= ~PORTSC_CHANGE_MASK;
+	val |= flag;
+	xudc_writel(xudc, val, PORTSC);
+}
+
+static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
+{
+	u32 portsc, porthalt;
+
+	porthalt = xudc_readl(xudc, PORTHALT);
+	if ((porthalt & PORTHALT_STCHG_REQ) &&
+	    (porthalt & PORTHALT_HALT_LTSSM)) {
+		dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
+		porthalt &= ~PORTHALT_HALT_LTSSM;
+		xudc_writel(xudc, porthalt, PORTHALT);
+	}
+
+	portsc = xudc_readl(xudc, PORTSC);
+	if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
+		dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
+		clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
+#define TOGGLE_VBUS_WAIT_MS 100
+		if (xudc->soc->port_reset_quirk) {
+			schedule_delayed_work(&xudc->port_reset_war_work,
+				msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
+			xudc->wait_for_sec_prc = 1;
+		}
+	}
+
+	if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
+		dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
+		clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
+		tegra_xudc_port_reset(xudc);
+		cancel_delayed_work(&xudc->port_reset_war_work);
+		xudc->wait_for_sec_prc = 0;
+	}
+
+	portsc = xudc_readl(xudc, PORTSC);
+	if (portsc & PORTSC_WRC) {
+		dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
+		clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
+		if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
+			tegra_xudc_port_reset(xudc);
+	}
+
+	portsc = xudc_readl(xudc, PORTSC);
+	if (portsc & PORTSC_CSC) {
+		dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
+		clear_port_change(xudc, PORTSC_CSC);
+
+		if (portsc & PORTSC_CCS)
+			tegra_xudc_port_connect(xudc);
+		else
+			tegra_xudc_port_disconnect(xudc);
+
+		if (xudc->wait_csc) {
+			cancel_delayed_work(&xudc->plc_reset_work);
+			xudc->wait_csc = false;
+		}
+	}
+
+	portsc = xudc_readl(xudc, PORTSC);
+	if (portsc & PORTSC_PLC) {
+		u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
+
+		dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
+		clear_port_change(xudc, PORTSC_PLC);
+		switch (pls) {
+		case PORTSC_PLS_U3:
+			tegra_xudc_port_suspend(xudc);
+			break;
+		case PORTSC_PLS_U0:
+			if (xudc->gadget.speed < USB_SPEED_SUPER)
+				tegra_xudc_port_resume(xudc);
+			break;
+		case PORTSC_PLS_RESUME:
+			if (xudc->gadget.speed == USB_SPEED_SUPER)
+				tegra_xudc_port_resume(xudc);
+			break;
+		case PORTSC_PLS_INACTIVE:
+			schedule_delayed_work(&xudc->plc_reset_work,
+					msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
+			xudc->wait_csc = true;
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (portsc & PORTSC_CEC) {
+		dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
+		clear_port_change(xudc, PORTSC_CEC);
+	}
+
+	dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
+}
+
+static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
+{
+	while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
+	       (xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
+		__tegra_xudc_handle_port_status(xudc);
+}
+
+static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
+				    struct tegra_xudc_trb *event)
+{
+	u32 type = trb_read_type(event);
+
+	dump_trb(xudc, "EVENT", event);
+
+	switch (type) {
+	case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
+		tegra_xudc_handle_port_status(xudc);
+		break;
+	case TRB_TYPE_TRANSFER_EVENT:
+		tegra_xudc_handle_transfer_event(xudc, event);
+		break;
+	case TRB_TYPE_SETUP_PACKET_EVENT:
+		tegra_xudc_handle_ep0_event(xudc, event);
+		break;
+	default:
+		dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
+		break;
+	}
+}
+
+static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
+{
+	struct tegra_xudc_trb *event;
+	dma_addr_t erdp;
+
+	while (true) {
+		event = xudc->event_ring[xudc->event_ring_index] +
+			xudc->event_ring_deq_ptr;
+
+		if (trb_read_cycle(event) != xudc->ccs)
+			break;
+
+		tegra_xudc_handle_event(xudc, event);
+
+		xudc->event_ring_deq_ptr++;
+		if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
+			xudc->event_ring_deq_ptr = 0;
+			xudc->event_ring_index++;
+		}
+
+		if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
+			xudc->event_ring_index = 0;
+			xudc->ccs = !xudc->ccs;
+		}
+	}
+
+	erdp = xudc->event_ring_phys[xudc->event_ring_index] +
+		xudc->event_ring_deq_ptr * sizeof(*event);
+
+	xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
+	xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
+}
+
+static irqreturn_t tegra_xudc_irq(int irq, void *data)
+{
+	struct tegra_xudc *xudc = data;
+	unsigned long flags;
+	u32 val;
+
+	val = xudc_readl(xudc, ST);
+	if (!(val & ST_IP))
+		return IRQ_NONE;
+	xudc_writel(xudc, ST_IP, ST);
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	tegra_xudc_process_event_ring(xudc);
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
+{
+	struct tegra_xudc_ep *ep = &xudc->ep[index];
+
+	ep->xudc = xudc;
+	ep->index = index;
+	ep->context = &xudc->ep_context[index];
+	INIT_LIST_HEAD(&ep->queue);
+
+	/*
+	 * EP1 would be the input endpoint corresponding to EP0, but since
+	 * EP0 is bi-directional, EP1 is unused.
+	 */
+	if (index == 1)
+		return 0;
+
+	ep->transfer_ring = dma_pool_alloc(xudc->transfer_ring_pool,
+					   GFP_KERNEL,
+					   &ep->transfer_ring_phys);
+	if (!ep->transfer_ring)
+		return -ENOMEM;
+
+	if (index) {
+		snprintf(ep->name, sizeof(ep->name), "ep%u%s", index / 2,
+			 (index % 2 == 0) ? "out" : "in");
+		ep->usb_ep.name = ep->name;
+		usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
+		ep->usb_ep.max_streams = 16;
+		ep->usb_ep.ops = &tegra_xudc_ep_ops;
+		ep->usb_ep.caps.type_bulk = true;
+		ep->usb_ep.caps.type_int = true;
+		if (index & 1)
+			ep->usb_ep.caps.dir_in = true;
+		else
+			ep->usb_ep.caps.dir_out = true;
+		list_add_tail(&ep->usb_ep.ep_list, &xudc->gadget.ep_list);
+	} else {
+		strscpy(ep->name, "ep0", 3);
+		ep->usb_ep.name = ep->name;
+		usb_ep_set_maxpacket_limit(&ep->usb_ep, 512);
+		ep->usb_ep.ops = &tegra_xudc_ep0_ops;
+		ep->usb_ep.caps.type_control = true;
+		ep->usb_ep.caps.dir_in = true;
+		ep->usb_ep.caps.dir_out = true;
+	}
+
+	return 0;
+}
+
+static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
+{
+	struct tegra_xudc_ep *ep = &xudc->ep[index];
+
+	/*
+	 * EP1 would be the input endpoint corresponding to EP0, but since
+	 * EP0 is bi-directional, EP1 is unused.
+	 */
+	if (index == 1)
+		return;
+
+	dma_pool_free(xudc->transfer_ring_pool, ep->transfer_ring,
+		      ep->transfer_ring_phys);
+}
+
+static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
+{
+	struct usb_request *req;
+	unsigned int i;
+	int err;
+
+	xudc->ep_context =
+		dma_alloc_coherent(xudc->dev, XUDC_NR_EPS *
+				    sizeof(*xudc->ep_context),
+				    &xudc->ep_context_phys, GFP_KERNEL);
+	if (!xudc->ep_context)
+		return -ENOMEM;
+
+	xudc->transfer_ring_pool =
+		dmam_pool_create(dev_name(xudc->dev), xudc->dev,
+				 XUDC_TRANSFER_RING_SIZE *
+				 sizeof(struct tegra_xudc_trb),
+				 sizeof(struct tegra_xudc_trb), 0);
+	if (!xudc->transfer_ring_pool) {
+		err = -ENOMEM;
+		goto free_ep_context;
+	}
+
+	INIT_LIST_HEAD(&xudc->gadget.ep_list);
+	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
+		err = tegra_xudc_alloc_ep(xudc, i);
+		if (err < 0)
+			goto free_eps;
+	}
+
+	req = tegra_xudc_ep_alloc_request(&xudc->ep[0].usb_ep, GFP_KERNEL);
+	if (!req) {
+		err = -ENOMEM;
+		goto free_eps;
+	}
+	xudc->ep0_req = to_xudc_req(req);
+
+	return 0;
+
+free_eps:
+	for (; i > 0; i--)
+		tegra_xudc_free_ep(xudc, i - 1);
+free_ep_context:
+	dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
+			  xudc->ep_context, xudc->ep_context_phys);
+	return err;
+}
+
+static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
+{
+	xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
+	xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
+}
+
+static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
+{
+	unsigned int i;
+
+	tegra_xudc_ep_free_request(&xudc->ep[0].usb_ep,
+				   &xudc->ep0_req->usb_req);
+
+	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
+		tegra_xudc_free_ep(xudc, i);
+
+	dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
+			  xudc->ep_context, xudc->ep_context_phys);
+}
+
+static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
+		xudc->event_ring[i] =
+			dma_alloc_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
+					   sizeof(*xudc->event_ring[i]),
+					   &xudc->event_ring_phys[i],
+					   GFP_KERNEL);
+		if (!xudc->event_ring[i])
+			goto free_dma;
+	}
+
+	return 0;
+
+free_dma:
+	for (; i > 0; i--) {
+		dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
+				  sizeof(*xudc->event_ring[i - 1]),
+				  xudc->event_ring[i - 1],
+				  xudc->event_ring_phys[i - 1]);
+	}
+	return -ENOMEM;
+}
+
+static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
+{
+	unsigned int i;
+	u32 val;
+
+	val = xudc_readl(xudc, SPARAM);
+	val &= ~(SPARAM_ERSTMAX_MASK);
+	val |= SPARAM_ERSTMAX(XUDC_NR_EVENT_RINGS);
+	xudc_writel(xudc, val, SPARAM);
+
+	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
+		memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
+		       sizeof(*xudc->event_ring[i]));
+
+		val = xudc_readl(xudc, ERSTSZ);
+		val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
+		val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
+		xudc_writel(xudc, val, ERSTSZ);
+
+		xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
+			    ERSTXBALO(i));
+		xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
+			    ERSTXBAHI(i));
+	}
+
+	val = lower_32_bits(xudc->event_ring_phys[0]);
+	xudc_writel(xudc, val, ERDPLO);
+	val |= EREPLO_ECS;
+	xudc_writel(xudc, val, EREPLO);
+
+	val = upper_32_bits(xudc->event_ring_phys[0]);
+	xudc_writel(xudc, val, ERDPHI);
+	xudc_writel(xudc, val, EREPHI);
+
+	xudc->ccs = true;
+	xudc->event_ring_index = 0;
+	xudc->event_ring_deq_ptr = 0;
+}
+
+static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
+		dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
+				  sizeof(*xudc->event_ring[i]),
+				  xudc->event_ring[i],
+				  xudc->event_ring_phys[i]);
+	}
+}
+
+static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
+{
+	u32 val;
+
+	if (xudc->soc->has_ipfs) {
+		val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
+		val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
+		ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
+		usleep_range(10, 15);
+	}
+
+	/* Enable bus master */
+	val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
+		XUSB_DEV_CFG_1_BUS_MASTER_EN;
+	fpci_writel(xudc, val, XUSB_DEV_CFG_1);
+
+	/* Program BAR0 space */
+	val = fpci_readl(xudc, XUSB_DEV_CFG_4);
+	val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
+	val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
+
+	fpci_writel(xudc, val, XUSB_DEV_CFG_4);
+	fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
+
+	usleep_range(100, 200);
+
+	if (xudc->soc->has_ipfs) {
+		/* Enable interrupt assertion */
+		val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
+		val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
+		ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
+	}
+}
+
+static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
+{
+	u32 val, imod;
+
+	if (xudc->soc->has_ipfs) {
+		val = xudc_readl(xudc, BLCG);
+		val |= BLCG_ALL;
+		val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
+				BLCG_COREPLL_PWRDN);
+		val |= BLCG_IOPLL_0_PWRDN;
+		val |= BLCG_IOPLL_1_PWRDN;
+		val |= BLCG_IOPLL_2_PWRDN;
+
+		xudc_writel(xudc, val, BLCG);
+	}
+
+	/* Set a reasonable U3 exit timer value. */
+	val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
+	val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
+	val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
+	xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
+
+	/* Default ping LFPS tBurst is too large. */
+	val = xudc_readl(xudc, SSPX_CORE_CNT0);
+	val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
+	val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
+	xudc_writel(xudc, val, SSPX_CORE_CNT0);
+
+	/* Default tPortConfiguration timeout is too small. */
+	val = xudc_readl(xudc, SSPX_CORE_CNT30);
+	val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
+	val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
+	xudc_writel(xudc, val, SSPX_CORE_CNT30);
+
+	if (xudc->soc->lpm_enable) {
+		/* Set L1 resume duration to 95 us. */
+		val = xudc_readl(xudc, HSFSPI_COUNT13);
+		val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
+		val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
+		xudc_writel(xudc, val, HSFSPI_COUNT13);
+	}
+
+	/*
+	 * Compliacne suite appears to be violating polling LFPS tBurst max
+	 * of 1.4us.  Send 1.45us instead.
+	 */
+	val = xudc_readl(xudc, SSPX_CORE_CNT32);
+	val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
+	val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
+	xudc_writel(xudc, val, SSPX_CORE_CNT32);
+
+	/* Direct HS/FS port instance to RxDetect. */
+	val = xudc_readl(xudc, CFG_DEV_FE);
+	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
+	val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
+	xudc_writel(xudc, val, CFG_DEV_FE);
+
+	val = xudc_readl(xudc, PORTSC);
+	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
+	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
+	xudc_writel(xudc, val, PORTSC);
+
+	/* Direct SS port instance to RxDetect. */
+	val = xudc_readl(xudc, CFG_DEV_FE);
+	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
+	val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
+	xudc_writel(xudc, val, CFG_DEV_FE);
+
+	val = xudc_readl(xudc, PORTSC);
+	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
+	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
+	xudc_writel(xudc, val, PORTSC);
+
+	/* Restore port instance. */
+	val = xudc_readl(xudc, CFG_DEV_FE);
+	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
+	xudc_writel(xudc, val, CFG_DEV_FE);
+
+	/*
+	 * Enable INFINITE_SS_RETRY to prevent device from entering
+	 * Disabled.Error when attached to buggy SuperSpeed hubs.
+	 */
+	val = xudc_readl(xudc, CFG_DEV_FE);
+	val |= CFG_DEV_FE_INFINITE_SS_RETRY;
+	xudc_writel(xudc, val, CFG_DEV_FE);
+
+	/* Set interrupt moderation. */
+	imod = XUDC_INTERRUPT_MODERATION_US * 4;
+	val = xudc_readl(xudc, RT_IMOD);
+	val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
+	val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
+	xudc_writel(xudc, val, RT_IMOD);
+
+	/* increase SSPI transaction timeout from 32us to 512us */
+	val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
+	val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
+	val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
+	xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
+}
+
+static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
+{
+	int err;
+
+	err = phy_init(xudc->utmi_phy);
+	if (err < 0) {
+		dev_err(xudc->dev, "utmi phy init failed: %d\n", err);
+		return err;
+	}
+
+	err = phy_init(xudc->usb3_phy);
+	if (err < 0) {
+		dev_err(xudc->dev, "usb3 phy init failed: %d\n", err);
+		goto exit_utmi_phy;
+	}
+
+	return 0;
+
+exit_utmi_phy:
+	phy_exit(xudc->utmi_phy);
+	return err;
+}
+
+static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
+{
+	phy_exit(xudc->usb3_phy);
+	phy_exit(xudc->utmi_phy);
+}
+
+static const char * const tegra210_xudc_supply_names[] = {
+	"hvdd-usb",
+	"avddio-usb",
+};
+
+static const char * const tegra210_xudc_clock_names[] = {
+	"dev",
+	"ss",
+	"ss_src",
+	"hs_src",
+	"fs_src",
+};
+
+static const char * const tegra186_xudc_clock_names[] = {
+	"dev",
+	"ss",
+	"ss_src",
+	"fs_src",
+};
+
+static struct tegra_xudc_soc tegra210_xudc_soc_data = {
+	.supply_names = tegra210_xudc_supply_names,
+	.num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
+	.clock_names = tegra210_xudc_clock_names,
+	.num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
+	.u1_enable = false,
+	.u2_enable = true,
+	.lpm_enable = false,
+	.invalid_seq_num = true,
+	.pls_quirk = true,
+	.port_reset_quirk = true,
+	.has_ipfs = true,
+};
+
+static struct tegra_xudc_soc tegra186_xudc_soc_data = {
+	.clock_names = tegra186_xudc_clock_names,
+	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
+	.u1_enable = true,
+	.u2_enable = true,
+	.lpm_enable = false,
+	.invalid_seq_num = false,
+	.pls_quirk = false,
+	.port_reset_quirk = false,
+	.has_ipfs = false,
+};
+
+static const struct of_device_id tegra_xudc_of_match[] = {
+	{
+		.compatible = "nvidia,tegra210-xudc",
+		.data = &tegra210_xudc_soc_data
+	},
+	{
+		.compatible = "nvidia,tegra186-xudc",
+		.data = &tegra186_xudc_soc_data
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
+
+static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
+{
+	if (xudc->genpd_dl_ss)
+		device_link_del(xudc->genpd_dl_ss);
+	if (xudc->genpd_dl_device)
+		device_link_del(xudc->genpd_dl_device);
+	if (xudc->genpd_dev_ss)
+		dev_pm_domain_detach(xudc->genpd_dev_ss, true);
+	if (xudc->genpd_dev_device)
+		dev_pm_domain_detach(xudc->genpd_dev_device, true);
+}
+
+static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
+{
+	struct device *dev = xudc->dev;
+	int err;
+
+	xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev,
+								"dev");
+	if (IS_ERR(xudc->genpd_dev_device)) {
+		err = PTR_ERR(xudc->genpd_dev_device);
+		dev_err(dev, "failed to get dev pm-domain: %d\n", err);
+		return err;
+	}
+
+	xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, "ss");
+	if (IS_ERR(xudc->genpd_dev_ss)) {
+		err = PTR_ERR(xudc->genpd_dev_ss);
+		dev_err(dev, "failed to get superspeed pm-domain: %d\n", err);
+		return err;
+	}
+
+	xudc->genpd_dl_device = device_link_add(dev, xudc->genpd_dev_device,
+					       DL_FLAG_PM_RUNTIME |
+					       DL_FLAG_STATELESS);
+	if (!xudc->genpd_dl_device) {
+		dev_err(dev, "adding usb device device link failed!\n");
+		return -ENODEV;
+	}
+
+	xudc->genpd_dl_ss = device_link_add(dev, xudc->genpd_dev_ss,
+					     DL_FLAG_PM_RUNTIME |
+					     DL_FLAG_STATELESS);
+	if (!xudc->genpd_dl_ss) {
+		dev_err(dev, "adding superspeed device link failed!\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int tegra_xudc_probe(struct platform_device *pdev)
+{
+	struct tegra_xudc *xudc;
+	struct resource *res;
+	struct usb_role_switch_desc role_sx_desc = { 0 };
+	unsigned int i;
+	int err;
+
+	xudc = devm_kzalloc(&pdev->dev, sizeof(*xudc), GFP_ATOMIC);
+	if (!xudc)
+		return -ENOMEM;
+
+	xudc->dev = &pdev->dev;
+	platform_set_drvdata(pdev, xudc);
+
+	xudc->soc = of_device_get_match_data(&pdev->dev);
+	if (!xudc->soc)
+		return -ENODEV;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
+	xudc->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(xudc->base))
+		return PTR_ERR(xudc->base);
+	xudc->phys_base = res->start;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fpci");
+	xudc->fpci = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(xudc->fpci))
+		return PTR_ERR(xudc->fpci);
+
+	if (xudc->soc->has_ipfs) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   "ipfs");
+		xudc->ipfs = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(xudc->ipfs))
+			return PTR_ERR(xudc->ipfs);
+	}
+
+	xudc->irq = platform_get_irq(pdev, 0);
+	if (xudc->irq < 0) {
+		dev_err(xudc->dev, "failed to get IRQ: %d\n",
+				xudc->irq);
+		return xudc->irq;
+	}
+
+	err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
+			       dev_name(&pdev->dev), xudc);
+	if (err < 0) {
+		dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
+			err);
+		return err;
+	}
+
+	xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks,
+				      sizeof(*xudc->clks), GFP_KERNEL);
+	if (!xudc->clks)
+		return -ENOMEM;
+
+	for (i = 0; i < xudc->soc->num_clks; i++)
+		xudc->clks[i].id = xudc->soc->clock_names[i];
+
+	err = devm_clk_bulk_get(&pdev->dev, xudc->soc->num_clks,
+				      xudc->clks);
+	if (err) {
+		dev_err(xudc->dev, "failed to request clks %d\n", err);
+		return err;
+	}
+
+	xudc->supplies = devm_kcalloc(&pdev->dev, xudc->soc->num_supplies,
+				      sizeof(*xudc->supplies), GFP_KERNEL);
+	if (!xudc->supplies)
+		return -ENOMEM;
+
+	for (i = 0; i < xudc->soc->num_supplies; i++)
+		xudc->supplies[i].supply = xudc->soc->supply_names[i];
+
+	err = devm_regulator_bulk_get(&pdev->dev, xudc->soc->num_supplies,
+				      xudc->supplies);
+	if (err) {
+		dev_err(xudc->dev, "failed to request regulators %d\n", err);
+		return err;
+	}
+
+	xudc->padctl = tegra_xusb_padctl_get(&pdev->dev);
+	if (IS_ERR(xudc->padctl))
+		return PTR_ERR(xudc->padctl);
+
+	err = regulator_bulk_enable(xudc->soc->num_supplies, xudc->supplies);
+	if (err) {
+		dev_err(xudc->dev, "failed to enable regulators %d\n", err);
+		goto put_padctl;
+	}
+
+	xudc->usb3_phy = devm_phy_optional_get(&pdev->dev, "usb3");
+	if (IS_ERR(xudc->usb3_phy)) {
+		err = PTR_ERR(xudc->usb3_phy);
+		dev_err(xudc->dev, "failed to get usb3 phy: %d\n", err);
+		goto disable_regulator;
+	}
+
+	xudc->utmi_phy = devm_phy_optional_get(&pdev->dev, "usb2");
+	if (IS_ERR(xudc->utmi_phy)) {
+		err = PTR_ERR(xudc->utmi_phy);
+		dev_err(xudc->dev, "failed to get usb2 phy: %d\n", err);
+		goto disable_regulator;
+	}
+
+	err = tegra_xudc_powerdomain_init(xudc);
+	if (err)
+		goto put_powerdomains;
+
+	err = tegra_xudc_phy_init(xudc);
+	if (err)
+		goto put_powerdomains;
+
+	err = tegra_xudc_alloc_event_ring(xudc);
+	if (err)
+		goto disable_phy;
+
+	err = tegra_xudc_alloc_eps(xudc);
+	if (err)
+		goto free_event_ring;
+
+	spin_lock_init(&xudc->lock);
+
+	init_completion(&xudc->disconnect_complete);
+
+	INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
+
+	INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
+
+	INIT_DELAYED_WORK(&xudc->port_reset_war_work,
+				tegra_xudc_port_reset_war_work);
+
+	if (of_property_read_bool(xudc->dev->of_node, "usb-role-switch")) {
+		role_sx_desc.set = tegra_xudc_usb_role_sw_set;
+		role_sx_desc.fwnode = dev_fwnode(xudc->dev);
+
+		xudc->usb_role_sw = usb_role_switch_register(xudc->dev,
+							&role_sx_desc);
+		if (IS_ERR(xudc->usb_role_sw)) {
+			err = PTR_ERR(xudc->usb_role_sw);
+			dev_err(xudc->dev, "Failed to register USB role SW: %d",
+					   err);
+			goto free_eps;
+		}
+	} else {
+		/* Set the mode as device mode and this keeps phy always ON */
+		dev_info(xudc->dev, "Set usb role to device mode always");
+		schedule_work(&xudc->usb_role_sw_work);
+	}
+
+	pm_runtime_enable(&pdev->dev);
+
+	xudc->gadget.ops = &tegra_xudc_gadget_ops;
+	xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
+	xudc->gadget.name = "tegra-xudc";
+	xudc->gadget.max_speed = USB_SPEED_SUPER;
+
+	err = usb_add_gadget_udc(&pdev->dev, &xudc->gadget);
+	if (err) {
+		dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
+		goto free_eps;
+	}
+
+	return 0;
+
+free_eps:
+	tegra_xudc_free_eps(xudc);
+free_event_ring:
+	tegra_xudc_free_event_ring(xudc);
+disable_phy:
+	tegra_xudc_phy_exit(xudc);
+put_powerdomains:
+	tegra_xudc_powerdomain_remove(xudc);
+disable_regulator:
+	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
+put_padctl:
+	tegra_xusb_padctl_put(xudc->padctl);
+
+	return err;
+}
+
+static int tegra_xudc_remove(struct platform_device *pdev)
+{
+	struct tegra_xudc *xudc = platform_get_drvdata(pdev);
+
+	pm_runtime_get_sync(xudc->dev);
+
+	cancel_delayed_work(&xudc->plc_reset_work);
+
+	if (xudc->usb_role_sw) {
+		usb_role_switch_unregister(xudc->usb_role_sw);
+		cancel_work_sync(&xudc->usb_role_sw_work);
+	}
+
+	usb_del_gadget_udc(&xudc->gadget);
+
+	tegra_xudc_free_eps(xudc);
+	tegra_xudc_free_event_ring(xudc);
+
+	tegra_xudc_powerdomain_remove(xudc);
+
+	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
+
+	phy_power_off(xudc->utmi_phy);
+	phy_power_off(xudc->usb3_phy);
+
+	tegra_xudc_phy_exit(xudc);
+
+	pm_runtime_disable(xudc->dev);
+	pm_runtime_put(xudc->dev);
+
+	tegra_xusb_padctl_put(xudc->padctl);
+
+	return 0;
+}
+
+static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
+{
+	unsigned long flags;
+
+	dev_dbg(xudc->dev, "entering ELPG\n");
+
+	spin_lock_irqsave(&xudc->lock, flags);
+
+	xudc->powergated = true;
+	xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
+	xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
+	xudc_writel(xudc, 0, CTRL);
+
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks);
+
+	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
+
+	dev_dbg(xudc->dev, "entering ELPG done\n");
+	return 0;
+}
+
+static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
+{
+	unsigned long flags;
+	int err;
+
+	dev_dbg(xudc->dev, "exiting ELPG\n");
+
+	err = regulator_bulk_enable(xudc->soc->num_supplies,
+			xudc->supplies);
+	if (err < 0)
+		return err;
+
+	err = clk_bulk_prepare_enable(xudc->soc->num_clks, xudc->clks);
+	if (err < 0)
+		return err;
+
+	tegra_xudc_fpci_ipfs_init(xudc);
+
+	tegra_xudc_device_params_init(xudc);
+
+	tegra_xudc_init_event_ring(xudc);
+
+	tegra_xudc_init_eps(xudc);
+
+	xudc_writel(xudc, xudc->saved_regs.portpm, PORTPM);
+	xudc_writel(xudc, xudc->saved_regs.ctrl, CTRL);
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	xudc->powergated = false;
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	dev_dbg(xudc->dev, "exiting ELPG done\n");
+	return 0;
+}
+
+static int __maybe_unused tegra_xudc_suspend(struct device *dev)
+{
+	struct tegra_xudc *xudc = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	xudc->suspended = true;
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	flush_work(&xudc->usb_role_sw_work);
+
+	/* Forcibly disconnect before powergating. */
+	tegra_xudc_device_mode_off(xudc);
+
+	if (!pm_runtime_status_suspended(dev))
+		tegra_xudc_powergate(xudc);
+
+	pm_runtime_disable(dev);
+
+	return 0;
+}
+
+static int __maybe_unused tegra_xudc_resume(struct device *dev)
+{
+	struct tegra_xudc *xudc = dev_get_drvdata(dev);
+	unsigned long flags;
+	int err;
+
+	err = tegra_xudc_unpowergate(xudc);
+	if (err < 0)
+		return err;
+
+	spin_lock_irqsave(&xudc->lock, flags);
+	xudc->suspended = false;
+	spin_unlock_irqrestore(&xudc->lock, flags);
+
+	schedule_work(&xudc->usb_role_sw_work);
+
+	pm_runtime_enable(dev);
+
+	return 0;
+}
+
+static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
+{
+	struct tegra_xudc *xudc = dev_get_drvdata(dev);
+
+	return tegra_xudc_powergate(xudc);
+}
+
+static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
+{
+	struct tegra_xudc *xudc = dev_get_drvdata(dev);
+
+	return tegra_xudc_unpowergate(xudc);
+}
+
+static const struct dev_pm_ops tegra_xudc_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
+	SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
+			   tegra_xudc_runtime_resume, NULL)
+};
+
+static struct platform_driver tegra_xudc_driver = {
+	.probe = tegra_xudc_probe,
+	.remove = tegra_xudc_remove,
+	.driver = {
+		.name = "tegra-xudc",
+		.pm = &tegra_xudc_pm_ops,
+		.of_match_table = tegra_xudc_of_match,
+	},
+};
+module_platform_driver(tegra_xudc_driver);
+
+MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
+MODULE_AUTHOR("Andrew Bresticker <[e-mail]>");
+MODULE_AUTHOR("Hui Fu <[e-mail]>");
+MODULE_AUTHOR("Nagarjuna Kristam <[e-mail]>");
+MODULE_LICENSE("GPL v2");