Implement a proper detach method for the PCI-PCI bridge driver.

- Add a pcib_detach() function for the PCI-PCI bridge driver.  It
  tears down the NEW_PCIB and hotplug state including destroying
  resource managers, deleting child devices, and disabling hotplug
  events.
- Add a detach method to the ACPI PCI-PCI bridge driver which calls
  pcib_detach() and then frees the copy of the _PRT interrupt routing
  table.
- Add a detach method to the PCI-Cardbus bridge driver which frees
  the PCI bus resources in addition to calling cbb_detach().
- Explicitly clear any pending hotplug events during attach to ensure
  future events will generate an interrupt.
- If a the Command Completed bit is set in the slot status register
  when the command completion timeout fires, treat it as if the
  command completed and the completion interrupt was just lost rather
  than forcing a detach.
- Don't wait for a Command Completed notification if Command Completion
  interrupts are disabled.  The spec explicitly says no interrupt is
  enabled when clearing CCIE, and on my T400 no interrupt is generated
  when CCIE is changed from cleared to set, either.  In addition, the
  T400 doesn't appear to set the Command Completed bit in the cases
  where it doesn't generate an interrupt, so don't schedule the timer
  either.  (If the CC bit were always set, one could always set the timer
  and rely on the logic of treating CC set as a missed interrupt.)

Reviewed by:	imp (older version)
Differential Revision:	https://reviews.freebsd.org/D6424

sys/dev/acpica/acpi_pcib_pci.c

@@ -66,6 +66,7 @@ struct acpi_pcib_lookup_info {
 
 static int		acpi_pcib_pci_probe(device_t bus);
 static int		acpi_pcib_pci_attach(device_t bus);
+static int		acpi_pcib_pci_detach(device_t bus);
 static int		acpi_pcib_read_ivar(device_t dev, device_t child,
 			    int which, uintptr_t *result);
 static int		acpi_pcib_pci_route_interrupt(device_t pcib,
@@ -75,6 +76,7 @@ static device_method_t acpi_pcib_pci_methods[] = {
     /* Device interface */
     DEVMETHOD(device_probe,		acpi_pcib_pci_probe),
     DEVMETHOD(device_attach,		acpi_pcib_pci_attach),
+    DEVMETHOD(device_detach,		acpi_pcib_pci_detach),
 
     /* Bus interface */
     DEVMETHOD(bus_read_ivar,		acpi_pcib_read_ivar),
@@ -126,6 +128,21 @@ acpi_pcib_pci_attach(device_t dev)
     return (pcib_attach_child(dev));
 }
 
+static int
+acpi_pcib_pci_detach(device_t dev)
+{
+    struct acpi_pcib_softc *sc;
+    int error;
+
+    ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
+
+    sc = device_get_softc(dev);
+    error = pcib_detach(dev);
+    if (error == 0)
+	    AcpiOsFree(sc->ap_prt.Pointer);
+    return (error);
+}
+
 static int
 acpi_pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
 {

sys/dev/pccbb/pccbb_pci.c

@@ -435,6 +435,22 @@ cbb_pci_attach(device_t brdev)
 	return (ENOMEM);
 }
 
+static int
+cbb_pci_detach(device_t brdev)
+{
+#if defined(NEW_PCIB) && defined(PCI_RES_BUS)
+	struct cbb_softc *sc = device_get_softc(brdev);
+#endif
+	int error;
+
+	error = cbb_detach(brdev);
+#if defined(NEW_PCIB) && defined(PCI_RES_BUS)
+	if (error == 0)
+		pcib_free_secbus(brdev, &sc->bus);
+#endif
+	return (error);
+}
+
 static void
 cbb_chipinit(struct cbb_softc *sc)
 {
@@ -917,7 +933,7 @@ static device_method_t cbb_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,			cbb_pci_probe),
 	DEVMETHOD(device_attach,		cbb_pci_attach),
-	DEVMETHOD(device_detach,		cbb_detach),
+	DEVMETHOD(device_detach,		cbb_pci_detach),
 	DEVMETHOD(device_shutdown,		cbb_pci_shutdown),
 	DEVMETHOD(device_suspend,		cbb_pci_suspend),
 	DEVMETHOD(device_resume,		cbb_pci_resume),

sys/dev/pci/pci_pci.c

@@ -81,7 +81,7 @@ static device_method_t pcib_methods[] = {
     /* Device interface */
     DEVMETHOD(device_probe,		pcib_probe),
     DEVMETHOD(device_attach,		pcib_attach),
-    DEVMETHOD(device_detach,		bus_generic_detach),
+    DEVMETHOD(device_detach,		pcib_detach),
     DEVMETHOD(device_shutdown,		bus_generic_shutdown),
     DEVMETHOD(device_suspend,		pcib_suspend),
     DEVMETHOD(device_resume,		pcib_resume),
@@ -544,6 +544,42 @@ pcib_probe_windows(struct pcib_softc *sc)
 	}
 }
 
+static void
+pcib_release_window(struct pcib_softc *sc, struct pcib_window *w, int type)
+{
+	device_t dev;
+	int error, i;
+
+	if (!w->valid)
+		return;
+
+	dev = sc->dev;
+	error = rman_fini(&w->rman);
+	if (error) {
+		device_printf(dev, "failed to release %s rman\n", w->name);
+		return;
+	}
+	free(__DECONST(char *, w->rman.rm_descr), M_DEVBUF);
+
+	for (i = 0; i < w->count; i++) {
+		error = bus_free_resource(dev, type, w->res[i]);
+		if (error)
+			device_printf(dev,
+			    "failed to release %s resource: %d\n", w->name,
+			    error);
+	}
+	free(w->res, M_DEVBUF);
+}
+
+static void
+pcib_free_windows(struct pcib_softc *sc)
+{
+
+	pcib_release_window(sc, &sc->pmem, SYS_RES_MEMORY);
+	pcib_release_window(sc, &sc->mem, SYS_RES_MEMORY);
+	pcib_release_window(sc, &sc->io, SYS_RES_IOPORT);
+}
+
 #ifdef PCI_RES_BUS
 /*
  * Allocate a suitable secondary bus for this bridge if needed and
@@ -618,6 +654,24 @@ pcib_setup_secbus(device_t dev, struct pcib_secbus *bus, int min_count)
 	}
 }
 
+void
+pcib_free_secbus(device_t dev, struct pcib_secbus *bus)
+{
+	int error;
+
+	error = rman_fini(&bus->rman);
+	if (error) {
+		device_printf(dev, "failed to release bus number rman\n");
+		return;
+	}
+	free(__DECONST(char *, bus->rman.rm_descr), M_DEVBUF);
+
+	error = bus_free_resource(dev, PCI_RES_BUS, bus->res);
+	if (error)
+		device_printf(dev,
+		    "failed to release bus numbers resource: %d\n", error);
+}
+
 static struct resource *
 pcib_suballoc_bus(struct pcib_secbus *bus, device_t child, int *rid,
     rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
@@ -896,7 +950,8 @@ pcib_pcie_hotplug_command(struct pcib_softc *sc, uint16_t val, uint16_t mask)
 	if (new == ctl)
 		return;
 	pcie_write_config(dev, PCIER_SLOT_CTL, new, 2);
-	if (!(sc->pcie_slot_cap & PCIEM_SLOT_CAP_NCCS)) {
+	if (!(sc->pcie_slot_cap & PCIEM_SLOT_CAP_NCCS) &&
+	    (ctl & new) & PCIEM_SLOT_CTL_CCIE) {
 		sc->flags |= PCIB_HOTPLUG_CMD_PENDING;
 		if (!cold)
 			callout_reset(&sc->pcie_cc_timer, hz,
@@ -917,6 +972,7 @@ pcib_pcie_hotplug_command_completed(struct pcib_softc *sc)
 		return;
 	callout_stop(&sc->pcie_cc_timer);
 	sc->flags &= ~PCIB_HOTPLUG_CMD_PENDING;
+	wakeup(sc);
 }
 
 /*
@@ -1153,16 +1209,22 @@ pcib_pcie_cc_timeout(void *arg)
 {
 	struct pcib_softc *sc;
 	device_t dev;
+	uint16_t sta;
 
 	sc = arg;
 	dev = sc->dev;
 	mtx_assert(&Giant, MA_OWNED);
-	if (sc->flags & PCIB_HOTPLUG_CMD_PENDING) {
+	sta = pcie_read_config(dev, PCIER_SLOT_STA, 2);
+	if (!(sta & PCIEM_SLOT_STA_CC)) {
 		device_printf(dev,
 		    "Hotplug Command Timed Out - forcing detach\n");
 		sc->flags &= ~(PCIB_HOTPLUG_CMD_PENDING | PCIB_DETACH_PENDING);
 		sc->flags |= PCIB_DETACHING;
 		pcib_pcie_hotplug_update(sc, 0, 0, true);
+	} else {
+		device_printf(dev,
+	    "Missed HotPlug interrupt waiting for Command Completion\n");
+		pcib_pcie_intr(sc);
 	}
 }
 
@@ -1242,6 +1304,22 @@ pcib_alloc_pcie_irq(struct pcib_softc *sc)
 	return (0);
 }
 
+static int
+pcib_release_pcie_irq(struct pcib_softc *sc)
+{
+	device_t dev;
+	int error;
+
+	dev = sc->dev;
+	error = bus_teardown_intr(dev, sc->pcie_irq, sc->pcie_ihand);
+	if (error)
+		return (error);
+	error = bus_free_resource(dev, SYS_RES_IRQ, sc->pcie_irq);
+	if (error)
+		return (error);
+	return (pci_release_msi(dev));
+}
+
 static void
 pcib_setup_hotplug(struct pcib_softc *sc)
 {
@@ -1261,6 +1339,9 @@ pcib_setup_hotplug(struct pcib_softc *sc)
 	sc->pcie_link_sta = pcie_read_config(dev, PCIER_LINK_STA, 2);
 	sc->pcie_slot_sta = pcie_read_config(dev, PCIER_SLOT_STA, 2);
 
+	/* Clear any events previously pending. */
+	pcie_write_config(dev, PCIER_SLOT_STA, sc->pcie_slot_sta, 2);
+
 	/* Enable HotPlug events. */
 	mask = PCIEM_SLOT_CTL_DLLSCE | PCIEM_SLOT_CTL_HPIE |
 	    PCIEM_SLOT_CTL_CCIE | PCIEM_SLOT_CTL_PDCE | PCIEM_SLOT_CTL_MRLSCE |
@@ -1285,6 +1366,49 @@ pcib_setup_hotplug(struct pcib_softc *sc)
 
 	pcib_pcie_hotplug_update(sc, val, mask, false);
 }
+
+static int
+pcib_detach_hotplug(struct pcib_softc *sc)
+{
+	uint16_t mask, val;
+	int error;
+
+	/* Disable the card in the slot and force it to detach. */
+	if (sc->flags & PCIB_DETACH_PENDING) {
+		sc->flags &= ~PCIB_DETACH_PENDING;
+		callout_stop(&sc->pcie_ab_timer);
+	}
+	sc->flags |= PCIB_DETACHING;
+
+	if (sc->flags & PCIB_HOTPLUG_CMD_PENDING) {
+		callout_stop(&sc->pcie_cc_timer);
+		tsleep(sc, 0, "hpcmd", hz);
+		sc->flags &= ~PCIB_HOTPLUG_CMD_PENDING;
+	}
+
+	/* Disable HotPlug events. */
+	mask = PCIEM_SLOT_CTL_DLLSCE | PCIEM_SLOT_CTL_HPIE |
+	    PCIEM_SLOT_CTL_CCIE | PCIEM_SLOT_CTL_PDCE | PCIEM_SLOT_CTL_MRLSCE |
+	    PCIEM_SLOT_CTL_PFDE | PCIEM_SLOT_CTL_ABPE;
+	val = 0;
+
+	/* Turn the attention indicator off. */
+	if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_AIP) {
+		mask |= PCIEM_SLOT_CTL_AIC;
+		val |= PCIEM_SLOT_CTL_AI_OFF;
+	}
+
+	pcib_pcie_hotplug_update(sc, val, mask, false);
+	
+	error = pcib_release_pcie_irq(sc);
+	if (error)
+		return (error);
+	taskqueue_drain(taskqueue_thread, &sc->pcie_hp_task);
+	callout_drain(&sc->pcie_ab_timer);
+	callout_drain(&sc->pcie_cc_timer);
+	callout_drain(&sc->pcie_dll_timer);
+	return (0);
+}
 #endif
 
 /*
@@ -1570,6 +1694,39 @@ pcib_attach(device_t dev)
     return (pcib_attach_child(dev));
 }
 
+int
+pcib_detach(device_t dev)
+{
+#if defined(PCI_HP) || defined(NEW_PCIB)
+	struct pcib_softc *sc;
+#endif
+	int error;
+
+#if defined(PCI_HP) || defined(NEW_PCIB)
+	sc = device_get_softc(dev);
+#endif
+	error = bus_generic_detach(dev);
+	if (error)
+		return (error);
+#ifdef PCI_HP
+	if (sc->flags & PCIB_HOTPLUG) {
+		error = pcib_detach_hotplug(sc);
+		if (error)
+			return (error);
+	}
+#endif
+	error = device_delete_children(dev);
+	if (error)
+		return (error);
+#ifdef NEW_PCIB
+	pcib_free_windows(sc);
+#ifdef PCI_RES_BUS
+	pcib_free_secbus(dev, &sc->bus);
+#endif
+#endif
+	return (0);
+}
+
 int
 pcib_suspend(device_t dev)
 {

sys/dev/pci/pcib_private.h

@@ -158,6 +158,7 @@ int		pci_domain_release_bus(int domain, device_t dev, int rid,
 struct resource *pcib_alloc_subbus(struct pcib_secbus *bus, device_t child,
 		    int *rid, rman_res_t start, rman_res_t end, rman_res_t count,
 		    u_int flags);
+void		pcib_free_secbus(device_t dev, struct pcib_secbus *bus);
 void		pcib_setup_secbus(device_t dev, struct pcib_secbus *bus,
     int min_count);
 #endif
@@ -169,6 +170,7 @@ void		pcib_bridge_init(device_t dev);
 const char	*pcib_child_name(device_t child);
 #endif
 int		pcib_child_present(device_t dev, device_t child);
+int		pcib_detach(device_t dev);
 int		pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result);
 int		pcib_write_ivar(device_t dev, device_t child, int which, uintptr_t value);
 struct resource *pcib_alloc_resource(device_t dev, device_t child, int type, int *rid,