freebsd-src/sys/dev/dpaa2/dpaa2_io.c

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright © 2021-2022 Dmitry Salychev
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/cdefs.h>
/*
 * QBMan command interface and the DPAA2 I/O (DPIO) driver.
 *
 * The DPIO object allows configuration of the QBMan software portal with
 * optional notification capabilities.
 *
 * Software portals are used by the driver to communicate with the QBMan. The
 * DPIO object’s main purpose is to enable the driver to perform I/O – enqueue
 * and dequeue operations, as well as buffer release and acquire operations –
 * using QBMan.
 */

#include "opt_rss.h"

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/_cpuset.h>
#include <sys/cpuset.h>
#include <sys/taskqueue.h>
#include <sys/smp.h>

#include <vm/vm.h>

#include <machine/bus.h>
#include <machine/resource.h>

#include <dev/pci/pcivar.h>

#ifdef RSS
#include <net/rss_config.h>
#endif

#include "pcib_if.h"
#include "pci_if.h"

#include "dpaa2_mc.h"
#include "dpaa2_mcp.h"
#include "dpaa2_swp.h"
#include "dpaa2_swp_if.h"
#include "dpaa2_cmd_if.h"
#include "dpaa2_io.h"
#include "dpaa2_ni.h"
#include "dpaa2_channel.h"

#define DPIO_IRQ_INDEX		0 /* index of the only DPIO IRQ */
#define DPIO_POLL_MAX		32

/*
 * Memory:
 *	0: cache-enabled part of the QBMan software portal.
 *	1: cache-inhibited part of the QBMan software portal.
 *	2: control registers of the QBMan software portal?
 *
 * Note that MSI should be allocated separately using pseudo-PCI interface.
 */
struct resource_spec dpaa2_io_spec[] = {
	/*
	 * System Memory resources.
	 */
#define MEM_RES_NUM	(3u)
#define MEM_RID_OFF	(0u)
#define MEM_RID(rid)	((rid) + MEM_RID_OFF)
	{ SYS_RES_MEMORY, MEM_RID(0),   RF_ACTIVE | RF_UNMAPPED },
	{ SYS_RES_MEMORY, MEM_RID(1),   RF_ACTIVE | RF_UNMAPPED },
	{ SYS_RES_MEMORY, MEM_RID(2),   RF_ACTIVE | RF_UNMAPPED | RF_OPTIONAL },
	/*
	 * DPMCP resources.
	 *
	 * NOTE: MC command portals (MCPs) are used to send commands to, and
	 *	 receive responses from, the MC firmware. One portal per DPIO.
	 */
#define MCP_RES_NUM	(1u)
#define MCP_RID_OFF	(MEM_RID_OFF + MEM_RES_NUM)
#define MCP_RID(rid)	((rid) + MCP_RID_OFF)
	/* --- */
	{ DPAA2_DEV_MCP,  MCP_RID(0),   RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL },
	/* --- */
	RESOURCE_SPEC_END
};

/* Configuration routines. */
static int dpaa2_io_setup_irqs(device_t dev);
static int dpaa2_io_release_irqs(device_t dev);
static int dpaa2_io_setup_msi(struct dpaa2_io_softc *sc);
static int dpaa2_io_release_msi(struct dpaa2_io_softc *sc);

/* Interrupt handlers */
static void dpaa2_io_intr(void *arg);

static int
dpaa2_io_probe(device_t dev)
{
	/* DPIO device will be added by a parent resource container itself. */
	device_set_desc(dev, "DPAA2 I/O");
	return (BUS_PROBE_DEFAULT);
}

static int
dpaa2_io_detach(device_t dev)
{
	device_t pdev = device_get_parent(dev);
	device_t child = dev;
	struct dpaa2_io_softc *sc = device_get_softc(dev);
	struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev);
	struct dpaa2_devinfo *dinfo = device_get_ivars(dev);
	struct dpaa2_cmd cmd;
	uint16_t rc_token, io_token;
	int error;

	DPAA2_CMD_INIT(&cmd);

	/* Tear down interrupt handler and release IRQ resources. */
	dpaa2_io_release_irqs(dev);

	/* Free software portal helper object. */
	dpaa2_swp_free_portal(sc->swp);

	error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token);
	if (error) {
		device_printf(dev, "%s: failed to open DPRC: error=%d\n",
		    __func__, error);
		goto err_exit;
	}
	error = DPAA2_CMD_IO_OPEN(dev, child, &cmd, dinfo->id, &io_token);
	if (error) {
		device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n",
		    __func__, dinfo->id, error);
		goto close_rc;
	}

	error = DPAA2_CMD_IO_DISABLE(dev, child, &cmd);
	if (error && bootverbose) {
		device_printf(dev, "%s: failed to disable DPIO: id=%d, "
		    "error=%d\n", __func__, dinfo->id, error);
	}

	(void)DPAA2_CMD_IO_CLOSE(dev, child, &cmd);
	(void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token));

	/* Unmap memory resources of the portal. */
	for (int i = 0; i < MEM_RES_NUM; i++) {
		if (sc->res[MEM_RID(i)] == NULL) {
			continue;
		}
		error = bus_unmap_resource(sc->dev, SYS_RES_MEMORY,
		    sc->res[MEM_RID(i)], &sc->map[MEM_RID(i)]);
		if (error && bootverbose) {
			device_printf(dev, "%s: failed to unmap memory "
			    "resource: rid=%d, error=%d\n", __func__, MEM_RID(i),
			    error);
		}
	}

	/* Release allocated resources. */
	bus_release_resources(dev, dpaa2_io_spec, sc->res);

	return (0);

close_rc:
	(void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token));
err_exit:
	return (error);
}

static int
dpaa2_io_attach(device_t dev)
{
	device_t pdev = device_get_parent(dev);
	device_t child = dev;
	device_t mcp_dev;
	struct dpaa2_io_softc *sc = device_get_softc(dev);
	struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev);
	struct dpaa2_devinfo *dinfo = device_get_ivars(dev);
	struct dpaa2_devinfo *mcp_dinfo;
	struct dpaa2_cmd cmd;
	struct resource_map_request req;
	struct {
		vm_memattr_t memattr;
		char *label;
	} map_args[MEM_RES_NUM] = {
		{ VM_MEMATTR_WRITE_BACK, "cache-enabled part" },
		{ VM_MEMATTR_DEVICE, "cache-inhibited part" },
		{ VM_MEMATTR_DEVICE, "control registers" }
	};
	uint16_t rc_token, io_token;
	int error;

	sc->dev = dev;
	sc->swp = NULL;
	sc->intr = NULL;
	sc->irq_resource = NULL;

	/* Allocate resources. */
	error = bus_alloc_resources(sc->dev, dpaa2_io_spec, sc->res);
	if (error) {
		device_printf(dev, "%s: failed to allocate resources: "
		    "error=%d\n", __func__, error);
		return (ENXIO);
	}

	/* Set allocated MC portal up. */
	mcp_dev = (device_t) rman_get_start(sc->res[MCP_RID(0)]);
	mcp_dinfo = device_get_ivars(mcp_dev);
	dinfo->portal = mcp_dinfo->portal;

	/* Map memory resources of the portal. */
	for (int i = 0; i < MEM_RES_NUM; i++) {
		if (sc->res[MEM_RID(i)] == NULL) {
			continue;
		}

		resource_init_map_request(&req);
		req.memattr = map_args[i].memattr;
		error = bus_map_resource(sc->dev, SYS_RES_MEMORY,
		    sc->res[MEM_RID(i)], &req, &sc->map[MEM_RID(i)]);
		if (error) {
			device_printf(dev, "%s: failed to map %s: error=%d\n",
			    __func__, map_args[i].label, error);
			goto err_exit;
		}
	}

	DPAA2_CMD_INIT(&cmd);

	error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token);
	if (error) {
		device_printf(dev, "%s: failed to open DPRC: error=%d\n",
		    __func__, error);
		goto err_exit;
	}
	error = DPAA2_CMD_IO_OPEN(dev, child, &cmd, dinfo->id, &io_token);
	if (error) {
		device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n",
		    __func__, dinfo->id, error);
		goto close_rc;
	}
	error = DPAA2_CMD_IO_RESET(dev, child, &cmd);
	if (error) {
		device_printf(dev, "%s: failed to reset DPIO: id=%d, error=%d\n",
		    __func__, dinfo->id, error);
		goto close_io;
	}
	error = DPAA2_CMD_IO_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr);
	if (error) {
		device_printf(dev, "%s: failed to get DPIO attributes: id=%d, "
		    "error=%d\n", __func__, dinfo->id, error);
		goto close_io;
	}
	error = DPAA2_CMD_IO_ENABLE(dev, child, &cmd);
	if (error) {
		device_printf(dev, "%s: failed to enable DPIO: id=%d, "
		    "error=%d\n", __func__, dinfo->id, error);
		goto close_io;
	}

	/* Prepare descriptor of the QBMan software portal. */
	sc->swp_desc.dpio_dev = dev;
	sc->swp_desc.swp_version = sc->attr.swp_version;
	sc->swp_desc.swp_clk = sc->attr.swp_clk;
	sc->swp_desc.swp_id = sc->attr.swp_id;
	sc->swp_desc.has_notif = sc->attr.priors_num ? true : false;
	sc->swp_desc.has_8prio = sc->attr.priors_num == 8u ? true : false;

	sc->swp_desc.cena_res = sc->res[0];
	sc->swp_desc.cena_map = &sc->map[0];
	sc->swp_desc.cinh_res = sc->res[1];
	sc->swp_desc.cinh_map = &sc->map[1];

	/*
	 * Compute how many 256 QBMAN cycles fit into one ns. This is because
	 * the interrupt timeout period register needs to be specified in QBMAN
	 * clock cycles in increments of 256.
	 */
	sc->swp_desc.swp_cycles_ratio = 256000 /
	    (sc->swp_desc.swp_clk / 1000000);

	/* Initialize QBMan software portal. */
	error = dpaa2_swp_init_portal(&sc->swp, &sc->swp_desc, DPAA2_SWP_DEF);
	if (error) {
		device_printf(dev, "%s: failed to initialize dpaa2_swp: "
		    "error=%d\n", __func__, error);
		goto err_exit;
	}

	error = dpaa2_io_setup_irqs(dev);
	if (error) {
		device_printf(dev, "%s: failed to setup IRQs: error=%d\n",
		    __func__, error);
		goto err_exit;
	}

	if (bootverbose) {
		device_printf(dev, "dpio_id=%d, swp_id=%d, chan_mode=%s, "
		    "notif_priors=%d, swp_version=0x%x\n",
		    sc->attr.id, sc->attr.swp_id,
		    sc->attr.chan_mode == DPAA2_IO_LOCAL_CHANNEL
		    ? "local_channel" : "no_channel", sc->attr.priors_num,
		    sc->attr.swp_version);
	}

	(void)DPAA2_CMD_IO_CLOSE(dev, child, &cmd);
	(void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token));
	return (0);

close_io:
	(void)DPAA2_CMD_IO_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, io_token));
close_rc:
	(void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token));
err_exit:
	dpaa2_io_detach(dev);
	return (ENXIO);
}

/**
 * @brief Enqueue multiple frames to a frame queue using one FQID.
 */
static int
dpaa2_io_enq_multiple_fq(device_t iodev, uint32_t fqid,
    struct dpaa2_fd *fd, int frames_n)
{
	struct dpaa2_io_softc *sc = device_get_softc(iodev);
	struct dpaa2_swp *swp = sc->swp;
	struct dpaa2_eq_desc ed;
	uint32_t flags = 0;

	memset(&ed, 0, sizeof(ed));

	/* Setup enqueue descriptor. */
	dpaa2_swp_set_ed_norp(&ed, false);
	dpaa2_swp_set_ed_fq(&ed, fqid);

	return (dpaa2_swp_enq_mult(swp, &ed, fd, &flags, frames_n));
}

/**
 * @brief Configure the channel data availability notification (CDAN)
 * in a particular WQ channel paired with DPIO.
 */
static int
dpaa2_io_conf_wq_channel(device_t iodev, struct dpaa2_io_notif_ctx *ctx)
{
	struct dpaa2_io_softc *sc = device_get_softc(iodev);

	/* Enable generation of the CDAN notifications. */
	if (ctx->cdan_en) {
		return (dpaa2_swp_conf_wq_channel(sc->swp, ctx->fq_chan_id,
		    DPAA2_WQCHAN_WE_EN | DPAA2_WQCHAN_WE_CTX, ctx->cdan_en,
		    ctx->qman_ctx));
	}

	return (0);
}

/**
 * @brief Query current configuration/state of the buffer pool.
 */
static int
dpaa2_io_query_bp(device_t iodev, uint16_t bpid, struct dpaa2_bp_conf *conf)
{
	struct dpaa2_io_softc *sc = device_get_softc(iodev);

	return (dpaa2_swp_query_bp(sc->swp, bpid, conf));
}

/**
 * @brief Release one or more buffer pointers to the QBMan buffer pool.
 */
static int
dpaa2_io_release_bufs(device_t iodev, uint16_t bpid, bus_addr_t *buf,
    uint32_t buf_num)
{
	struct dpaa2_io_softc *sc = device_get_softc(iodev);

	return (dpaa2_swp_release_bufs(sc->swp, bpid, buf, buf_num));
}

/**
 * @brief Configure DPNI object to generate interrupts.
 */
static int
dpaa2_io_setup_irqs(device_t dev)
{
	struct dpaa2_io_softc *sc = device_get_softc(dev);
	int error;

	/*
	 * Setup interrupts generated by the software portal.
	 */
	dpaa2_swp_set_intr_trigger(sc->swp, DPAA2_SWP_INTR_DQRI);
	dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu);

	/* Configure IRQs. */
	error = dpaa2_io_setup_msi(sc);
	if (error) {
		device_printf(dev, "%s: failed to allocate MSI: error=%d\n",
		    __func__, error);
		return (error);
	}
	if ((sc->irq_resource = bus_alloc_resource_any(dev, SYS_RES_IRQ,
	    &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) {
		device_printf(dev, "%s: failed to allocate IRQ resource\n",
		    __func__);
		return (ENXIO);
	}
	if (bus_setup_intr(dev, sc->irq_resource, INTR_TYPE_NET | INTR_MPSAFE |
	    INTR_ENTROPY, NULL, dpaa2_io_intr, sc, &sc->intr)) {
		device_printf(dev, "%s: failed to setup IRQ resource\n",
		    __func__);
		return (ENXIO);
	}

	/* Wrap DPIO ID around number of CPUs/RSS buckets */
#ifdef RSS
	sc->cpu = rss_getcpu(sc->attr.id % rss_getnumbuckets());
#else
	sc->cpu = sc->attr.id % mp_ncpus;
#endif
	CPU_SETOF(sc->cpu, &sc->cpu_mask);
	bus_bind_intr(dev, sc->irq_resource, sc->cpu);

	/*
	 * Setup and enable Static Dequeue Command to receive CDANs from
	 * channel 0.
	 */
	if (sc->swp_desc.has_notif)
		dpaa2_swp_set_push_dequeue(sc->swp, 0, true);

	return (0);
}

static int
dpaa2_io_release_irqs(device_t dev)
{
	struct dpaa2_io_softc *sc = device_get_softc(dev);

	/* Disable receiving CDANs from channel 0. */
	if (sc->swp_desc.has_notif)
		dpaa2_swp_set_push_dequeue(sc->swp, 0, false);

	/* Release IRQ resources. */
	if (sc->intr != NULL)
		bus_teardown_intr(dev, sc->irq_resource, &sc->intr);
	if (sc->irq_resource != NULL)
		bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid[0],
		    sc->irq_resource);

	(void)dpaa2_io_release_msi(device_get_softc(dev));

	/* Configure software portal to stop generating interrupts. */
	dpaa2_swp_set_intr_trigger(sc->swp, 0);
	dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu);

	return (0);
}

/**
 * @brief Allocate MSI interrupts for this DPAA2 I/O object.
 */
static int
dpaa2_io_setup_msi(struct dpaa2_io_softc *sc)
{
	int val;

	val = pci_msi_count(sc->dev);
	if (val < DPAA2_IO_MSI_COUNT)
		device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val,
		    DPAA2_IO_MSI_COUNT);
	val = MIN(val, DPAA2_IO_MSI_COUNT);

	if (pci_alloc_msi(sc->dev, &val) != 0)
		return (EINVAL);

	for (int i = 0; i < val; i++)
		sc->irq_rid[i] = i + 1;

	return (0);
}

static int
dpaa2_io_release_msi(struct dpaa2_io_softc *sc)
{
	int error;

	error = pci_release_msi(sc->dev);
	if (error) {
		device_printf(sc->dev, "%s: failed to release MSI: error=%d/n",
		    __func__, error);
		return (error);
	}

	return (0);
}

/**
 * @brief DPAA2 I/O interrupt handler.
 */
static void
dpaa2_io_intr(void *arg)
{
	struct dpaa2_io_softc *sc = (struct dpaa2_io_softc *) arg;
	/* struct dpaa2_ni_softc *nisc = NULL; */
	struct dpaa2_io_notif_ctx *ctx[DPIO_POLL_MAX];
	struct dpaa2_channel *chan;
	struct dpaa2_dq dq;
	uint32_t idx, status;
	uint16_t flags;
	int rc, cdan_n = 0;

	status = dpaa2_swp_read_intr_status(sc->swp);
	if (status == 0) {
		return;
	}

	DPAA2_SWP_LOCK(sc->swp, &flags);
	if (flags & DPAA2_SWP_DESTROYED) {
		/* Terminate operation if portal is destroyed. */
		DPAA2_SWP_UNLOCK(sc->swp);
		return;
	}

	for (int i = 0; i < DPIO_POLL_MAX; i++) {
		rc = dpaa2_swp_dqrr_next_locked(sc->swp, &dq, &idx);
		if (rc) {
			break;
		}

		if ((dq.common.verb & DPAA2_DQRR_RESULT_MASK) ==
		    DPAA2_DQRR_RESULT_CDAN) {
			ctx[cdan_n++] = (struct dpaa2_io_notif_ctx *) dq.scn.ctx;
		} else {
			/* TODO: Report unknown DQRR entry. */
		}
		dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_DCAP, idx);
	}
	DPAA2_SWP_UNLOCK(sc->swp);

	for (int i = 0; i < cdan_n; i++) {
		chan = (struct dpaa2_channel *)ctx[i]->channel;
		/* nisc = device_get_softc(chan->ni_dev); */
		taskqueue_enqueue(chan->cleanup_tq, &chan->cleanup_task);
	}

	/* Enable software portal interrupts back */
	dpaa2_swp_clear_intr_status(sc->swp, status);
	dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_IIR, 0);
}

static device_method_t dpaa2_io_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe,		dpaa2_io_probe),
	DEVMETHOD(device_attach,	dpaa2_io_attach),
	DEVMETHOD(device_detach,	dpaa2_io_detach),

	/* QBMan software portal interface */
	DEVMETHOD(dpaa2_swp_enq_multiple_fq,	dpaa2_io_enq_multiple_fq),
	DEVMETHOD(dpaa2_swp_conf_wq_channel,	dpaa2_io_conf_wq_channel),
	DEVMETHOD(dpaa2_swp_query_bp,		dpaa2_io_query_bp),
	DEVMETHOD(dpaa2_swp_release_bufs,	dpaa2_io_release_bufs),

	DEVMETHOD_END
};

static driver_t dpaa2_io_driver = {
	"dpaa2_io",
	dpaa2_io_methods,
	sizeof(struct dpaa2_io_softc),
};

DRIVER_MODULE(dpaa2_io, dpaa2_rc, dpaa2_io_driver, 0, 0);