mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
dmaengine updates for v5.11-rc1
New drivers/devices - Qualcomm ADM driver - Qualcomm GPI driver - Allwinner A100 DMA support - Microchip Sama7g5 support - Mediatek MT8516 apdma - Updates: - more updates to idxd driver and support for IAX config - runtime PM support for dw driver - TI keystone drivers for 5.11 included here due to dependency for TI drivers -----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEE+vs47OPLdNbVcHzyfBQHDyUjg0cFAl/bkxEACgkQfBQHDyUj g0dxNA//WzpVy9QJnj6OgXIjM+9sBjqls0iPVfy1JeeMmVW8cgCwLyBNZcssKRye mJ9+VnTx4JQBj4KD2cFBdpr46GvBFbSbcWNSCdm179NtHI4G6tjtynOcWaI9Clu2 0KHoa/EHIj8/jD3Hsbm+WZ1zCoY4VKBXsEq6x1Sj2tpp0/ocDhH4XLAsWTHE9OAD sc+0OtHr1wU4EdV6TKNTT0jXsdtzxOPPsvRsoaKncnR+Mkrgv0FvMBfBLhOb3a+m wUHEkwrEP1pT4Xcew6ZkYs4RYwJI3pllu2gUTs5qtidc723ol/C4kJ27q55N4azb j5buA8AhEwqIDH8qNuV07qaIu2VTdTdbYid3xgAeFygwM1npecZvOf8k6rTjxR/6 XN8jaDuhc2uISY7Gt5c6tOe8rG3ffNhYrmEuGD5HI0hcglpALiE4NcgalaaQS9J1 suQ6AUtCslReD+6M/lfarn9Zd3UAKGbxU8vCNPq0EcSAGUz9u9VK2VwKiGnAZ8bb ED3QDUzZYjTDWpiVodsuJlONgaMLsRCQecZWMDRNpzmf1rCXnkY0eDGiSMz+IiXZ 87IdD66u3d/Mkm6jVdwp6+tKZ/ohj+dtIWKhMd8cKXv5zTzS+4IokxpkxdjBsHPx z+G73IMHjQo8xl/P0IhhwZw+7cBrkntLq8lRSbYxjSTP09QerNE= =uNBn -----END PGP SIGNATURE----- Merge tag 'dmaengine-5.11-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine Pull dmaengine updates from Vinod Koul: "The last dmaengine updates for this year :) This contains couple of new drivers, new device support and updates to bunch of drivers. New drivers/devices: - Qualcomm ADM driver - Qualcomm GPI driver - Allwinner A100 DMA support - Microchip Sama7g5 support - Mediatek MT8516 apdma Updates: - more updates to idxd driver and support for IAX config - runtime PM support for dw driver - TI drivers" * tag 'dmaengine-5.11-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (75 commits) soc: ti: k3-ringacc: Use correct error casting in k3_ringacc_dmarings_init dmaengine: ti: k3-udma-glue: Add support for K3 PKTDMA dmaengine: ti: k3-udma: Initial support for K3 PKTDMA dmaengine: ti: k3-udma: Add support for BCDMA channel TPL handling dmaengine: ti: k3-udma: Initial support for K3 BCDMA soc: ti: k3-ringacc: add AM64 DMA rings support. dmaengine: ti: Add support for k3 event routers dmaengine: ti: k3-psil: Add initial map for AM64 dmaengine: ti: k3-psil: Extend psil_endpoint_config for K3 PKTDMA dt-bindings: dma: ti: Add document for K3 PKTDMA dt-bindings: dma: ti: Add document for K3 BCDMA dmaengine: dmatest: Use dmaengine_get_dma_device dmaengine: doc: client: Update for dmaengine_get_dma_device() usage dmaengine: Add support for per channel coherency handling dmaengine: of-dma: Add support for optional router configuration callback dmaengine: ti: k3-udma-glue: Configure the dma_dev for rings dmaengine: ti: k3-udma-glue: Get the ringacc from udma_dev dmaengine: ti: k3-udma-glue: Add function to get device pointer for DMA API dmaengine: ti: k3-udma: Add support for second resource range from sysfw dmaengine: ti: k3-udma: Wait for peer teardown completion if supported ...
This commit is contained in:
commit
6daa90439e
|
@ -77,6 +77,13 @@ Contact: dmaengine@vger.kernel.org
|
|||
Description: The operation capability bit mask specify the operation types
|
||||
supported by the this device.
|
||||
|
||||
What: /sys/bus/dsa/devices/dsa<m>/pasid_enabled
|
||||
Date: Oct 27, 2020
|
||||
KernelVersion: 5.11.0
|
||||
Contact: dmaengine@vger.kernel.org
|
||||
Description: To indicate if PASID (process address space identifier) is
|
||||
enabled or not for this device.
|
||||
|
||||
What: /sys/bus/dsa/devices/dsa<m>/state
|
||||
Date: Oct 25, 2019
|
||||
KernelVersion: 5.6.0
|
||||
|
@ -122,6 +129,13 @@ KernelVersion: 5.10.0
|
|||
Contact: dmaengine@vger.kernel.org
|
||||
Description: The last executed device administrative command's status/error.
|
||||
|
||||
What: /sys/bus/dsa/devices/wq<m>.<n>/block_on_fault
|
||||
Date: Oct 27, 2020
|
||||
KernelVersion: 5.11.0
|
||||
Contact: dmaengine@vger.kernel.org
|
||||
Description: To indicate block on fault is allowed or not for the work queue
|
||||
to support on demand paging.
|
||||
|
||||
What: /sys/bus/dsa/devices/wq<m>.<n>/group_id
|
||||
Date: Oct 25, 2019
|
||||
KernelVersion: 5.6.0
|
||||
|
@ -190,6 +204,13 @@ Contact: dmaengine@vger.kernel.org
|
|||
Description: The max batch size for this workqueue. Cannot exceed device
|
||||
max batch size. Configurable parameter.
|
||||
|
||||
What: /sys/bus/dsa/devices/wq<m>.<n>/ats_disable
|
||||
Date: Nov 13, 2020
|
||||
KernelVersion: 5.11.0
|
||||
Contact: dmaengine@vger.kernel.org
|
||||
Description: Indicate whether ATS disable is turned on for the workqueue.
|
||||
0 indicates ATS is on, and 1 indicates ATS is off for the workqueue.
|
||||
|
||||
What: /sys/bus/dsa/devices/engine<m>.<n>/group_id
|
||||
Date: Oct 25, 2019
|
||||
KernelVersion: 5.6.0
|
||||
|
|
|
@ -21,6 +21,7 @@ properties:
|
|||
compatible:
|
||||
oneOf:
|
||||
- const: allwinner,sun50i-a64-dma
|
||||
- const: allwinner,sun50i-a100-dma
|
||||
- const: allwinner,sun50i-h6-dma
|
||||
- items:
|
||||
- const: allwinner,sun8i-r40-dma
|
||||
|
@ -56,7 +57,9 @@ required:
|
|||
if:
|
||||
properties:
|
||||
compatible:
|
||||
const: allwinner,sun50i-h6-dma
|
||||
enum:
|
||||
- allwinner,sun50i-a100-dma
|
||||
- allwinner,sun50i-h6-dma
|
||||
|
||||
then:
|
||||
properties:
|
||||
|
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
* XDMA Controller
|
||||
Required properties:
|
||||
- compatible: Should be "atmel,sama5d4-dma" or "microchip,sam9x60-dma".
|
||||
- compatible: Should be "atmel,sama5d4-dma", "microchip,sam9x60-dma" or
|
||||
"microchip,sama7g5-dma".
|
||||
- reg: Should contain DMA registers location and length.
|
||||
- interrupts: Should contain DMA interrupt.
|
||||
- #dma-cells: Must be <1>, used to represent the number of integer cells in
|
||||
|
|
|
@ -4,6 +4,7 @@ Required properties:
|
|||
- compatible should contain:
|
||||
* "mediatek,mt2712-uart-dma" for MT2712 compatible APDMA
|
||||
* "mediatek,mt6577-uart-dma" for MT6577 and all of the above
|
||||
* "mediatek,mt8516-uart-dma", "mediatek,mt6577" for MT8516 SoC
|
||||
|
||||
- reg: The base address of the APDMA register bank.
|
||||
|
||||
|
|
88
Documentation/devicetree/bindings/dma/qcom,gpi.yaml
Normal file
88
Documentation/devicetree/bindings/dma/qcom,gpi.yaml
Normal file
|
@ -0,0 +1,88 @@
|
|||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/dma/qcom,gpi.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: Qualcomm Technologies Inc GPI DMA controller
|
||||
|
||||
maintainers:
|
||||
- Vinod Koul <vkoul@kernel.org>
|
||||
|
||||
description: |
|
||||
QCOM GPI DMA controller provides DMA capabilities for
|
||||
peripheral buses such as I2C, UART, and SPI.
|
||||
|
||||
allOf:
|
||||
- $ref: "dma-controller.yaml#"
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
enum:
|
||||
- qcom,sdm845-gpi-dma
|
||||
|
||||
reg:
|
||||
maxItems: 1
|
||||
|
||||
interrupts:
|
||||
description:
|
||||
Interrupt lines for each GPI instance
|
||||
maxItems: 13
|
||||
|
||||
"#dma-cells":
|
||||
const: 3
|
||||
description: >
|
||||
DMA clients must use the format described in dma.txt, giving a phandle
|
||||
to the DMA controller plus the following 3 integer cells:
|
||||
- channel: if set to 0xffffffff, any available channel will be allocated
|
||||
for the client. Otherwise, the exact channel specified will be used.
|
||||
- seid: serial id of the client as defined in the SoC documentation.
|
||||
- client: type of the client as defined in dt-bindings/dma/qcom-gpi.h
|
||||
|
||||
iommus:
|
||||
maxItems: 1
|
||||
|
||||
dma-channels:
|
||||
maximum: 31
|
||||
|
||||
dma-channel-mask:
|
||||
maxItems: 1
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
- interrupts
|
||||
- "#dma-cells"
|
||||
- iommus
|
||||
- dma-channels
|
||||
- dma-channel-mask
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
examples:
|
||||
- |
|
||||
#include <dt-bindings/interrupt-controller/arm-gic.h>
|
||||
#include <dt-bindings/dma/qcom-gpi.h>
|
||||
gpi_dma0: dma-controller@800000 {
|
||||
compatible = "qcom,gpi-dma";
|
||||
#dma-cells = <3>;
|
||||
reg = <0x00800000 0x60000>;
|
||||
iommus = <&apps_smmu 0x0016 0x0>;
|
||||
dma-channels = <13>;
|
||||
dma-channel-mask = <0xfa>;
|
||||
interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 251 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 253 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>,
|
||||
<GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>;
|
||||
};
|
||||
|
||||
...
|
164
Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml
Normal file
164
Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml
Normal file
|
@ -0,0 +1,164 @@
|
|||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/dma/ti/k3-bcdma.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: Texas Instruments K3 DMSS BCDMA Device Tree Bindings
|
||||
|
||||
maintainers:
|
||||
- Peter Ujfalusi <peter.ujfalusi@ti.com>
|
||||
|
||||
description: |
|
||||
The Block Copy DMA (BCDMA) is intended to perform similar functions as the TR
|
||||
mode channels of K3 UDMA-P.
|
||||
BCDMA includes block copy channels and Split channels.
|
||||
|
||||
Block copy channels mainly used for memory to memory transfers, but with
|
||||
optional triggers a block copy channel can service peripherals by accessing
|
||||
directly to memory mapped registers or area.
|
||||
|
||||
Split channels can be used to service PSI-L based peripherals.
|
||||
The peripherals can be PSI-L native or legacy, non PSI-L native peripherals
|
||||
with PDMAs. PDMA is tasked to act as a bridge between the PSI-L fabric and the
|
||||
legacy peripheral.
|
||||
|
||||
PDMAs can be configured via BCDMA split channel's peer registers to match with
|
||||
the configuration of the legacy peripheral.
|
||||
|
||||
allOf:
|
||||
- $ref: /schemas/dma/dma-controller.yaml#
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
const: ti,am64-dmss-bcdma
|
||||
|
||||
"#dma-cells":
|
||||
const: 3
|
||||
description: |
|
||||
cell 1: type of the BCDMA channel to be used to service the peripheral:
|
||||
0 - split channel
|
||||
1 - block copy channel using global trigger 1
|
||||
2 - block copy channel using global trigger 2
|
||||
3 - block copy channel using local trigger
|
||||
|
||||
cell 2: parameter for the channel:
|
||||
if cell 1 is 0 (split channel):
|
||||
PSI-L thread ID of the remote (to BCDMA) end.
|
||||
Valid ranges for thread ID depends on the data movement direction:
|
||||
for source thread IDs (rx): 0 - 0x7fff
|
||||
for destination thread IDs (tx): 0x8000 - 0xffff
|
||||
|
||||
Please refer to the device documentation for the PSI-L thread map and
|
||||
also the PSI-L peripheral chapter for the correct thread ID.
|
||||
if cell 1 is 1 or 2 (block copy channel using global trigger):
|
||||
Unused, ignored
|
||||
|
||||
The trigger must be configured for the channel externally to BCDMA,
|
||||
channels using global triggers should not be requested directly, but
|
||||
via DMA event router.
|
||||
if cell 1 is 3 (block copy channel using local trigger):
|
||||
bchan number of the locally triggered channel
|
||||
|
||||
cell 3: ASEL value for the channel
|
||||
|
||||
reg:
|
||||
maxItems: 5
|
||||
|
||||
reg-names:
|
||||
items:
|
||||
- const: gcfg
|
||||
- const: bchanrt
|
||||
- const: rchanrt
|
||||
- const: tchanrt
|
||||
- const: ringrt
|
||||
|
||||
msi-parent: true
|
||||
|
||||
ti,asel:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32
|
||||
description: ASEL value for non slave channels
|
||||
|
||||
ti,sci-rm-range-bchan:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of BCDMA block-copy channel resource subtypes for resource
|
||||
allocation for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
ti,sci-rm-range-tchan:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of BCDMA split tx channel resource subtypes for resource allocation
|
||||
for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
ti,sci-rm-range-rchan:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of BCDMA split rx channel resource subtypes for resource allocation
|
||||
for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- "#dma-cells"
|
||||
- reg
|
||||
- reg-names
|
||||
- msi-parent
|
||||
- ti,sci
|
||||
- ti,sci-dev-id
|
||||
- ti,sci-rm-range-bchan
|
||||
- ti,sci-rm-range-tchan
|
||||
- ti,sci-rm-range-rchan
|
||||
|
||||
unevaluatedProperties: false
|
||||
|
||||
examples:
|
||||
- |+
|
||||
cbass_main {
|
||||
#address-cells = <2>;
|
||||
#size-cells = <2>;
|
||||
|
||||
main_dmss {
|
||||
compatible = "simple-mfd";
|
||||
#address-cells = <2>;
|
||||
#size-cells = <2>;
|
||||
dma-ranges;
|
||||
ranges;
|
||||
|
||||
ti,sci-dev-id = <25>;
|
||||
|
||||
main_bcdma: dma-controller@485c0100 {
|
||||
compatible = "ti,am64-dmss-bcdma";
|
||||
|
||||
reg = <0x0 0x485c0100 0x0 0x100>,
|
||||
<0x0 0x4c000000 0x0 0x20000>,
|
||||
<0x0 0x4a820000 0x0 0x20000>,
|
||||
<0x0 0x4aa40000 0x0 0x20000>,
|
||||
<0x0 0x4bc00000 0x0 0x100000>;
|
||||
reg-names = "gcfg", "bchanrt", "rchanrt", "tchanrt", "ringrt";
|
||||
msi-parent = <&inta_main_dmss>;
|
||||
#dma-cells = <3>;
|
||||
|
||||
ti,sci = <&dmsc>;
|
||||
ti,sci-dev-id = <26>;
|
||||
|
||||
ti,sci-rm-range-bchan = <0x20>; /* BLOCK_COPY_CHAN */
|
||||
ti,sci-rm-range-rchan = <0x21>; /* SPLIT_TR_RX_CHAN */
|
||||
ti,sci-rm-range-tchan = <0x22>; /* SPLIT_TR_TX_CHAN */
|
||||
};
|
||||
};
|
||||
};
|
172
Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml
Normal file
172
Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml
Normal file
|
@ -0,0 +1,172 @@
|
|||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/dma/ti/k3-pktdma.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: Texas Instruments K3 DMSS PKTDMA Device Tree Bindings
|
||||
|
||||
maintainers:
|
||||
- Peter Ujfalusi <peter.ujfalusi@ti.com>
|
||||
|
||||
description: |
|
||||
The Packet DMA (PKTDMA) is intended to perform similar functions as the packet
|
||||
mode channels of K3 UDMA-P.
|
||||
PKTDMA only includes Split channels to service PSI-L based peripherals.
|
||||
|
||||
The peripherals can be PSI-L native or legacy, non PSI-L native peripherals
|
||||
with PDMAs. PDMA is tasked to act as a bridge between the PSI-L fabric and the
|
||||
legacy peripheral.
|
||||
|
||||
PDMAs can be configured via PKTDMA split channel's peer registers to match
|
||||
with the configuration of the legacy peripheral.
|
||||
|
||||
allOf:
|
||||
- $ref: /schemas/dma/dma-controller.yaml#
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
const: ti,am64-dmss-pktdma
|
||||
|
||||
"#dma-cells":
|
||||
const: 2
|
||||
description: |
|
||||
The first cell is the PSI-L thread ID of the remote (to PKTDMA) end.
|
||||
Valid ranges for thread ID depends on the data movement direction:
|
||||
for source thread IDs (rx): 0 - 0x7fff
|
||||
for destination thread IDs (tx): 0x8000 - 0xffff
|
||||
|
||||
Please refer to the device documentation for the PSI-L thread map and also
|
||||
the PSI-L peripheral chapter for the correct thread ID.
|
||||
|
||||
The second cell is the ASEL value for the channel
|
||||
|
||||
reg:
|
||||
maxItems: 4
|
||||
|
||||
reg-names:
|
||||
items:
|
||||
- const: gcfg
|
||||
- const: rchanrt
|
||||
- const: tchanrt
|
||||
- const: ringrt
|
||||
|
||||
msi-parent: true
|
||||
|
||||
ti,sci-rm-range-tchan:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of PKTDMA split tx channel resource subtypes for resource allocation
|
||||
for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
ti,sci-rm-range-tflow:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of PKTDMA split tx flow resource subtypes for resource allocation
|
||||
for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
ti,sci-rm-range-rchan:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of PKTDMA split rx channel resource subtypes for resource allocation
|
||||
for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
ti,sci-rm-range-rflow:
|
||||
$ref: /schemas/types.yaml#/definitions/uint32-array
|
||||
description: |
|
||||
Array of PKTDMA split rx flow resource subtypes for resource allocation
|
||||
for this host
|
||||
minItems: 1
|
||||
# Should be enough
|
||||
maxItems: 255
|
||||
items:
|
||||
maximum: 0x3f
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- "#dma-cells"
|
||||
- reg
|
||||
- reg-names
|
||||
- msi-parent
|
||||
- ti,sci
|
||||
- ti,sci-dev-id
|
||||
- ti,sci-rm-range-tchan
|
||||
- ti,sci-rm-range-tflow
|
||||
- ti,sci-rm-range-rchan
|
||||
- ti,sci-rm-range-rflow
|
||||
|
||||
unevaluatedProperties: false
|
||||
|
||||
examples:
|
||||
- |+
|
||||
cbass_main {
|
||||
#address-cells = <2>;
|
||||
#size-cells = <2>;
|
||||
|
||||
main_dmss {
|
||||
compatible = "simple-mfd";
|
||||
#address-cells = <2>;
|
||||
#size-cells = <2>;
|
||||
dma-ranges;
|
||||
ranges;
|
||||
|
||||
ti,sci-dev-id = <25>;
|
||||
|
||||
main_pktdma: dma-controller@485c0000 {
|
||||
compatible = "ti,am64-dmss-pktdma";
|
||||
|
||||
reg = <0x0 0x485c0000 0x0 0x100>,
|
||||
<0x0 0x4a800000 0x0 0x20000>,
|
||||
<0x0 0x4aa00000 0x0 0x40000>,
|
||||
<0x0 0x4b800000 0x0 0x400000>;
|
||||
reg-names = "gcfg", "rchanrt", "tchanrt", "ringrt";
|
||||
msi-parent = <&inta_main_dmss>;
|
||||
#dma-cells = <2>;
|
||||
|
||||
ti,sci = <&dmsc>;
|
||||
ti,sci-dev-id = <30>;
|
||||
|
||||
ti,sci-rm-range-tchan = <0x23>, /* UNMAPPED_TX_CHAN */
|
||||
<0x24>, /* CPSW_TX_CHAN */
|
||||
<0x25>, /* SAUL_TX_0_CHAN */
|
||||
<0x26>, /* SAUL_TX_1_CHAN */
|
||||
<0x27>, /* ICSSG_0_TX_CHAN */
|
||||
<0x28>; /* ICSSG_1_TX_CHAN */
|
||||
ti,sci-rm-range-tflow = <0x10>, /* RING_UNMAPPED_TX_CHAN */
|
||||
<0x11>, /* RING_CPSW_TX_CHAN */
|
||||
<0x12>, /* RING_SAUL_TX_0_CHAN */
|
||||
<0x13>, /* RING_SAUL_TX_1_CHAN */
|
||||
<0x14>, /* RING_ICSSG_0_TX_CHAN */
|
||||
<0x15>; /* RING_ICSSG_1_TX_CHAN */
|
||||
ti,sci-rm-range-rchan = <0x29>, /* UNMAPPED_RX_CHAN */
|
||||
<0x2b>, /* CPSW_RX_CHAN */
|
||||
<0x2d>, /* SAUL_RX_0_CHAN */
|
||||
<0x2f>, /* SAUL_RX_1_CHAN */
|
||||
<0x31>, /* SAUL_RX_2_CHAN */
|
||||
<0x33>, /* SAUL_RX_3_CHAN */
|
||||
<0x35>, /* ICSSG_0_RX_CHAN */
|
||||
<0x37>; /* ICSSG_1_RX_CHAN */
|
||||
ti,sci-rm-range-rflow = <0x2a>, /* FLOW_UNMAPPED_RX_CHAN */
|
||||
<0x2c>, /* FLOW_CPSW_RX_CHAN */
|
||||
<0x2e>, /* FLOW_SAUL_RX_0/1_CHAN */
|
||||
<0x32>, /* FLOW_SAUL_RX_2/3_CHAN */
|
||||
<0x36>, /* FLOW_ICSSG_0_RX_CHAN */
|
||||
<0x38>; /* FLOW_ICSSG_1_RX_CHAN */
|
||||
};
|
||||
};
|
||||
};
|
|
@ -120,7 +120,9 @@ The details of these operations are:
|
|||
|
||||
.. code-block:: c
|
||||
|
||||
nr_sg = dma_map_sg(chan->device->dev, sgl, sg_len);
|
||||
struct device *dma_dev = dmaengine_get_dma_device(chan);
|
||||
|
||||
nr_sg = dma_map_sg(dma_dev, sgl, sg_len);
|
||||
if (nr_sg == 0)
|
||||
/* error */
|
||||
|
||||
|
|
|
@ -296,6 +296,16 @@ config INTEL_IDXD
|
|||
|
||||
If unsure, say N.
|
||||
|
||||
# Config symbol that collects all the dependencies that's necessary to
|
||||
# support shared virtual memory for the devices supported by idxd.
|
||||
config INTEL_IDXD_SVM
|
||||
bool "Accelerator Shared Virtual Memory Support"
|
||||
depends on INTEL_IDXD
|
||||
depends on INTEL_IOMMU_SVM
|
||||
depends on PCI_PRI
|
||||
depends on PCI_PASID
|
||||
depends on PCI_IOV
|
||||
|
||||
config INTEL_IOATDMA
|
||||
tristate "Intel I/OAT DMA support"
|
||||
depends on PCI && X86_64
|
||||
|
|
|
@ -30,7 +30,24 @@
|
|||
#define AT_XDMAC_FIFO_SZ(i) (((i) >> 5) & 0x7FF) /* Number of Bytes */
|
||||
#define AT_XDMAC_NB_REQ(i) ((((i) >> 16) & 0x3F) + 1) /* Number of Peripheral Requests Minus One */
|
||||
#define AT_XDMAC_GCFG 0x04 /* Global Configuration Register */
|
||||
#define AT_XDMAC_WRHP(i) (((i) & 0xF) << 4)
|
||||
#define AT_XDMAC_WRMP(i) (((i) & 0xF) << 8)
|
||||
#define AT_XDMAC_WRLP(i) (((i) & 0xF) << 12)
|
||||
#define AT_XDMAC_RDHP(i) (((i) & 0xF) << 16)
|
||||
#define AT_XDMAC_RDMP(i) (((i) & 0xF) << 20)
|
||||
#define AT_XDMAC_RDLP(i) (((i) & 0xF) << 24)
|
||||
#define AT_XDMAC_RDSG(i) (((i) & 0xF) << 28)
|
||||
#define AT_XDMAC_GCFG_M2M (AT_XDMAC_RDLP(0xF) | AT_XDMAC_WRLP(0xF))
|
||||
#define AT_XDMAC_GCFG_P2M (AT_XDMAC_RDSG(0x1) | AT_XDMAC_RDHP(0x3) | \
|
||||
AT_XDMAC_WRHP(0x5))
|
||||
#define AT_XDMAC_GWAC 0x08 /* Global Weighted Arbiter Configuration Register */
|
||||
#define AT_XDMAC_PW0(i) (((i) & 0xF) << 0)
|
||||
#define AT_XDMAC_PW1(i) (((i) & 0xF) << 4)
|
||||
#define AT_XDMAC_PW2(i) (((i) & 0xF) << 8)
|
||||
#define AT_XDMAC_PW3(i) (((i) & 0xF) << 12)
|
||||
#define AT_XDMAC_GWAC_M2M 0
|
||||
#define AT_XDMAC_GWAC_P2M (AT_XDMAC_PW0(0xF) | AT_XDMAC_PW2(0xF))
|
||||
|
||||
#define AT_XDMAC_GIE 0x0C /* Global Interrupt Enable Register */
|
||||
#define AT_XDMAC_GID 0x10 /* Global Interrupt Disable Register */
|
||||
#define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */
|
||||
|
@ -38,13 +55,6 @@
|
|||
#define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */
|
||||
#define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */
|
||||
#define AT_XDMAC_GS 0x24 /* Global Channel Status Register */
|
||||
#define AT_XDMAC_GRS 0x28 /* Global Channel Read Suspend Register */
|
||||
#define AT_XDMAC_GWS 0x2C /* Global Write Suspend Register */
|
||||
#define AT_XDMAC_GRWS 0x30 /* Global Channel Read Write Suspend Register */
|
||||
#define AT_XDMAC_GRWR 0x34 /* Global Channel Read Write Resume Register */
|
||||
#define AT_XDMAC_GSWR 0x38 /* Global Channel Software Request Register */
|
||||
#define AT_XDMAC_GSWS 0x3C /* Global channel Software Request Status Register */
|
||||
#define AT_XDMAC_GSWF 0x40 /* Global Channel Software Flush Request Register */
|
||||
#define AT_XDMAC_VERSION 0xFFC /* XDMAC Version Register */
|
||||
|
||||
/* Channel relative registers offsets */
|
||||
|
@ -150,8 +160,6 @@
|
|||
#define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */
|
||||
#define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */
|
||||
|
||||
#define AT_XDMAC_CHAN_REG_BASE 0x50 /* Channel registers base address */
|
||||
|
||||
/* Microblock control members */
|
||||
#define AT_XDMAC_MBR_UBC_UBLEN_MAX 0xFFFFFFUL /* Maximum Microblock Length */
|
||||
#define AT_XDMAC_MBR_UBC_NDE (0x1 << 24) /* Next Descriptor Enable */
|
||||
|
@ -179,6 +187,29 @@ enum atc_status {
|
|||
AT_XDMAC_CHAN_IS_PAUSED,
|
||||
};
|
||||
|
||||
struct at_xdmac_layout {
|
||||
/* Global Channel Read Suspend Register */
|
||||
u8 grs;
|
||||
/* Global Write Suspend Register */
|
||||
u8 gws;
|
||||
/* Global Channel Read Write Suspend Register */
|
||||
u8 grws;
|
||||
/* Global Channel Read Write Resume Register */
|
||||
u8 grwr;
|
||||
/* Global Channel Software Request Register */
|
||||
u8 gswr;
|
||||
/* Global channel Software Request Status Register */
|
||||
u8 gsws;
|
||||
/* Global Channel Software Flush Request Register */
|
||||
u8 gswf;
|
||||
/* Channel reg base */
|
||||
u8 chan_cc_reg_base;
|
||||
/* Source/Destination Interface must be specified or not */
|
||||
bool sdif;
|
||||
/* AXI queue priority configuration supported */
|
||||
bool axi_config;
|
||||
};
|
||||
|
||||
/* ----- Channels ----- */
|
||||
struct at_xdmac_chan {
|
||||
struct dma_chan chan;
|
||||
|
@ -212,6 +243,7 @@ struct at_xdmac {
|
|||
struct clk *clk;
|
||||
u32 save_gim;
|
||||
struct dma_pool *at_xdmac_desc_pool;
|
||||
const struct at_xdmac_layout *layout;
|
||||
struct at_xdmac_chan chan[];
|
||||
};
|
||||
|
||||
|
@ -244,9 +276,35 @@ struct at_xdmac_desc {
|
|||
struct list_head xfer_node;
|
||||
} __aligned(sizeof(u64));
|
||||
|
||||
static const struct at_xdmac_layout at_xdmac_sama5d4_layout = {
|
||||
.grs = 0x28,
|
||||
.gws = 0x2C,
|
||||
.grws = 0x30,
|
||||
.grwr = 0x34,
|
||||
.gswr = 0x38,
|
||||
.gsws = 0x3C,
|
||||
.gswf = 0x40,
|
||||
.chan_cc_reg_base = 0x50,
|
||||
.sdif = true,
|
||||
.axi_config = false,
|
||||
};
|
||||
|
||||
static const struct at_xdmac_layout at_xdmac_sama7g5_layout = {
|
||||
.grs = 0x30,
|
||||
.gws = 0x38,
|
||||
.grws = 0x40,
|
||||
.grwr = 0x44,
|
||||
.gswr = 0x48,
|
||||
.gsws = 0x4C,
|
||||
.gswf = 0x50,
|
||||
.chan_cc_reg_base = 0x60,
|
||||
.sdif = false,
|
||||
.axi_config = true,
|
||||
};
|
||||
|
||||
static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)
|
||||
{
|
||||
return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);
|
||||
return atxdmac->regs + (atxdmac->layout->chan_cc_reg_base + chan_nb * 0x40);
|
||||
}
|
||||
|
||||
#define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))
|
||||
|
@ -345,8 +403,10 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
|
|||
first->active_xfer = true;
|
||||
|
||||
/* Tell xdmac where to get the first descriptor. */
|
||||
reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys)
|
||||
| AT_XDMAC_CNDA_NDAIF(atchan->memif);
|
||||
reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys);
|
||||
if (atxdmac->layout->sdif)
|
||||
reg |= AT_XDMAC_CNDA_NDAIF(atchan->memif);
|
||||
|
||||
at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);
|
||||
|
||||
/*
|
||||
|
@ -541,6 +601,7 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
|
|||
enum dma_transfer_direction direction)
|
||||
{
|
||||
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
|
||||
struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
|
||||
int csize, dwidth;
|
||||
|
||||
if (direction == DMA_DEV_TO_MEM) {
|
||||
|
@ -548,12 +609,14 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
|
|||
AT91_XDMAC_DT_PERID(atchan->perid)
|
||||
| AT_XDMAC_CC_DAM_INCREMENTED_AM
|
||||
| AT_XDMAC_CC_SAM_FIXED_AM
|
||||
| AT_XDMAC_CC_DIF(atchan->memif)
|
||||
| AT_XDMAC_CC_SIF(atchan->perif)
|
||||
| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
|
||||
| AT_XDMAC_CC_DSYNC_PER2MEM
|
||||
| AT_XDMAC_CC_MBSIZE_SIXTEEN
|
||||
| AT_XDMAC_CC_TYPE_PER_TRAN;
|
||||
if (atxdmac->layout->sdif)
|
||||
atchan->cfg |= AT_XDMAC_CC_DIF(atchan->memif) |
|
||||
AT_XDMAC_CC_SIF(atchan->perif);
|
||||
|
||||
csize = ffs(atchan->sconfig.src_maxburst) - 1;
|
||||
if (csize < 0) {
|
||||
dev_err(chan2dev(chan), "invalid src maxburst value\n");
|
||||
|
@ -571,12 +634,14 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
|
|||
AT91_XDMAC_DT_PERID(atchan->perid)
|
||||
| AT_XDMAC_CC_DAM_FIXED_AM
|
||||
| AT_XDMAC_CC_SAM_INCREMENTED_AM
|
||||
| AT_XDMAC_CC_DIF(atchan->perif)
|
||||
| AT_XDMAC_CC_SIF(atchan->memif)
|
||||
| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
|
||||
| AT_XDMAC_CC_DSYNC_MEM2PER
|
||||
| AT_XDMAC_CC_MBSIZE_SIXTEEN
|
||||
| AT_XDMAC_CC_TYPE_PER_TRAN;
|
||||
if (atxdmac->layout->sdif)
|
||||
atchan->cfg |= AT_XDMAC_CC_DIF(atchan->perif) |
|
||||
AT_XDMAC_CC_SIF(atchan->memif);
|
||||
|
||||
csize = ffs(atchan->sconfig.dst_maxburst) - 1;
|
||||
if (csize < 0) {
|
||||
dev_err(chan2dev(chan), "invalid src maxburst value\n");
|
||||
|
@ -866,10 +931,12 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
|
|||
* ERRATA: Even if useless for memory transfers, the PERID has to not
|
||||
* match the one of another channel. If not, it could lead to spurious
|
||||
* flag status.
|
||||
* For SAMA7G5x case, the SIF and DIF fields are no longer used.
|
||||
* Thus, no need to have the SIF/DIF interfaces here.
|
||||
* For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as
|
||||
* zero.
|
||||
*/
|
||||
u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
|
||||
| AT_XDMAC_CC_DIF(0)
|
||||
| AT_XDMAC_CC_SIF(0)
|
||||
u32 chan_cc = AT_XDMAC_CC_PERID(0x7f)
|
||||
| AT_XDMAC_CC_MBSIZE_SIXTEEN
|
||||
| AT_XDMAC_CC_TYPE_MEM_TRAN;
|
||||
|
||||
|
@ -1048,12 +1115,14 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
|
|||
* ERRATA: Even if useless for memory transfers, the PERID has to not
|
||||
* match the one of another channel. If not, it could lead to spurious
|
||||
* flag status.
|
||||
* For SAMA7G5x case, the SIF and DIF fields are no longer used.
|
||||
* Thus, no need to have the SIF/DIF interfaces here.
|
||||
* For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as
|
||||
* zero.
|
||||
*/
|
||||
u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
|
||||
u32 chan_cc = AT_XDMAC_CC_PERID(0x7f)
|
||||
| AT_XDMAC_CC_DAM_INCREMENTED_AM
|
||||
| AT_XDMAC_CC_SAM_INCREMENTED_AM
|
||||
| AT_XDMAC_CC_DIF(0)
|
||||
| AT_XDMAC_CC_SIF(0)
|
||||
| AT_XDMAC_CC_MBSIZE_SIXTEEN
|
||||
| AT_XDMAC_CC_TYPE_MEM_TRAN;
|
||||
unsigned long irqflags;
|
||||
|
@ -1154,12 +1223,14 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
|
|||
* ERRATA: Even if useless for memory transfers, the PERID has to not
|
||||
* match the one of another channel. If not, it could lead to spurious
|
||||
* flag status.
|
||||
* For SAMA7G5x case, the SIF and DIF fields are no longer used.
|
||||
* Thus, no need to have the SIF/DIF interfaces here.
|
||||
* For SAMA5D4x and SAMA5D2x the SIF and DIF are already configured as
|
||||
* zero.
|
||||
*/
|
||||
u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
|
||||
u32 chan_cc = AT_XDMAC_CC_PERID(0x7f)
|
||||
| AT_XDMAC_CC_DAM_UBS_AM
|
||||
| AT_XDMAC_CC_SAM_INCREMENTED_AM
|
||||
| AT_XDMAC_CC_DIF(0)
|
||||
| AT_XDMAC_CC_SIF(0)
|
||||
| AT_XDMAC_CC_MBSIZE_SIXTEEN
|
||||
| AT_XDMAC_CC_MEMSET_HW_MODE
|
||||
| AT_XDMAC_CC_TYPE_MEM_TRAN;
|
||||
|
@ -1438,7 +1509,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
|
|||
mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC;
|
||||
value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM;
|
||||
if ((desc->lld.mbr_cfg & mask) == value) {
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
|
||||
at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);
|
||||
while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
|
||||
cpu_relax();
|
||||
}
|
||||
|
@ -1496,7 +1567,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
|
|||
* FIFO flush ensures that data are really written.
|
||||
*/
|
||||
if ((desc->lld.mbr_cfg & mask) == value) {
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
|
||||
at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);
|
||||
while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
|
||||
cpu_relax();
|
||||
}
|
||||
|
@ -1761,7 +1832,7 @@ static int at_xdmac_device_pause(struct dma_chan *chan)
|
|||
return 0;
|
||||
|
||||
spin_lock_irqsave(&atchan->lock, flags);
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);
|
||||
at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask);
|
||||
while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)
|
||||
& (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP))
|
||||
cpu_relax();
|
||||
|
@ -1784,7 +1855,7 @@ static int at_xdmac_device_resume(struct dma_chan *chan)
|
|||
return 0;
|
||||
}
|
||||
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);
|
||||
at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask);
|
||||
clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
|
||||
spin_unlock_irqrestore(&atchan->lock, flags);
|
||||
|
||||
|
@ -1947,6 +2018,30 @@ static int atmel_xdmac_resume(struct device *dev)
|
|||
}
|
||||
#endif /* CONFIG_PM_SLEEP */
|
||||
|
||||
static void at_xdmac_axi_config(struct platform_device *pdev)
|
||||
{
|
||||
struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
|
||||
bool dev_m2m = false;
|
||||
u32 dma_requests;
|
||||
|
||||
if (!atxdmac->layout->axi_config)
|
||||
return; /* Not supported */
|
||||
|
||||
if (!of_property_read_u32(pdev->dev.of_node, "dma-requests",
|
||||
&dma_requests)) {
|
||||
dev_info(&pdev->dev, "controller in mem2mem mode.\n");
|
||||
dev_m2m = true;
|
||||
}
|
||||
|
||||
if (dev_m2m) {
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);
|
||||
} else {
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);
|
||||
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);
|
||||
}
|
||||
}
|
||||
|
||||
static int at_xdmac_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct at_xdmac *atxdmac;
|
||||
|
@ -1986,6 +2081,10 @@ static int at_xdmac_probe(struct platform_device *pdev)
|
|||
atxdmac->regs = base;
|
||||
atxdmac->irq = irq;
|
||||
|
||||
atxdmac->layout = of_device_get_match_data(&pdev->dev);
|
||||
if (!atxdmac->layout)
|
||||
return -ENODEV;
|
||||
|
||||
atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");
|
||||
if (IS_ERR(atxdmac->clk)) {
|
||||
dev_err(&pdev->dev, "can't get dma_clk\n");
|
||||
|
@ -2087,6 +2186,8 @@ static int at_xdmac_probe(struct platform_device *pdev)
|
|||
dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n",
|
||||
nr_channels, atxdmac->regs);
|
||||
|
||||
at_xdmac_axi_config(pdev);
|
||||
|
||||
return 0;
|
||||
|
||||
err_dma_unregister:
|
||||
|
@ -2128,6 +2229,10 @@ static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = {
|
|||
static const struct of_device_id atmel_xdmac_dt_ids[] = {
|
||||
{
|
||||
.compatible = "atmel,sama5d4-dma",
|
||||
.data = &at_xdmac_sama5d4_layout,
|
||||
}, {
|
||||
.compatible = "microchip,sama7g5-dma",
|
||||
.data = &at_xdmac_sama7g5_layout,
|
||||
}, {
|
||||
/* sentinel */
|
||||
}
|
||||
|
|
|
@ -1044,7 +1044,7 @@ static struct platform_driver jz4780_dma_driver = {
|
|||
.remove = jz4780_dma_remove,
|
||||
.driver = {
|
||||
.name = "jz4780-dma",
|
||||
.of_match_table = of_match_ptr(jz4780_dma_dt_match),
|
||||
.of_match_table = jz4780_dma_dt_match,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
|
@ -573,6 +573,7 @@ static int dmatest_func(void *data)
|
|||
struct dmatest_params *params;
|
||||
struct dma_chan *chan;
|
||||
struct dma_device *dev;
|
||||
struct device *dma_dev;
|
||||
unsigned int error_count;
|
||||
unsigned int failed_tests = 0;
|
||||
unsigned int total_tests = 0;
|
||||
|
@ -606,6 +607,8 @@ static int dmatest_func(void *data)
|
|||
params = &info->params;
|
||||
chan = thread->chan;
|
||||
dev = chan->device;
|
||||
dma_dev = dmaengine_get_dma_device(chan);
|
||||
|
||||
src = &thread->src;
|
||||
dst = &thread->dst;
|
||||
if (thread->type == DMA_MEMCPY) {
|
||||
|
@ -730,7 +733,7 @@ static int dmatest_func(void *data)
|
|||
filltime = ktime_add(filltime, diff);
|
||||
}
|
||||
|
||||
um = dmaengine_get_unmap_data(dev->dev, src->cnt + dst->cnt,
|
||||
um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
|
||||
GFP_KERNEL);
|
||||
if (!um) {
|
||||
failed_tests++;
|
||||
|
@ -745,10 +748,10 @@ static int dmatest_func(void *data)
|
|||
struct page *pg = virt_to_page(buf);
|
||||
unsigned long pg_off = offset_in_page(buf);
|
||||
|
||||
um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
|
||||
um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
|
||||
um->len, DMA_TO_DEVICE);
|
||||
srcs[i] = um->addr[i] + src->off;
|
||||
ret = dma_mapping_error(dev->dev, um->addr[i]);
|
||||
ret = dma_mapping_error(dma_dev, um->addr[i]);
|
||||
if (ret) {
|
||||
result("src mapping error", total_tests,
|
||||
src->off, dst->off, len, ret);
|
||||
|
@ -763,9 +766,9 @@ static int dmatest_func(void *data)
|
|||
struct page *pg = virt_to_page(buf);
|
||||
unsigned long pg_off = offset_in_page(buf);
|
||||
|
||||
dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
|
||||
dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
|
||||
DMA_BIDIRECTIONAL);
|
||||
ret = dma_mapping_error(dev->dev, dsts[i]);
|
||||
ret = dma_mapping_error(dma_dev, dsts[i]);
|
||||
if (ret) {
|
||||
result("dst mapping error", total_tests,
|
||||
src->off, dst->off, len, ret);
|
||||
|
|
|
@ -992,7 +992,7 @@ static struct platform_driver dw_driver = {
|
|||
.remove = dw_remove,
|
||||
.driver = {
|
||||
.name = KBUILD_MODNAME,
|
||||
.of_match_table = of_match_ptr(dw_dma_of_id_table),
|
||||
.of_match_table = dw_dma_of_id_table,
|
||||
.pm = &dw_axi_dma_pm_ops,
|
||||
},
|
||||
};
|
||||
|
|
|
@ -982,8 +982,11 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
|
|||
|
||||
dev_vdbg(chan2dev(chan), "%s\n", __func__);
|
||||
|
||||
pm_runtime_get_sync(dw->dma.dev);
|
||||
|
||||
/* ASSERT: channel is idle */
|
||||
if (dma_readl(dw, CH_EN) & dwc->mask) {
|
||||
pm_runtime_put_sync_suspend(dw->dma.dev);
|
||||
dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
|
||||
return -EIO;
|
||||
}
|
||||
|
@ -1000,6 +1003,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
|
|||
* We need controller-specific data to set up slave transfers.
|
||||
*/
|
||||
if (chan->private && !dw_dma_filter(chan, chan->private)) {
|
||||
pm_runtime_put_sync_suspend(dw->dma.dev);
|
||||
dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
@ -1043,6 +1047,8 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
|
|||
if (!dw->in_use)
|
||||
do_dw_dma_off(dw);
|
||||
|
||||
pm_runtime_put_sync_suspend(dw->dma.dev);
|
||||
|
||||
dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
|
||||
}
|
||||
|
||||
|
|
|
@ -431,9 +431,8 @@ static irqreturn_t hisi_dma_irq(int irq, void *data)
|
|||
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
|
||||
struct hisi_dma_desc *desc;
|
||||
struct hisi_dma_cqe *cqe;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&chan->vc.lock, flags);
|
||||
spin_lock(&chan->vc.lock);
|
||||
|
||||
desc = chan->desc;
|
||||
cqe = chan->cq + chan->cq_head;
|
||||
|
@ -452,7 +451,7 @@ static irqreturn_t hisi_dma_irq(int irq, void *data)
|
|||
chan->desc = NULL;
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&chan->vc.lock, flags);
|
||||
spin_unlock(&chan->vc.lock);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
|
|
@ -667,9 +667,7 @@ static int idma64_platform_remove(struct platform_device *pdev)
|
|||
return idma64_remove(chip);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PM_SLEEP
|
||||
|
||||
static int idma64_pm_suspend(struct device *dev)
|
||||
static int __maybe_unused idma64_pm_suspend(struct device *dev)
|
||||
{
|
||||
struct idma64_chip *chip = dev_get_drvdata(dev);
|
||||
|
||||
|
@ -677,7 +675,7 @@ static int idma64_pm_suspend(struct device *dev)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int idma64_pm_resume(struct device *dev)
|
||||
static int __maybe_unused idma64_pm_resume(struct device *dev)
|
||||
{
|
||||
struct idma64_chip *chip = dev_get_drvdata(dev);
|
||||
|
||||
|
@ -685,8 +683,6 @@ static int idma64_pm_resume(struct device *dev)
|
|||
return 0;
|
||||
}
|
||||
|
||||
#endif /* CONFIG_PM_SLEEP */
|
||||
|
||||
static const struct dev_pm_ops idma64_dev_pm_ops = {
|
||||
SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend, idma64_pm_resume)
|
||||
};
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
#include <linux/cdev.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/poll.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <uapi/linux/idxd.h>
|
||||
#include "registers.h"
|
||||
#include "idxd.h"
|
||||
|
@ -27,12 +28,15 @@ struct idxd_cdev_context {
|
|||
*/
|
||||
static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
|
||||
{ .name = "dsa" },
|
||||
{ .name = "iax" }
|
||||
};
|
||||
|
||||
struct idxd_user_context {
|
||||
struct idxd_wq *wq;
|
||||
struct task_struct *task;
|
||||
unsigned int pasid;
|
||||
unsigned int flags;
|
||||
struct iommu_sva *sva;
|
||||
};
|
||||
|
||||
enum idxd_cdev_cleanup {
|
||||
|
@ -75,6 +79,8 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
|
|||
struct idxd_wq *wq;
|
||||
struct device *dev;
|
||||
int rc = 0;
|
||||
struct iommu_sva *sva;
|
||||
unsigned int pasid;
|
||||
|
||||
wq = inode_wq(inode);
|
||||
idxd = wq->idxd;
|
||||
|
@ -95,6 +101,34 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
|
|||
|
||||
ctx->wq = wq;
|
||||
filp->private_data = ctx;
|
||||
|
||||
if (device_pasid_enabled(idxd)) {
|
||||
sva = iommu_sva_bind_device(dev, current->mm, NULL);
|
||||
if (IS_ERR(sva)) {
|
||||
rc = PTR_ERR(sva);
|
||||
dev_err(dev, "pasid allocation failed: %d\n", rc);
|
||||
goto failed;
|
||||
}
|
||||
|
||||
pasid = iommu_sva_get_pasid(sva);
|
||||
if (pasid == IOMMU_PASID_INVALID) {
|
||||
iommu_sva_unbind_device(sva);
|
||||
goto failed;
|
||||
}
|
||||
|
||||
ctx->sva = sva;
|
||||
ctx->pasid = pasid;
|
||||
|
||||
if (wq_dedicated(wq)) {
|
||||
rc = idxd_wq_set_pasid(wq, pasid);
|
||||
if (rc < 0) {
|
||||
iommu_sva_unbind_device(sva);
|
||||
dev_err(dev, "wq set pasid failed: %d\n", rc);
|
||||
goto failed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
idxd_wq_get(wq);
|
||||
mutex_unlock(&wq->wq_lock);
|
||||
return 0;
|
||||
|
@ -111,13 +145,27 @@ static int idxd_cdev_release(struct inode *node, struct file *filep)
|
|||
struct idxd_wq *wq = ctx->wq;
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
struct device *dev = &idxd->pdev->dev;
|
||||
int rc;
|
||||
|
||||
dev_dbg(dev, "%s called\n", __func__);
|
||||
filep->private_data = NULL;
|
||||
|
||||
/* Wait for in-flight operations to complete. */
|
||||
idxd_wq_drain(wq);
|
||||
if (wq_shared(wq)) {
|
||||
idxd_device_drain_pasid(idxd, ctx->pasid);
|
||||
} else {
|
||||
if (device_pasid_enabled(idxd)) {
|
||||
/* The wq disable in the disable pasid function will drain the wq */
|
||||
rc = idxd_wq_disable_pasid(wq);
|
||||
if (rc < 0)
|
||||
dev_err(dev, "wq disable pasid failed.\n");
|
||||
} else {
|
||||
idxd_wq_drain(wq);
|
||||
}
|
||||
}
|
||||
|
||||
if (ctx->sva)
|
||||
iommu_sva_unbind_device(ctx->sva);
|
||||
kfree(ctx);
|
||||
mutex_lock(&wq->wq_lock);
|
||||
idxd_wq_put(wq);
|
||||
|
|
|
@ -131,6 +131,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
|
|||
struct idxd_device *idxd = wq->idxd;
|
||||
struct device *dev = &idxd->pdev->dev;
|
||||
int rc, num_descs, i;
|
||||
int align;
|
||||
u64 tmp;
|
||||
|
||||
if (wq->type != IDXD_WQT_KERNEL)
|
||||
return 0;
|
||||
|
@ -142,14 +144,27 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
|
|||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
|
||||
wq->compls = dma_alloc_coherent(dev, wq->compls_size,
|
||||
&wq->compls_addr, GFP_KERNEL);
|
||||
if (!wq->compls) {
|
||||
if (idxd->type == IDXD_TYPE_DSA)
|
||||
align = 32;
|
||||
else if (idxd->type == IDXD_TYPE_IAX)
|
||||
align = 64;
|
||||
else
|
||||
return -ENODEV;
|
||||
|
||||
wq->compls_size = num_descs * idxd->compl_size + align;
|
||||
wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size,
|
||||
&wq->compls_addr_raw, GFP_KERNEL);
|
||||
if (!wq->compls_raw) {
|
||||
rc = -ENOMEM;
|
||||
goto fail_alloc_compls;
|
||||
}
|
||||
|
||||
/* Adjust alignment */
|
||||
wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1);
|
||||
tmp = (u64)wq->compls_raw;
|
||||
tmp = (tmp + (align - 1)) & ~(align - 1);
|
||||
wq->compls = (struct dsa_completion_record *)tmp;
|
||||
|
||||
rc = alloc_descs(wq, num_descs);
|
||||
if (rc < 0)
|
||||
goto fail_alloc_descs;
|
||||
|
@ -163,9 +178,11 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
|
|||
struct idxd_desc *desc = wq->descs[i];
|
||||
|
||||
desc->hw = wq->hw_descs[i];
|
||||
desc->completion = &wq->compls[i];
|
||||
desc->compl_dma = wq->compls_addr +
|
||||
sizeof(struct dsa_completion_record) * i;
|
||||
if (idxd->type == IDXD_TYPE_DSA)
|
||||
desc->completion = &wq->compls[i];
|
||||
else if (idxd->type == IDXD_TYPE_IAX)
|
||||
desc->iax_completion = &wq->iax_compls[i];
|
||||
desc->compl_dma = wq->compls_addr + idxd->compl_size * i;
|
||||
desc->id = i;
|
||||
desc->wq = wq;
|
||||
desc->cpu = -1;
|
||||
|
@ -178,7 +195,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
|
|||
fail_sbitmap_init:
|
||||
free_descs(wq);
|
||||
fail_alloc_descs:
|
||||
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
|
||||
dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
|
||||
wq->compls_addr_raw);
|
||||
fail_alloc_compls:
|
||||
free_hw_descs(wq);
|
||||
return rc;
|
||||
|
@ -193,7 +211,8 @@ void idxd_wq_free_resources(struct idxd_wq *wq)
|
|||
|
||||
free_hw_descs(wq);
|
||||
free_descs(wq);
|
||||
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
|
||||
dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
|
||||
wq->compls_addr_raw);
|
||||
sbitmap_queue_free(&wq->sbq);
|
||||
}
|
||||
|
||||
|
@ -273,10 +292,9 @@ int idxd_wq_map_portal(struct idxd_wq *wq)
|
|||
start = pci_resource_start(pdev, IDXD_WQ_BAR);
|
||||
start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
|
||||
|
||||
wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
|
||||
if (!wq->dportal)
|
||||
wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
|
||||
if (!wq->portal)
|
||||
return -ENOMEM;
|
||||
dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -285,7 +303,61 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq)
|
|||
{
|
||||
struct device *dev = &wq->idxd->pdev->dev;
|
||||
|
||||
devm_iounmap(dev, wq->dportal);
|
||||
devm_iounmap(dev, wq->portal);
|
||||
}
|
||||
|
||||
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
|
||||
{
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
int rc;
|
||||
union wqcfg wqcfg;
|
||||
unsigned int offset;
|
||||
unsigned long flags;
|
||||
|
||||
rc = idxd_wq_disable(wq);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
|
||||
spin_lock_irqsave(&idxd->dev_lock, flags);
|
||||
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
|
||||
wqcfg.pasid_en = 1;
|
||||
wqcfg.pasid = pasid;
|
||||
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
|
||||
spin_unlock_irqrestore(&idxd->dev_lock, flags);
|
||||
|
||||
rc = idxd_wq_enable(wq);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int idxd_wq_disable_pasid(struct idxd_wq *wq)
|
||||
{
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
int rc;
|
||||
union wqcfg wqcfg;
|
||||
unsigned int offset;
|
||||
unsigned long flags;
|
||||
|
||||
rc = idxd_wq_disable(wq);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
|
||||
spin_lock_irqsave(&idxd->dev_lock, flags);
|
||||
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
|
||||
wqcfg.pasid_en = 0;
|
||||
wqcfg.pasid = 0;
|
||||
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
|
||||
spin_unlock_irqrestore(&idxd->dev_lock, flags);
|
||||
|
||||
rc = idxd_wq_enable(wq);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void idxd_wq_disable_cleanup(struct idxd_wq *wq)
|
||||
|
@ -301,6 +373,7 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq)
|
|||
wq->group = NULL;
|
||||
wq->threshold = 0;
|
||||
wq->priority = 0;
|
||||
wq->ats_dis = 0;
|
||||
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
|
||||
memset(wq->name, 0, WQ_NAME_SIZE);
|
||||
|
||||
|
@ -468,6 +541,17 @@ void idxd_device_reset(struct idxd_device *idxd)
|
|||
spin_unlock_irqrestore(&idxd->dev_lock, flags);
|
||||
}
|
||||
|
||||
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
|
||||
{
|
||||
struct device *dev = &idxd->pdev->dev;
|
||||
u32 operand;
|
||||
|
||||
operand = pasid;
|
||||
dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand);
|
||||
idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL);
|
||||
dev_dbg(dev, "pasid %d drained\n", pasid);
|
||||
}
|
||||
|
||||
/* Device configuration bits */
|
||||
static void idxd_group_config_write(struct idxd_group *group)
|
||||
{
|
||||
|
@ -479,24 +563,22 @@ static void idxd_group_config_write(struct idxd_group *group)
|
|||
dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
|
||||
|
||||
/* setup GRPWQCFG */
|
||||
for (i = 0; i < 4; i++) {
|
||||
grpcfg_offset = idxd->grpcfg_offset +
|
||||
group->id * 64 + i * sizeof(u64);
|
||||
iowrite64(group->grpcfg.wqs[i],
|
||||
idxd->reg_base + grpcfg_offset);
|
||||
for (i = 0; i < GRPWQCFG_STRIDES; i++) {
|
||||
grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
|
||||
iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset);
|
||||
dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
|
||||
group->id, i, grpcfg_offset,
|
||||
ioread64(idxd->reg_base + grpcfg_offset));
|
||||
}
|
||||
|
||||
/* setup GRPENGCFG */
|
||||
grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
|
||||
grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
|
||||
iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
|
||||
dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
|
||||
grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
|
||||
|
||||
/* setup GRPFLAGS */
|
||||
grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
|
||||
grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
|
||||
iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
|
||||
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
|
||||
group->id, grpcfg_offset,
|
||||
|
@ -554,9 +636,24 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
|
|||
|
||||
/* byte 8-11 */
|
||||
wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
|
||||
wq->wqcfg->mode = 1;
|
||||
if (wq_dedicated(wq))
|
||||
wq->wqcfg->mode = 1;
|
||||
|
||||
if (device_pasid_enabled(idxd)) {
|
||||
wq->wqcfg->pasid_en = 1;
|
||||
if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq))
|
||||
wq->wqcfg->pasid = idxd->pasid;
|
||||
}
|
||||
|
||||
wq->wqcfg->priority = wq->priority;
|
||||
|
||||
if (idxd->hw.gen_cap.block_on_fault &&
|
||||
test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags))
|
||||
wq->wqcfg->bof = 1;
|
||||
|
||||
if (idxd->hw.wq_cap.wq_ats_support)
|
||||
wq->wqcfg->wq_ats_disable = wq->ats_dis;
|
||||
|
||||
/* bytes 12-15 */
|
||||
wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
|
||||
wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
|
||||
|
@ -664,8 +761,8 @@ static int idxd_wqs_setup(struct idxd_device *idxd)
|
|||
if (!wq->size)
|
||||
continue;
|
||||
|
||||
if (!wq_dedicated(wq)) {
|
||||
dev_warn(dev, "No shared workqueue support.\n");
|
||||
if (wq_shared(wq) && !device_swq_supported(idxd)) {
|
||||
dev_warn(dev, "No shared wq support but configured.\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
|
|
|
@ -61,8 +61,6 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
|
|||
u64 addr_f1, u64 addr_f2, u64 len,
|
||||
u64 compl, u32 flags)
|
||||
{
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
|
||||
hw->flags = flags;
|
||||
hw->opcode = opcode;
|
||||
hw->src_addr = addr_f1;
|
||||
|
@ -70,13 +68,6 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
|
|||
hw->xfer_size = len;
|
||||
hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
|
||||
hw->completion_addr = compl;
|
||||
|
||||
/*
|
||||
* Descriptor completion vectors are 1-8 for MSIX. We will round
|
||||
* robin through the 8 vectors.
|
||||
*/
|
||||
wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
|
||||
hw->int_handle = wq->vec_ptr;
|
||||
}
|
||||
|
||||
static struct dma_async_tx_descriptor *
|
||||
|
|
|
@ -20,7 +20,8 @@ extern struct kmem_cache *idxd_desc_pool;
|
|||
enum idxd_type {
|
||||
IDXD_TYPE_UNKNOWN = -1,
|
||||
IDXD_TYPE_DSA = 0,
|
||||
IDXD_TYPE_MAX
|
||||
IDXD_TYPE_IAX,
|
||||
IDXD_TYPE_MAX,
|
||||
};
|
||||
|
||||
#define IDXD_NAME_SIZE 128
|
||||
|
@ -34,6 +35,11 @@ struct idxd_irq_entry {
|
|||
int id;
|
||||
struct llist_head pending_llist;
|
||||
struct list_head work_list;
|
||||
/*
|
||||
* Lock to protect access between irq thread process descriptor
|
||||
* and irq thread processing error descriptor.
|
||||
*/
|
||||
spinlock_t list_lock;
|
||||
};
|
||||
|
||||
struct idxd_group {
|
||||
|
@ -59,6 +65,7 @@ enum idxd_wq_state {
|
|||
|
||||
enum idxd_wq_flag {
|
||||
WQ_FLAG_DEDICATED = 0,
|
||||
WQ_FLAG_BLOCK_ON_FAULT,
|
||||
};
|
||||
|
||||
enum idxd_wq_type {
|
||||
|
@ -86,10 +93,11 @@ enum idxd_op_type {
|
|||
enum idxd_complete_type {
|
||||
IDXD_COMPLETE_NORMAL = 0,
|
||||
IDXD_COMPLETE_ABORT,
|
||||
IDXD_COMPLETE_DEV_FAIL,
|
||||
};
|
||||
|
||||
struct idxd_wq {
|
||||
void __iomem *dportal;
|
||||
void __iomem *portal;
|
||||
struct device conf_dev;
|
||||
struct idxd_cdev idxd_cdev;
|
||||
struct idxd_device *idxd;
|
||||
|
@ -107,8 +115,13 @@ struct idxd_wq {
|
|||
u32 vec_ptr; /* interrupt steering */
|
||||
struct dsa_hw_desc **hw_descs;
|
||||
int num_descs;
|
||||
struct dsa_completion_record *compls;
|
||||
union {
|
||||
struct dsa_completion_record *compls;
|
||||
struct iax_completion_record *iax_compls;
|
||||
};
|
||||
void *compls_raw;
|
||||
dma_addr_t compls_addr;
|
||||
dma_addr_t compls_addr_raw;
|
||||
int compls_size;
|
||||
struct idxd_desc **descs;
|
||||
struct sbitmap_queue sbq;
|
||||
|
@ -116,6 +129,7 @@ struct idxd_wq {
|
|||
char name[WQ_NAME_SIZE + 1];
|
||||
u64 max_xfer_bytes;
|
||||
u32 max_batch_size;
|
||||
bool ats_dis;
|
||||
};
|
||||
|
||||
struct idxd_engine {
|
||||
|
@ -145,6 +159,7 @@ enum idxd_device_state {
|
|||
enum idxd_device_flag {
|
||||
IDXD_FLAG_CONFIGURABLE = 0,
|
||||
IDXD_FLAG_CMD_RUNNING,
|
||||
IDXD_FLAG_PASID_ENABLED,
|
||||
};
|
||||
|
||||
struct idxd_device {
|
||||
|
@ -167,6 +182,9 @@ struct idxd_device {
|
|||
struct idxd_wq *wqs;
|
||||
struct idxd_engine *engines;
|
||||
|
||||
struct iommu_sva *sva;
|
||||
unsigned int pasid;
|
||||
|
||||
int num_groups;
|
||||
|
||||
u32 msix_perm_offset;
|
||||
|
@ -184,6 +202,7 @@ struct idxd_device {
|
|||
int token_limit;
|
||||
int nr_tokens; /* non-reserved tokens */
|
||||
unsigned int wqcfg_size;
|
||||
int compl_size;
|
||||
|
||||
union sw_err_reg sw_err;
|
||||
wait_queue_head_t cmd_waitq;
|
||||
|
@ -198,9 +217,15 @@ struct idxd_device {
|
|||
|
||||
/* IDXD software descriptor */
|
||||
struct idxd_desc {
|
||||
struct dsa_hw_desc *hw;
|
||||
union {
|
||||
struct dsa_hw_desc *hw;
|
||||
struct iax_hw_desc *iax_hw;
|
||||
};
|
||||
dma_addr_t desc_dma;
|
||||
struct dsa_completion_record *completion;
|
||||
union {
|
||||
struct dsa_completion_record *completion;
|
||||
struct iax_completion_record *iax_completion;
|
||||
};
|
||||
dma_addr_t compl_dma;
|
||||
struct dma_async_tx_descriptor txd;
|
||||
struct llist_node llnode;
|
||||
|
@ -214,12 +239,30 @@ struct idxd_desc {
|
|||
#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
|
||||
|
||||
extern struct bus_type dsa_bus_type;
|
||||
extern struct bus_type iax_bus_type;
|
||||
|
||||
extern bool support_enqcmd;
|
||||
|
||||
static inline bool wq_dedicated(struct idxd_wq *wq)
|
||||
{
|
||||
return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
|
||||
}
|
||||
|
||||
static inline bool wq_shared(struct idxd_wq *wq)
|
||||
{
|
||||
return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
|
||||
}
|
||||
|
||||
static inline bool device_pasid_enabled(struct idxd_device *idxd)
|
||||
{
|
||||
return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
|
||||
}
|
||||
|
||||
static inline bool device_swq_supported(struct idxd_device *idxd)
|
||||
{
|
||||
return (support_enqcmd && device_pasid_enabled(idxd));
|
||||
}
|
||||
|
||||
enum idxd_portal_prot {
|
||||
IDXD_PORTAL_UNLIMITED = 0,
|
||||
IDXD_PORTAL_LIMITED,
|
||||
|
@ -242,6 +285,8 @@ static inline void idxd_set_type(struct idxd_device *idxd)
|
|||
|
||||
if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
|
||||
idxd->type = IDXD_TYPE_DSA;
|
||||
else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0)
|
||||
idxd->type = IDXD_TYPE_IAX;
|
||||
else
|
||||
idxd->type = IDXD_TYPE_UNKNOWN;
|
||||
}
|
||||
|
@ -288,6 +333,7 @@ void idxd_device_reset(struct idxd_device *idxd);
|
|||
void idxd_device_cleanup(struct idxd_device *idxd);
|
||||
int idxd_device_config(struct idxd_device *idxd);
|
||||
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
|
||||
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
|
||||
|
||||
/* work queue control */
|
||||
int idxd_wq_alloc_resources(struct idxd_wq *wq);
|
||||
|
@ -298,6 +344,8 @@ void idxd_wq_drain(struct idxd_wq *wq);
|
|||
int idxd_wq_map_portal(struct idxd_wq *wq);
|
||||
void idxd_wq_unmap_portal(struct idxd_wq *wq);
|
||||
void idxd_wq_disable_cleanup(struct idxd_wq *wq);
|
||||
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
|
||||
int idxd_wq_disable_pasid(struct idxd_wq *wq);
|
||||
|
||||
/* submission */
|
||||
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
|
||||
|
|
|
@ -14,6 +14,8 @@
|
|||
#include <linux/io-64-nonatomic-lo-hi.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/idr.h>
|
||||
#include <linux/intel-svm.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <uapi/linux/idxd.h>
|
||||
#include <linux/dmaengine.h>
|
||||
#include "../dmaengine.h"
|
||||
|
@ -26,18 +28,24 @@ MODULE_AUTHOR("Intel Corporation");
|
|||
|
||||
#define DRV_NAME "idxd"
|
||||
|
||||
bool support_enqcmd;
|
||||
|
||||
static struct idr idxd_idrs[IDXD_TYPE_MAX];
|
||||
static struct mutex idxd_idr_lock;
|
||||
|
||||
static struct pci_device_id idxd_pci_tbl[] = {
|
||||
/* DSA ver 1.0 platforms */
|
||||
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
|
||||
|
||||
/* IAX ver 1.0 platforms */
|
||||
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) },
|
||||
{ 0, }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
|
||||
|
||||
static char *idxd_name[] = {
|
||||
"dsa",
|
||||
"iax"
|
||||
};
|
||||
|
||||
const char *idxd_get_dev_name(struct idxd_device *idxd)
|
||||
|
@ -53,6 +61,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
|
|||
struct idxd_irq_entry *irq_entry;
|
||||
int i, msixcnt;
|
||||
int rc = 0;
|
||||
union msix_perm mperm;
|
||||
|
||||
msixcnt = pci_msix_vec_count(pdev);
|
||||
if (msixcnt < 0) {
|
||||
|
@ -92,6 +101,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
|
|||
for (i = 0; i < msixcnt; i++) {
|
||||
idxd->irq_entries[i].id = i;
|
||||
idxd->irq_entries[i].idxd = idxd;
|
||||
spin_lock_init(&idxd->irq_entries[i].list_lock);
|
||||
}
|
||||
|
||||
msix = &idxd->msix_entries[0];
|
||||
|
@ -131,6 +141,13 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
|
|||
|
||||
idxd_unmask_error_interrupts(idxd);
|
||||
|
||||
/* Setup MSIX permission table */
|
||||
mperm.bits = 0;
|
||||
mperm.pasid = idxd->pasid;
|
||||
mperm.pasid_en = device_pasid_enabled(idxd);
|
||||
for (i = 1; i < msixcnt; i++)
|
||||
iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8);
|
||||
|
||||
return 0;
|
||||
|
||||
err_no_irq:
|
||||
|
@ -201,17 +218,14 @@ static void idxd_read_table_offsets(struct idxd_device *idxd)
|
|||
struct device *dev = &idxd->pdev->dev;
|
||||
|
||||
offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET);
|
||||
offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET
|
||||
+ sizeof(u64));
|
||||
idxd->grpcfg_offset = offsets.grpcfg * 0x100;
|
||||
offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64));
|
||||
idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT;
|
||||
dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset);
|
||||
idxd->wqcfg_offset = offsets.wqcfg * 0x100;
|
||||
dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n",
|
||||
idxd->wqcfg_offset);
|
||||
idxd->msix_perm_offset = offsets.msix_perm * 0x100;
|
||||
dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n",
|
||||
idxd->msix_perm_offset);
|
||||
idxd->perfmon_offset = offsets.perfmon * 0x100;
|
||||
idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT;
|
||||
dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset);
|
||||
idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT;
|
||||
dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset);
|
||||
idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT;
|
||||
dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
|
||||
}
|
||||
|
||||
|
@ -265,8 +279,7 @@ static void idxd_read_caps(struct idxd_device *idxd)
|
|||
}
|
||||
}
|
||||
|
||||
static struct idxd_device *idxd_alloc(struct pci_dev *pdev,
|
||||
void __iomem * const *iomap)
|
||||
static struct idxd_device *idxd_alloc(struct pci_dev *pdev)
|
||||
{
|
||||
struct device *dev = &pdev->dev;
|
||||
struct idxd_device *idxd;
|
||||
|
@ -276,12 +289,45 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev,
|
|||
return NULL;
|
||||
|
||||
idxd->pdev = pdev;
|
||||
idxd->reg_base = iomap[IDXD_MMIO_BAR];
|
||||
spin_lock_init(&idxd->dev_lock);
|
||||
|
||||
return idxd;
|
||||
}
|
||||
|
||||
static int idxd_enable_system_pasid(struct idxd_device *idxd)
|
||||
{
|
||||
int flags;
|
||||
unsigned int pasid;
|
||||
struct iommu_sva *sva;
|
||||
|
||||
flags = SVM_FLAG_SUPERVISOR_MODE;
|
||||
|
||||
sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
|
||||
if (IS_ERR(sva)) {
|
||||
dev_warn(&idxd->pdev->dev,
|
||||
"iommu sva bind failed: %ld\n", PTR_ERR(sva));
|
||||
return PTR_ERR(sva);
|
||||
}
|
||||
|
||||
pasid = iommu_sva_get_pasid(sva);
|
||||
if (pasid == IOMMU_PASID_INVALID) {
|
||||
iommu_sva_unbind_device(sva);
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
idxd->sva = sva;
|
||||
idxd->pasid = pasid;
|
||||
dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void idxd_disable_system_pasid(struct idxd_device *idxd)
|
||||
{
|
||||
|
||||
iommu_sva_unbind_device(idxd->sva);
|
||||
idxd->sva = NULL;
|
||||
}
|
||||
|
||||
static int idxd_probe(struct idxd_device *idxd)
|
||||
{
|
||||
struct pci_dev *pdev = idxd->pdev;
|
||||
|
@ -292,6 +338,14 @@ static int idxd_probe(struct idxd_device *idxd)
|
|||
idxd_device_init_reset(idxd);
|
||||
dev_dbg(dev, "IDXD reset complete\n");
|
||||
|
||||
if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) {
|
||||
rc = idxd_enable_system_pasid(idxd);
|
||||
if (rc < 0)
|
||||
dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
|
||||
else
|
||||
set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
|
||||
}
|
||||
|
||||
idxd_read_caps(idxd);
|
||||
idxd_read_table_offsets(idxd);
|
||||
|
||||
|
@ -322,29 +376,37 @@ static int idxd_probe(struct idxd_device *idxd)
|
|||
idxd_mask_error_interrupts(idxd);
|
||||
idxd_mask_msix_vectors(idxd);
|
||||
err_setup:
|
||||
if (device_pasid_enabled(idxd))
|
||||
idxd_disable_system_pasid(idxd);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static void idxd_type_init(struct idxd_device *idxd)
|
||||
{
|
||||
if (idxd->type == IDXD_TYPE_DSA)
|
||||
idxd->compl_size = sizeof(struct dsa_completion_record);
|
||||
else if (idxd->type == IDXD_TYPE_IAX)
|
||||
idxd->compl_size = sizeof(struct iax_completion_record);
|
||||
}
|
||||
|
||||
static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
||||
{
|
||||
void __iomem * const *iomap;
|
||||
struct device *dev = &pdev->dev;
|
||||
struct idxd_device *idxd;
|
||||
int rc;
|
||||
unsigned int mask;
|
||||
|
||||
rc = pcim_enable_device(pdev);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
dev_dbg(dev, "Mapping BARs\n");
|
||||
mask = (1 << IDXD_MMIO_BAR);
|
||||
rc = pcim_iomap_regions(pdev, mask, DRV_NAME);
|
||||
if (rc)
|
||||
return rc;
|
||||
dev_dbg(dev, "Alloc IDXD context\n");
|
||||
idxd = idxd_alloc(pdev);
|
||||
if (!idxd)
|
||||
return -ENOMEM;
|
||||
|
||||
iomap = pcim_iomap_table(pdev);
|
||||
if (!iomap)
|
||||
dev_dbg(dev, "Mapping BARs\n");
|
||||
idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0);
|
||||
if (!idxd->reg_base)
|
||||
return -ENOMEM;
|
||||
|
||||
dev_dbg(dev, "Set DMA masks\n");
|
||||
|
@ -360,13 +422,10 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
|||
if (rc)
|
||||
return rc;
|
||||
|
||||
dev_dbg(dev, "Alloc IDXD context\n");
|
||||
idxd = idxd_alloc(pdev, iomap);
|
||||
if (!idxd)
|
||||
return -ENOMEM;
|
||||
|
||||
idxd_set_type(idxd);
|
||||
|
||||
idxd_type_init(idxd);
|
||||
|
||||
dev_dbg(dev, "Set PCI master\n");
|
||||
pci_set_master(pdev);
|
||||
pci_set_drvdata(pdev, idxd);
|
||||
|
@ -452,6 +511,8 @@ static void idxd_remove(struct pci_dev *pdev)
|
|||
dev_dbg(&pdev->dev, "%s called\n", __func__);
|
||||
idxd_cleanup_sysfs(idxd);
|
||||
idxd_shutdown(pdev);
|
||||
if (device_pasid_enabled(idxd))
|
||||
idxd_disable_system_pasid(idxd);
|
||||
mutex_lock(&idxd_idr_lock);
|
||||
idr_remove(&idxd_idrs[idxd->type], idxd->id);
|
||||
mutex_unlock(&idxd_idr_lock);
|
||||
|
@ -470,7 +531,7 @@ static int __init idxd_init_module(void)
|
|||
int err, i;
|
||||
|
||||
/*
|
||||
* If the CPU does not support write512, there's no point in
|
||||
* If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
|
||||
* enumerating the device. We can not utilize it.
|
||||
*/
|
||||
if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) {
|
||||
|
@ -478,8 +539,10 @@ static int __init idxd_init_module(void)
|
|||
return -ENODEV;
|
||||
}
|
||||
|
||||
pr_info("%s: Intel(R) Accelerator Devices Driver %s\n",
|
||||
DRV_NAME, IDXD_DRIVER_VERSION);
|
||||
if (!boot_cpu_has(X86_FEATURE_ENQCMD))
|
||||
pr_warn("Platform does not have ENQCMD(S) support.\n");
|
||||
else
|
||||
support_enqcmd = true;
|
||||
|
||||
mutex_init(&idxd_idr_lock);
|
||||
for (i = 0; i < IDXD_TYPE_MAX; i++)
|
||||
|
|
|
@ -11,6 +11,24 @@
|
|||
#include "idxd.h"
|
||||
#include "registers.h"
|
||||
|
||||
enum irq_work_type {
|
||||
IRQ_WORK_NORMAL = 0,
|
||||
IRQ_WORK_PROCESS_FAULT,
|
||||
};
|
||||
|
||||
struct idxd_fault {
|
||||
struct work_struct work;
|
||||
u64 addr;
|
||||
struct idxd_device *idxd;
|
||||
};
|
||||
|
||||
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
|
||||
enum irq_work_type wtype,
|
||||
int *processed, u64 data);
|
||||
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
|
||||
enum irq_work_type wtype,
|
||||
int *processed, u64 data);
|
||||
|
||||
static void idxd_device_reinit(struct work_struct *work)
|
||||
{
|
||||
struct idxd_device *idxd = container_of(work, struct idxd_device, work);
|
||||
|
@ -44,6 +62,46 @@ static void idxd_device_reinit(struct work_struct *work)
|
|||
idxd_device_wqs_clear_state(idxd);
|
||||
}
|
||||
|
||||
static void idxd_device_fault_work(struct work_struct *work)
|
||||
{
|
||||
struct idxd_fault *fault = container_of(work, struct idxd_fault, work);
|
||||
struct idxd_irq_entry *ie;
|
||||
int i;
|
||||
int processed;
|
||||
int irqcnt = fault->idxd->num_wq_irqs + 1;
|
||||
|
||||
for (i = 1; i < irqcnt; i++) {
|
||||
ie = &fault->idxd->irq_entries[i];
|
||||
irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT,
|
||||
&processed, fault->addr);
|
||||
if (processed)
|
||||
break;
|
||||
|
||||
irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT,
|
||||
&processed, fault->addr);
|
||||
if (processed)
|
||||
break;
|
||||
}
|
||||
|
||||
kfree(fault);
|
||||
}
|
||||
|
||||
static int idxd_device_schedule_fault_process(struct idxd_device *idxd,
|
||||
u64 fault_addr)
|
||||
{
|
||||
struct idxd_fault *fault;
|
||||
|
||||
fault = kmalloc(sizeof(*fault), GFP_ATOMIC);
|
||||
if (!fault)
|
||||
return -ENOMEM;
|
||||
|
||||
fault->addr = fault_addr;
|
||||
fault->idxd = idxd;
|
||||
INIT_WORK(&fault->work, idxd_device_fault_work);
|
||||
queue_work(idxd->wq, &fault->work);
|
||||
return 0;
|
||||
}
|
||||
|
||||
irqreturn_t idxd_irq_handler(int vec, void *data)
|
||||
{
|
||||
struct idxd_irq_entry *irq_entry = data;
|
||||
|
@ -125,6 +183,15 @@ irqreturn_t idxd_misc_thread(int vec, void *data)
|
|||
if (!err)
|
||||
goto out;
|
||||
|
||||
/*
|
||||
* This case should rarely happen and typically is due to software
|
||||
* programming error by the driver.
|
||||
*/
|
||||
if (idxd->sw_err.valid &&
|
||||
idxd->sw_err.desc_valid &&
|
||||
idxd->sw_err.fault_addr)
|
||||
idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr);
|
||||
|
||||
gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
|
||||
if (gensts.state == IDXD_DEVICE_STATE_HALT) {
|
||||
idxd->state = IDXD_DEV_HALTED;
|
||||
|
@ -152,57 +219,110 @@ irqreturn_t idxd_misc_thread(int vec, void *data)
|
|||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
static bool process_fault(struct idxd_desc *desc, u64 fault_addr)
|
||||
{
|
||||
/*
|
||||
* Completion address can be bad as well. Check fault address match for descriptor
|
||||
* and completion address.
|
||||
*/
|
||||
if ((u64)desc->hw == fault_addr ||
|
||||
(u64)desc->completion == fault_addr) {
|
||||
idxd_dma_complete_txd(desc, IDXD_COMPLETE_DEV_FAIL);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool complete_desc(struct idxd_desc *desc)
|
||||
{
|
||||
if (desc->completion->status) {
|
||||
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
|
||||
int *processed)
|
||||
enum irq_work_type wtype,
|
||||
int *processed, u64 data)
|
||||
{
|
||||
struct idxd_desc *desc, *t;
|
||||
struct llist_node *head;
|
||||
int queued = 0;
|
||||
bool completed = false;
|
||||
unsigned long flags;
|
||||
|
||||
*processed = 0;
|
||||
head = llist_del_all(&irq_entry->pending_llist);
|
||||
if (!head)
|
||||
return 0;
|
||||
goto out;
|
||||
|
||||
llist_for_each_entry_safe(desc, t, head, llnode) {
|
||||
if (desc->completion->status) {
|
||||
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
|
||||
if (wtype == IRQ_WORK_NORMAL)
|
||||
completed = complete_desc(desc);
|
||||
else if (wtype == IRQ_WORK_PROCESS_FAULT)
|
||||
completed = process_fault(desc, data);
|
||||
|
||||
if (completed) {
|
||||
idxd_free_desc(desc->wq, desc);
|
||||
(*processed)++;
|
||||
if (wtype == IRQ_WORK_PROCESS_FAULT)
|
||||
break;
|
||||
} else {
|
||||
list_add_tail(&desc->list, &irq_entry->work_list);
|
||||
spin_lock_irqsave(&irq_entry->list_lock, flags);
|
||||
list_add_tail(&desc->list,
|
||||
&irq_entry->work_list);
|
||||
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
|
||||
queued++;
|
||||
}
|
||||
}
|
||||
|
||||
out:
|
||||
return queued;
|
||||
}
|
||||
|
||||
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
|
||||
int *processed)
|
||||
enum irq_work_type wtype,
|
||||
int *processed, u64 data)
|
||||
{
|
||||
struct list_head *node, *next;
|
||||
int queued = 0;
|
||||
bool completed = false;
|
||||
unsigned long flags;
|
||||
|
||||
*processed = 0;
|
||||
spin_lock_irqsave(&irq_entry->list_lock, flags);
|
||||
if (list_empty(&irq_entry->work_list))
|
||||
return 0;
|
||||
goto out;
|
||||
|
||||
list_for_each_safe(node, next, &irq_entry->work_list) {
|
||||
struct idxd_desc *desc =
|
||||
container_of(node, struct idxd_desc, list);
|
||||
|
||||
if (desc->completion->status) {
|
||||
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
|
||||
if (wtype == IRQ_WORK_NORMAL)
|
||||
completed = complete_desc(desc);
|
||||
else if (wtype == IRQ_WORK_PROCESS_FAULT)
|
||||
completed = process_fault(desc, data);
|
||||
|
||||
if (completed) {
|
||||
spin_lock_irqsave(&irq_entry->list_lock, flags);
|
||||
list_del(&desc->list);
|
||||
/* process and callback */
|
||||
idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
|
||||
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
|
||||
idxd_free_desc(desc->wq, desc);
|
||||
(*processed)++;
|
||||
if (wtype == IRQ_WORK_PROCESS_FAULT)
|
||||
return queued;
|
||||
} else {
|
||||
queued++;
|
||||
}
|
||||
spin_lock_irqsave(&irq_entry->list_lock, flags);
|
||||
}
|
||||
|
||||
out:
|
||||
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
|
||||
return queued;
|
||||
}
|
||||
|
||||
|
@ -230,12 +350,14 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
|
|||
* 5. Repeat until no more descriptors.
|
||||
*/
|
||||
do {
|
||||
rc = irq_process_work_list(irq_entry, &processed);
|
||||
rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL,
|
||||
&processed, 0);
|
||||
total += processed;
|
||||
if (rc != 0)
|
||||
continue;
|
||||
|
||||
rc = irq_process_pending_llist(irq_entry, &processed);
|
||||
rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL,
|
||||
&processed, 0);
|
||||
total += processed;
|
||||
} while (rc != 0);
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@
|
|||
|
||||
/* PCI Config */
|
||||
#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25
|
||||
#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe
|
||||
|
||||
#define IDXD_MMIO_BAR 0
|
||||
#define IDXD_WQ_BAR 2
|
||||
|
@ -47,7 +48,7 @@ union wq_cap_reg {
|
|||
u64 rsvd:20;
|
||||
u64 shared_mode:1;
|
||||
u64 dedicated_mode:1;
|
||||
u64 rsvd2:1;
|
||||
u64 wq_ats_support:1;
|
||||
u64 priority:1;
|
||||
u64 occupancy:1;
|
||||
u64 occupancy_int:1;
|
||||
|
@ -102,6 +103,8 @@ union offsets_reg {
|
|||
u64 bits[2];
|
||||
} __packed;
|
||||
|
||||
#define IDXD_TABLE_MULT 0x100
|
||||
|
||||
#define IDXD_GENCFG_OFFSET 0x80
|
||||
union gencfg_reg {
|
||||
struct {
|
||||
|
@ -301,7 +304,8 @@ union wqcfg {
|
|||
/* bytes 8-11 */
|
||||
u32 mode:1; /* shared or dedicated */
|
||||
u32 bof:1; /* block on fault */
|
||||
u32 rsvd2:2;
|
||||
u32 wq_ats_disable:1;
|
||||
u32 rsvd2:1;
|
||||
u32 priority:4;
|
||||
u32 pasid:20;
|
||||
u32 pasid_en:1;
|
||||
|
@ -336,6 +340,8 @@ union wqcfg {
|
|||
u32 bits[8];
|
||||
} __packed;
|
||||
|
||||
#define WQCFG_PASID_IDX 2
|
||||
|
||||
/*
|
||||
* This macro calculates the offset into the WQCFG register
|
||||
* idxd - struct idxd *
|
||||
|
@ -354,4 +360,22 @@ union wqcfg {
|
|||
|
||||
#define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32))
|
||||
|
||||
#define GRPCFG_SIZE 64
|
||||
#define GRPWQCFG_STRIDES 4
|
||||
|
||||
/*
|
||||
* This macro calculates the offset into the GRPCFG register
|
||||
* idxd - struct idxd *
|
||||
* n - wq id
|
||||
* ofs - the index of the 32b dword for the config register
|
||||
*
|
||||
* The WQCFG register block is divided into groups per each wq. The n index
|
||||
* allows us to move to the register group that's for that particular wq.
|
||||
* Each register is 32bits. The ofs gives us the number of register to access.
|
||||
*/
|
||||
#define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\
|
||||
(n) * GRPCFG_SIZE + sizeof(u64) * (ofs))
|
||||
#define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32)
|
||||
#define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40)
|
||||
|
||||
#endif
|
||||
|
|
|
@ -11,11 +11,22 @@
|
|||
static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
|
||||
{
|
||||
struct idxd_desc *desc;
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
|
||||
desc = wq->descs[idx];
|
||||
memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
|
||||
memset(desc->completion, 0, sizeof(struct dsa_completion_record));
|
||||
memset(desc->completion, 0, idxd->compl_size);
|
||||
desc->cpu = cpu;
|
||||
|
||||
if (device_pasid_enabled(idxd))
|
||||
desc->hw->pasid = idxd->pasid;
|
||||
|
||||
/*
|
||||
* Descriptor completion vectors are 1-8 for MSIX. We will round
|
||||
* robin through the 8 vectors.
|
||||
*/
|
||||
wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
|
||||
desc->hw->int_handle = wq->vec_ptr;
|
||||
return desc;
|
||||
}
|
||||
|
||||
|
@ -70,18 +81,32 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
|
|||
struct idxd_device *idxd = wq->idxd;
|
||||
int vec = desc->hw->int_handle;
|
||||
void __iomem *portal;
|
||||
int rc;
|
||||
|
||||
if (idxd->state != IDXD_DEV_ENABLED)
|
||||
return -EIO;
|
||||
|
||||
portal = wq->dportal;
|
||||
portal = wq->portal;
|
||||
|
||||
/*
|
||||
* The wmb() flushes writes to coherent DMA data before possibly
|
||||
* triggering a DMA read. The wmb() is necessary even on UP because
|
||||
* the recipient is a device.
|
||||
* The wmb() flushes writes to coherent DMA data before
|
||||
* possibly triggering a DMA read. The wmb() is necessary
|
||||
* even on UP because the recipient is a device.
|
||||
*/
|
||||
wmb();
|
||||
iosubmit_cmds512(portal, desc->hw, 1);
|
||||
if (wq_dedicated(wq)) {
|
||||
iosubmit_cmds512(portal, desc->hw, 1);
|
||||
} else {
|
||||
/*
|
||||
* It's not likely that we would receive queue full rejection
|
||||
* since the descriptor allocation gates at wq size. If we
|
||||
* receive a -EAGAIN, that means something went wrong such as the
|
||||
* device is not accepting descriptor at all.
|
||||
*/
|
||||
rc = enqcmds(portal, desc->hw);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
* Pending the descriptor to the lockless list for the irq_entry
|
||||
|
|
|
@ -41,14 +41,24 @@ static struct device_type dsa_device_type = {
|
|||
.release = idxd_conf_device_release,
|
||||
};
|
||||
|
||||
static struct device_type iax_device_type = {
|
||||
.name = "iax",
|
||||
.release = idxd_conf_device_release,
|
||||
};
|
||||
|
||||
static inline bool is_dsa_dev(struct device *dev)
|
||||
{
|
||||
return dev ? dev->type == &dsa_device_type : false;
|
||||
}
|
||||
|
||||
static inline bool is_iax_dev(struct device *dev)
|
||||
{
|
||||
return dev ? dev->type == &iax_device_type : false;
|
||||
}
|
||||
|
||||
static inline bool is_idxd_dev(struct device *dev)
|
||||
{
|
||||
return is_dsa_dev(dev);
|
||||
return is_dsa_dev(dev) || is_iax_dev(dev);
|
||||
}
|
||||
|
||||
static inline bool is_idxd_wq_dev(struct device *dev)
|
||||
|
@ -175,6 +185,30 @@ static int idxd_config_bus_probe(struct device *dev)
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Shared WQ checks */
|
||||
if (wq_shared(wq)) {
|
||||
if (!device_swq_supported(idxd)) {
|
||||
dev_warn(dev,
|
||||
"PASID not enabled and shared WQ.\n");
|
||||
mutex_unlock(&wq->wq_lock);
|
||||
return -ENXIO;
|
||||
}
|
||||
/*
|
||||
* Shared wq with the threshold set to 0 means the user
|
||||
* did not set the threshold or transitioned from a
|
||||
* dedicated wq but did not set threshold. A value
|
||||
* of 0 would effectively disable the shared wq. The
|
||||
* driver does not allow a value of 0 to be set for
|
||||
* threshold via sysfs.
|
||||
*/
|
||||
if (wq->threshold == 0) {
|
||||
dev_warn(dev,
|
||||
"Shared WQ and threshold 0.\n");
|
||||
mutex_unlock(&wq->wq_lock);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
rc = idxd_wq_alloc_resources(wq);
|
||||
if (rc < 0) {
|
||||
mutex_unlock(&wq->wq_lock);
|
||||
|
@ -335,8 +369,17 @@ struct bus_type dsa_bus_type = {
|
|||
.shutdown = idxd_config_bus_shutdown,
|
||||
};
|
||||
|
||||
struct bus_type iax_bus_type = {
|
||||
.name = "iax",
|
||||
.match = idxd_config_bus_match,
|
||||
.probe = idxd_config_bus_probe,
|
||||
.remove = idxd_config_bus_remove,
|
||||
.shutdown = idxd_config_bus_shutdown,
|
||||
};
|
||||
|
||||
static struct bus_type *idxd_bus_types[] = {
|
||||
&dsa_bus_type
|
||||
&dsa_bus_type,
|
||||
&iax_bus_type
|
||||
};
|
||||
|
||||
static struct idxd_device_driver dsa_drv = {
|
||||
|
@ -348,8 +391,18 @@ static struct idxd_device_driver dsa_drv = {
|
|||
},
|
||||
};
|
||||
|
||||
static struct idxd_device_driver iax_drv = {
|
||||
.drv = {
|
||||
.name = "iax",
|
||||
.bus = &iax_bus_type,
|
||||
.owner = THIS_MODULE,
|
||||
.mod_name = KBUILD_MODNAME,
|
||||
},
|
||||
};
|
||||
|
||||
static struct idxd_device_driver *idxd_drvs[] = {
|
||||
&dsa_drv
|
||||
&dsa_drv,
|
||||
&iax_drv
|
||||
};
|
||||
|
||||
struct bus_type *idxd_get_bus_type(struct idxd_device *idxd)
|
||||
|
@ -361,6 +414,8 @@ static struct device_type *idxd_get_device_type(struct idxd_device *idxd)
|
|||
{
|
||||
if (idxd->type == IDXD_TYPE_DSA)
|
||||
return &dsa_device_type;
|
||||
else if (idxd->type == IDXD_TYPE_IAX)
|
||||
return &iax_device_type;
|
||||
else
|
||||
return NULL;
|
||||
}
|
||||
|
@ -501,6 +556,9 @@ static ssize_t group_tokens_reserved_store(struct device *dev,
|
|||
if (rc < 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (idxd->type == IDXD_TYPE_IAX)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
|
||||
return -EPERM;
|
||||
|
||||
|
@ -546,6 +604,9 @@ static ssize_t group_tokens_allowed_store(struct device *dev,
|
|||
if (rc < 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (idxd->type == IDXD_TYPE_IAX)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
|
||||
return -EPERM;
|
||||
|
||||
|
@ -588,6 +649,9 @@ static ssize_t group_use_token_limit_store(struct device *dev,
|
|||
if (rc < 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (idxd->type == IDXD_TYPE_IAX)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
|
||||
return -EPERM;
|
||||
|
||||
|
@ -875,6 +939,8 @@ static ssize_t wq_mode_store(struct device *dev,
|
|||
if (sysfs_streq(buf, "dedicated")) {
|
||||
set_bit(WQ_FLAG_DEDICATED, &wq->flags);
|
||||
wq->threshold = 0;
|
||||
} else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) {
|
||||
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
|
||||
} else {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
@ -973,6 +1039,87 @@ static ssize_t wq_priority_store(struct device *dev,
|
|||
static struct device_attribute dev_attr_wq_priority =
|
||||
__ATTR(priority, 0644, wq_priority_show, wq_priority_store);
|
||||
|
||||
static ssize_t wq_block_on_fault_show(struct device *dev,
|
||||
struct device_attribute *attr, char *buf)
|
||||
{
|
||||
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
|
||||
|
||||
return sprintf(buf, "%u\n",
|
||||
test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags));
|
||||
}
|
||||
|
||||
static ssize_t wq_block_on_fault_store(struct device *dev,
|
||||
struct device_attribute *attr,
|
||||
const char *buf, size_t count)
|
||||
{
|
||||
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
bool bof;
|
||||
int rc;
|
||||
|
||||
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
|
||||
return -EPERM;
|
||||
|
||||
if (wq->state != IDXD_WQ_DISABLED)
|
||||
return -ENXIO;
|
||||
|
||||
rc = kstrtobool(buf, &bof);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
if (bof)
|
||||
set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
|
||||
else
|
||||
clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static struct device_attribute dev_attr_wq_block_on_fault =
|
||||
__ATTR(block_on_fault, 0644, wq_block_on_fault_show,
|
||||
wq_block_on_fault_store);
|
||||
|
||||
static ssize_t wq_threshold_show(struct device *dev,
|
||||
struct device_attribute *attr, char *buf)
|
||||
{
|
||||
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
|
||||
|
||||
return sprintf(buf, "%u\n", wq->threshold);
|
||||
}
|
||||
|
||||
static ssize_t wq_threshold_store(struct device *dev,
|
||||
struct device_attribute *attr,
|
||||
const char *buf, size_t count)
|
||||
{
|
||||
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
unsigned int val;
|
||||
int rc;
|
||||
|
||||
rc = kstrtouint(buf, 0, &val);
|
||||
if (rc < 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (val > wq->size || val <= 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
|
||||
return -EPERM;
|
||||
|
||||
if (wq->state != IDXD_WQ_DISABLED)
|
||||
return -ENXIO;
|
||||
|
||||
if (test_bit(WQ_FLAG_DEDICATED, &wq->flags))
|
||||
return -EINVAL;
|
||||
|
||||
wq->threshold = val;
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static struct device_attribute dev_attr_wq_threshold =
|
||||
__ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store);
|
||||
|
||||
static ssize_t wq_type_show(struct device *dev,
|
||||
struct device_attribute *attr, char *buf)
|
||||
{
|
||||
|
@ -1044,6 +1191,13 @@ static ssize_t wq_name_store(struct device *dev,
|
|||
if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* This is temporarily placed here until we have SVM support for
|
||||
* dmaengine.
|
||||
*/
|
||||
if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd))
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
memset(wq->name, 0, WQ_NAME_SIZE + 1);
|
||||
strncpy(wq->name, buf, WQ_NAME_SIZE);
|
||||
strreplace(wq->name, '\n', '\0');
|
||||
|
@ -1147,6 +1301,39 @@ static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribu
|
|||
static struct device_attribute dev_attr_wq_max_batch_size =
|
||||
__ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store);
|
||||
|
||||
static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
|
||||
|
||||
return sprintf(buf, "%u\n", wq->ats_dis);
|
||||
}
|
||||
|
||||
static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr,
|
||||
const char *buf, size_t count)
|
||||
{
|
||||
struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
|
||||
struct idxd_device *idxd = wq->idxd;
|
||||
bool ats_dis;
|
||||
int rc;
|
||||
|
||||
if (wq->state != IDXD_WQ_DISABLED)
|
||||
return -EPERM;
|
||||
|
||||
if (!idxd->hw.wq_cap.wq_ats_support)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
rc = kstrtobool(buf, &ats_dis);
|
||||
if (rc < 0)
|
||||
return rc;
|
||||
|
||||
wq->ats_dis = ats_dis;
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
static struct device_attribute dev_attr_wq_ats_disable =
|
||||
__ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store);
|
||||
|
||||
static struct attribute *idxd_wq_attributes[] = {
|
||||
&dev_attr_wq_clients.attr,
|
||||
&dev_attr_wq_state.attr,
|
||||
|
@ -1154,11 +1341,14 @@ static struct attribute *idxd_wq_attributes[] = {
|
|||
&dev_attr_wq_mode.attr,
|
||||
&dev_attr_wq_size.attr,
|
||||
&dev_attr_wq_priority.attr,
|
||||
&dev_attr_wq_block_on_fault.attr,
|
||||
&dev_attr_wq_threshold.attr,
|
||||
&dev_attr_wq_type.attr,
|
||||
&dev_attr_wq_name.attr,
|
||||
&dev_attr_wq_cdev_minor.attr,
|
||||
&dev_attr_wq_max_transfer_size.attr,
|
||||
&dev_attr_wq_max_batch_size.attr,
|
||||
&dev_attr_wq_ats_disable.attr,
|
||||
NULL,
|
||||
};
|
||||
|
||||
|
@ -1305,6 +1495,16 @@ static ssize_t clients_show(struct device *dev,
|
|||
}
|
||||
static DEVICE_ATTR_RO(clients);
|
||||
|
||||
static ssize_t pasid_enabled_show(struct device *dev,
|
||||
struct device_attribute *attr, char *buf)
|
||||
{
|
||||
struct idxd_device *idxd =
|
||||
container_of(dev, struct idxd_device, conf_dev);
|
||||
|
||||
return sprintf(buf, "%u\n", device_pasid_enabled(idxd));
|
||||
}
|
||||
static DEVICE_ATTR_RO(pasid_enabled);
|
||||
|
||||
static ssize_t state_show(struct device *dev,
|
||||
struct device_attribute *attr, char *buf)
|
||||
{
|
||||
|
@ -1424,6 +1624,7 @@ static struct attribute *idxd_device_attributes[] = {
|
|||
&dev_attr_gen_cap.attr,
|
||||
&dev_attr_configurable.attr,
|
||||
&dev_attr_clients.attr,
|
||||
&dev_attr_pasid_enabled.attr,
|
||||
&dev_attr_state.attr,
|
||||
&dev_attr_errors.attr,
|
||||
&dev_attr_max_tokens.attr,
|
||||
|
|
|
@ -191,32 +191,13 @@ struct imxdma_filter_data {
|
|||
int request;
|
||||
};
|
||||
|
||||
static const struct platform_device_id imx_dma_devtype[] = {
|
||||
{
|
||||
.name = "imx1-dma",
|
||||
.driver_data = IMX1_DMA,
|
||||
}, {
|
||||
.name = "imx21-dma",
|
||||
.driver_data = IMX21_DMA,
|
||||
}, {
|
||||
.name = "imx27-dma",
|
||||
.driver_data = IMX27_DMA,
|
||||
}, {
|
||||
/* sentinel */
|
||||
}
|
||||
};
|
||||
MODULE_DEVICE_TABLE(platform, imx_dma_devtype);
|
||||
|
||||
static const struct of_device_id imx_dma_of_dev_id[] = {
|
||||
{
|
||||
.compatible = "fsl,imx1-dma",
|
||||
.data = &imx_dma_devtype[IMX1_DMA],
|
||||
.compatible = "fsl,imx1-dma", .data = (const void *)IMX1_DMA,
|
||||
}, {
|
||||
.compatible = "fsl,imx21-dma",
|
||||
.data = &imx_dma_devtype[IMX21_DMA],
|
||||
.compatible = "fsl,imx21-dma", .data = (const void *)IMX21_DMA,
|
||||
}, {
|
||||
.compatible = "fsl,imx27-dma",
|
||||
.data = &imx_dma_devtype[IMX27_DMA],
|
||||
.compatible = "fsl,imx27-dma", .data = (const void *)IMX27_DMA,
|
||||
}, {
|
||||
/* sentinel */
|
||||
}
|
||||
|
@ -1056,20 +1037,15 @@ static int __init imxdma_probe(struct platform_device *pdev)
|
|||
{
|
||||
struct imxdma_engine *imxdma;
|
||||
struct resource *res;
|
||||
const struct of_device_id *of_id;
|
||||
int ret, i;
|
||||
int irq, irq_err;
|
||||
|
||||
of_id = of_match_device(imx_dma_of_dev_id, &pdev->dev);
|
||||
if (of_id)
|
||||
pdev->id_entry = of_id->data;
|
||||
|
||||
imxdma = devm_kzalloc(&pdev->dev, sizeof(*imxdma), GFP_KERNEL);
|
||||
if (!imxdma)
|
||||
return -ENOMEM;
|
||||
|
||||
imxdma->dev = &pdev->dev;
|
||||
imxdma->devtype = pdev->id_entry->driver_data;
|
||||
imxdma->devtype = (enum imx_dma_type)of_device_get_match_data(&pdev->dev);
|
||||
|
||||
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
||||
imxdma->base = devm_ioremap_resource(&pdev->dev, res);
|
||||
|
@ -1263,7 +1239,6 @@ static struct platform_driver imxdma_driver = {
|
|||
.name = "imx-dma",
|
||||
.of_match_table = imx_dma_of_dev_id,
|
||||
},
|
||||
.id_table = imx_dma_devtype,
|
||||
.remove = imxdma_remove,
|
||||
};
|
||||
|
||||
|
|
|
@ -566,37 +566,6 @@ static struct sdma_driver_data sdma_imx8mq = {
|
|||
.check_ratio = 1,
|
||||
};
|
||||
|
||||
static const struct platform_device_id sdma_devtypes[] = {
|
||||
{
|
||||
.name = "imx25-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx25,
|
||||
}, {
|
||||
.name = "imx31-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx31,
|
||||
}, {
|
||||
.name = "imx35-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx35,
|
||||
}, {
|
||||
.name = "imx51-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx51,
|
||||
}, {
|
||||
.name = "imx53-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx53,
|
||||
}, {
|
||||
.name = "imx6q-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx6q,
|
||||
}, {
|
||||
.name = "imx7d-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx7d,
|
||||
}, {
|
||||
.name = "imx8mq-sdma",
|
||||
.driver_data = (unsigned long)&sdma_imx8mq,
|
||||
}, {
|
||||
/* sentinel */
|
||||
}
|
||||
};
|
||||
MODULE_DEVICE_TABLE(platform, sdma_devtypes);
|
||||
|
||||
static const struct of_device_id sdma_dt_ids[] = {
|
||||
{ .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
|
||||
{ .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
|
||||
|
@ -1998,11 +1967,7 @@ static int sdma_probe(struct platform_device *pdev)
|
|||
s32 *saddr_arr;
|
||||
const struct sdma_driver_data *drvdata = NULL;
|
||||
|
||||
if (of_id)
|
||||
drvdata = of_id->data;
|
||||
else if (pdev->id_entry)
|
||||
drvdata = (void *)pdev->id_entry->driver_data;
|
||||
|
||||
drvdata = of_id->data;
|
||||
if (!drvdata) {
|
||||
dev_err(&pdev->dev, "unable to find driver data\n");
|
||||
return -EINVAL;
|
||||
|
@ -2211,7 +2176,6 @@ static struct platform_driver sdma_driver = {
|
|||
.name = "imx-sdma",
|
||||
.of_match_table = sdma_dt_ids,
|
||||
},
|
||||
.id_table = sdma_devtypes,
|
||||
.remove = sdma_remove,
|
||||
.probe = sdma_probe,
|
||||
};
|
||||
|
|
|
@ -1160,14 +1160,13 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
|
|||
struct idmac_tx_desc *desc, *descnew;
|
||||
bool done = false;
|
||||
u32 ready0, ready1, curbuf, err;
|
||||
unsigned long flags;
|
||||
struct dmaengine_desc_callback cb;
|
||||
|
||||
/* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
|
||||
|
||||
dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);
|
||||
|
||||
spin_lock_irqsave(&ipu_data.lock, flags);
|
||||
spin_lock(&ipu_data.lock);
|
||||
|
||||
ready0 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
|
||||
ready1 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
|
||||
|
@ -1176,7 +1175,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
|
|||
|
||||
if (err & (1 << chan_id)) {
|
||||
idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4);
|
||||
spin_unlock_irqrestore(&ipu_data.lock, flags);
|
||||
spin_unlock(&ipu_data.lock);
|
||||
/*
|
||||
* Doing this
|
||||
* ichan->sg[0] = ichan->sg[1] = NULL;
|
||||
|
@ -1188,7 +1187,7 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
|
|||
chan_id, ready0, ready1, curbuf);
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
spin_unlock_irqrestore(&ipu_data.lock, flags);
|
||||
spin_unlock(&ipu_data.lock);
|
||||
|
||||
/* Other interrupts do not interfere with this channel */
|
||||
spin_lock(&ichan->lock);
|
||||
|
@ -1251,9 +1250,9 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id)
|
|||
if (unlikely(sgnew)) {
|
||||
ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);
|
||||
} else {
|
||||
spin_lock_irqsave(&ipu_data.lock, flags);
|
||||
spin_lock(&ipu_data.lock);
|
||||
ipu_ic_disable_task(&ipu_data, chan_id);
|
||||
spin_unlock_irqrestore(&ipu_data.lock, flags);
|
||||
spin_unlock(&ipu_data.lock);
|
||||
ichan->status = IPU_CHANNEL_READY;
|
||||
/* Continue to check for complete descriptor */
|
||||
}
|
||||
|
|
|
@ -223,24 +223,23 @@ static irqreturn_t k3_dma_int_handler(int irq, void *dev_id)
|
|||
i = __ffs(stat);
|
||||
stat &= ~BIT(i);
|
||||
if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) {
|
||||
unsigned long flags;
|
||||
|
||||
p = &d->phy[i];
|
||||
c = p->vchan;
|
||||
if (c && (tc1 & BIT(i))) {
|
||||
spin_lock_irqsave(&c->vc.lock, flags);
|
||||
spin_lock(&c->vc.lock);
|
||||
if (p->ds_run != NULL) {
|
||||
vchan_cookie_complete(&p->ds_run->vd);
|
||||
p->ds_done = p->ds_run;
|
||||
p->ds_run = NULL;
|
||||
}
|
||||
spin_unlock_irqrestore(&c->vc.lock, flags);
|
||||
spin_unlock(&c->vc.lock);
|
||||
}
|
||||
if (c && (tc2 & BIT(i))) {
|
||||
spin_lock_irqsave(&c->vc.lock, flags);
|
||||
spin_lock(&c->vc.lock);
|
||||
if (p->ds_run != NULL)
|
||||
vchan_cyclic_callback(&p->ds_run->vd);
|
||||
spin_unlock_irqrestore(&c->vc.lock, flags);
|
||||
spin_unlock(&c->vc.lock);
|
||||
}
|
||||
irq_chan |= BIT(i);
|
||||
}
|
||||
|
|
|
@ -160,10 +160,9 @@ static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id)
|
|||
{
|
||||
struct milbeaut_xdmac_chan *mc = dev_id;
|
||||
struct milbeaut_xdmac_desc *md;
|
||||
unsigned long flags;
|
||||
u32 val;
|
||||
|
||||
spin_lock_irqsave(&mc->vc.lock, flags);
|
||||
spin_lock(&mc->vc.lock);
|
||||
|
||||
/* Ack and Stop */
|
||||
val = FIELD_PREP(M10V_XDDSD_IS_MASK, 0x0);
|
||||
|
@ -177,7 +176,7 @@ static irqreturn_t milbeaut_xdmac_interrupt(int irq, void *dev_id)
|
|||
|
||||
milbeaut_xdmac_start(mc);
|
||||
out:
|
||||
spin_unlock_irqrestore(&mc->vc.lock, flags);
|
||||
spin_unlock(&mc->vc.lock);
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
|
|
|
@ -524,7 +524,6 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid)
|
|||
struct moxart_dmadev *mc = devid;
|
||||
struct moxart_chan *ch = &mc->slave_chans[0];
|
||||
unsigned int i;
|
||||
unsigned long flags;
|
||||
u32 ctrl;
|
||||
|
||||
dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);
|
||||
|
@ -541,14 +540,14 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid)
|
|||
if (ctrl & APB_DMA_FIN_INT_STS) {
|
||||
ctrl &= ~APB_DMA_FIN_INT_STS;
|
||||
if (ch->desc) {
|
||||
spin_lock_irqsave(&ch->vc.lock, flags);
|
||||
spin_lock(&ch->vc.lock);
|
||||
if (++ch->sgidx < ch->desc->sglen) {
|
||||
moxart_dma_start_sg(ch, ch->sgidx);
|
||||
} else {
|
||||
vchan_cookie_complete(&ch->desc->vd);
|
||||
moxart_dma_start_desc(&ch->vc.chan);
|
||||
}
|
||||
spin_unlock_irqrestore(&ch->vc.lock, flags);
|
||||
spin_unlock(&ch->vc.lock);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1455,7 +1455,7 @@ static struct platform_driver mv_xor_driver = {
|
|||
.resume = mv_xor_resume,
|
||||
.driver = {
|
||||
.name = MV_XOR_NAME,
|
||||
.of_match_table = of_match_ptr(mv_xor_dt_ids),
|
||||
.of_match_table = mv_xor_dt_ids,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
|
@ -771,8 +771,10 @@ static int mv_xor_v2_probe(struct platform_device *pdev)
|
|||
goto disable_clk;
|
||||
|
||||
msi_desc = first_msi_entry(&pdev->dev);
|
||||
if (!msi_desc)
|
||||
if (!msi_desc) {
|
||||
ret = -ENODEV;
|
||||
goto free_msi_irqs;
|
||||
}
|
||||
xor_dev->msi_desc = msi_desc;
|
||||
|
||||
ret = devm_request_irq(&pdev->dev, msi_desc->irq,
|
||||
|
|
|
@ -167,29 +167,11 @@ static struct mxs_dma_type mxs_dma_types[] = {
|
|||
}
|
||||
};
|
||||
|
||||
static const struct platform_device_id mxs_dma_ids[] = {
|
||||
{
|
||||
.name = "imx23-dma-apbh",
|
||||
.driver_data = (kernel_ulong_t) &mxs_dma_types[0],
|
||||
}, {
|
||||
.name = "imx23-dma-apbx",
|
||||
.driver_data = (kernel_ulong_t) &mxs_dma_types[1],
|
||||
}, {
|
||||
.name = "imx28-dma-apbh",
|
||||
.driver_data = (kernel_ulong_t) &mxs_dma_types[2],
|
||||
}, {
|
||||
.name = "imx28-dma-apbx",
|
||||
.driver_data = (kernel_ulong_t) &mxs_dma_types[3],
|
||||
}, {
|
||||
/* end of list */
|
||||
}
|
||||
};
|
||||
|
||||
static const struct of_device_id mxs_dma_dt_ids[] = {
|
||||
{ .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], },
|
||||
{ .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], },
|
||||
{ .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], },
|
||||
{ .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], },
|
||||
{ .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_types[0], },
|
||||
{ .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_types[1], },
|
||||
{ .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_types[2], },
|
||||
{ .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_types[3], },
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids);
|
||||
|
@ -762,8 +744,6 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec,
|
|||
static int __init mxs_dma_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct device_node *np = pdev->dev.of_node;
|
||||
const struct platform_device_id *id_entry;
|
||||
const struct of_device_id *of_id;
|
||||
const struct mxs_dma_type *dma_type;
|
||||
struct mxs_dma_engine *mxs_dma;
|
||||
struct resource *iores;
|
||||
|
@ -779,13 +759,7 @@ static int __init mxs_dma_probe(struct platform_device *pdev)
|
|||
return ret;
|
||||
}
|
||||
|
||||
of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev);
|
||||
if (of_id)
|
||||
id_entry = of_id->data;
|
||||
else
|
||||
id_entry = platform_get_device_id(pdev);
|
||||
|
||||
dma_type = (struct mxs_dma_type *)id_entry->driver_data;
|
||||
dma_type = (struct mxs_dma_type *)of_device_get_match_data(&pdev->dev);
|
||||
mxs_dma->type = dma_type->type;
|
||||
mxs_dma->dev_id = dma_type->id;
|
||||
|
||||
|
@ -865,7 +839,6 @@ static struct platform_driver mxs_dma_driver = {
|
|||
.name = "mxs-dma",
|
||||
.of_match_table = mxs_dma_dt_ids,
|
||||
},
|
||||
.id_table = mxs_dma_ids,
|
||||
};
|
||||
|
||||
static int __init mxs_dma_module_init(void)
|
||||
|
|
|
@ -75,8 +75,18 @@ static struct dma_chan *of_dma_router_xlate(struct of_phandle_args *dma_spec,
|
|||
ofdma->dma_router->route_free(ofdma->dma_router->dev,
|
||||
route_data);
|
||||
} else {
|
||||
int ret = 0;
|
||||
|
||||
chan->router = ofdma->dma_router;
|
||||
chan->route_data = route_data;
|
||||
|
||||
if (chan->device->device_router_config)
|
||||
ret = chan->device->device_router_config(chan);
|
||||
|
||||
if (ret) {
|
||||
dma_release_channel(chan);
|
||||
chan = ERR_PTR(ret);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -1527,8 +1527,6 @@ static int pl330_submit_req(struct pl330_thread *thrd,
|
|||
|
||||
/* First dry run to check if req is acceptable */
|
||||
ret = _setup_req(pl330, 1, thrd, idx, &xs);
|
||||
if (ret < 0)
|
||||
goto xfer_exit;
|
||||
|
||||
if (ret > pl330->mcbufsz / 2) {
|
||||
dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",
|
||||
|
|
|
@ -69,7 +69,7 @@ struct ppc_dma_chan_ref {
|
|||
};
|
||||
|
||||
/* The list of channels exported by ppc440spe ADMA */
|
||||
struct list_head
|
||||
static struct list_head
|
||||
ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
|
||||
|
||||
/* This flag is set when want to refetch the xor chain in the interrupt
|
||||
|
@ -559,7 +559,6 @@ static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
|
|||
int sg_index, unsigned char mult_value)
|
||||
{
|
||||
struct dma_cdb *dma_hw_desc;
|
||||
struct xor_cb *xor_hw_desc;
|
||||
u32 *psgu;
|
||||
|
||||
switch (chan->device->id) {
|
||||
|
@ -590,7 +589,6 @@ static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
|
|||
*psgu |= cpu_to_le32(mult_value << mult_index);
|
||||
break;
|
||||
case PPC440SPE_XOR_ID:
|
||||
xor_hw_desc = desc->hw_desc;
|
||||
break;
|
||||
default:
|
||||
BUG();
|
||||
|
|
|
@ -606,7 +606,6 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
|
|||
struct pxad_chan *chan = phy->vchan;
|
||||
struct virt_dma_desc *vd, *tmp;
|
||||
unsigned int dcsr;
|
||||
unsigned long flags;
|
||||
bool vd_completed;
|
||||
dma_cookie_t last_started = 0;
|
||||
|
||||
|
@ -616,7 +615,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
|
|||
if (dcsr & PXA_DCSR_RUN)
|
||||
return IRQ_NONE;
|
||||
|
||||
spin_lock_irqsave(&chan->vc.lock, flags);
|
||||
spin_lock(&chan->vc.lock);
|
||||
list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
|
||||
vd_completed = is_desc_completed(vd);
|
||||
dev_dbg(&chan->vc.chan.dev->device,
|
||||
|
@ -658,7 +657,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
|
|||
pxad_launch_chan(chan, to_pxad_sw_desc(vd));
|
||||
}
|
||||
}
|
||||
spin_unlock_irqrestore(&chan->vc.lock, flags);
|
||||
spin_unlock(&chan->vc.lock);
|
||||
wake_up(&chan->wq_state);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
|
|
|
@ -1,4 +1,15 @@
|
|||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
config QCOM_ADM
|
||||
tristate "Qualcomm ADM support"
|
||||
depends on (ARCH_QCOM || COMPILE_TEST) && !PHYS_ADDR_T_64BIT
|
||||
select DMA_ENGINE
|
||||
select DMA_VIRTUAL_CHANNELS
|
||||
help
|
||||
Enable support for the Qualcomm Application Data Mover (ADM) DMA
|
||||
controller, as present on MSM8x60, APQ8064, and IPQ8064 devices.
|
||||
This controller provides DMA capabilities for both general purpose
|
||||
and on-chip peripheral devices.
|
||||
|
||||
config QCOM_BAM_DMA
|
||||
tristate "QCOM BAM DMA support"
|
||||
depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM)
|
||||
|
@ -8,6 +19,18 @@ config QCOM_BAM_DMA
|
|||
Enable support for the QCOM BAM DMA controller. This controller
|
||||
provides DMA capabilities for a variety of on-chip devices.
|
||||
|
||||
config QCOM_GPI_DMA
|
||||
tristate "Qualcomm Technologies GPI DMA support"
|
||||
depends on ARCH_QCOM
|
||||
select DMA_ENGINE
|
||||
select DMA_VIRTUAL_CHANNELS
|
||||
help
|
||||
Enable support for the QCOM GPI DMA controller. This controller
|
||||
provides DMA capabilities for a variety of peripheral buses such
|
||||
as I2C, UART, and SPI. By using GPI dmaengine driver, bus drivers
|
||||
can use a standardize interface that is protocol independent to
|
||||
transfer data between DDR and peripheral.
|
||||
|
||||
config QCOM_HIDMA_MGMT
|
||||
tristate "Qualcomm Technologies HIDMA Management support"
|
||||
select DMA_ENGINE
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
# SPDX-License-Identifier: GPL-2.0
|
||||
obj-$(CONFIG_QCOM_ADM) += qcom_adm.o
|
||||
obj-$(CONFIG_QCOM_BAM_DMA) += bam_dma.o
|
||||
obj-$(CONFIG_QCOM_GPI_DMA) += gpi.o
|
||||
obj-$(CONFIG_QCOM_HIDMA_MGMT) += hdma_mgmt.o
|
||||
hdma_mgmt-objs := hidma_mgmt.o hidma_mgmt_sys.o
|
||||
obj-$(CONFIG_QCOM_HIDMA) += hdma.o
|
||||
|
|
|
@ -875,7 +875,7 @@ static irqreturn_t bam_dma_irq(int irq, void *data)
|
|||
|
||||
ret = bam_pm_runtime_get_sync(bdev->dev);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
return IRQ_NONE;
|
||||
|
||||
if (srcs & BAM_IRQ) {
|
||||
clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));
|
||||
|
|
2303
drivers/dma/qcom/gpi.c
Normal file
2303
drivers/dma/qcom/gpi.c
Normal file
File diff suppressed because it is too large
Load diff
905
drivers/dma/qcom/qcom_adm.c
Normal file
905
drivers/dma/qcom/qcom_adm.c
Normal file
|
@ -0,0 +1,905 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/clk.h>
|
||||
#include <linux/delay.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/dmaengine.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/io.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/of.h>
|
||||
#include <linux/of_address.h>
|
||||
#include <linux/of_irq.h>
|
||||
#include <linux/of_dma.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/reset.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
#include "../dmaengine.h"
|
||||
#include "../virt-dma.h"
|
||||
|
||||
/* ADM registers - calculated from channel number and security domain */
|
||||
#define ADM_CHAN_MULTI 0x4
|
||||
#define ADM_CI_MULTI 0x4
|
||||
#define ADM_CRCI_MULTI 0x4
|
||||
#define ADM_EE_MULTI 0x800
|
||||
#define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan))
|
||||
#define ADM_EE_OFFS(ee) (ADM_EE_MULTI * (ee))
|
||||
#define ADM_CHAN_EE_OFFS(chan, ee) (ADM_CHAN_OFFS(chan) + ADM_EE_OFFS(ee))
|
||||
#define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan))
|
||||
#define ADM_CI_OFFS(ci) (ADM_CHAN_OFF(ci))
|
||||
#define ADM_CH_CMD_PTR(chan, ee) (ADM_CHAN_EE_OFFS(chan, ee))
|
||||
#define ADM_CH_RSLT(chan, ee) (0x40 + ADM_CHAN_EE_OFFS(chan, ee))
|
||||
#define ADM_CH_FLUSH_STATE0(chan, ee) (0x80 + ADM_CHAN_EE_OFFS(chan, ee))
|
||||
#define ADM_CH_STATUS_SD(chan, ee) (0x200 + ADM_CHAN_EE_OFFS(chan, ee))
|
||||
#define ADM_CH_CONF(chan) (0x240 + ADM_CHAN_OFFS(chan))
|
||||
#define ADM_CH_RSLT_CONF(chan, ee) (0x300 + ADM_CHAN_EE_OFFS(chan, ee))
|
||||
#define ADM_SEC_DOMAIN_IRQ_STATUS(ee) (0x380 + ADM_EE_OFFS(ee))
|
||||
#define ADM_CI_CONF(ci) (0x390 + (ci) * ADM_CI_MULTI)
|
||||
#define ADM_GP_CTL 0x3d8
|
||||
#define ADM_CRCI_CTL(crci, ee) (0x400 + (crci) * ADM_CRCI_MULTI + \
|
||||
ADM_EE_OFFS(ee))
|
||||
|
||||
/* channel status */
|
||||
#define ADM_CH_STATUS_VALID BIT(1)
|
||||
|
||||
/* channel result */
|
||||
#define ADM_CH_RSLT_VALID BIT(31)
|
||||
#define ADM_CH_RSLT_ERR BIT(3)
|
||||
#define ADM_CH_RSLT_FLUSH BIT(2)
|
||||
#define ADM_CH_RSLT_TPD BIT(1)
|
||||
|
||||
/* channel conf */
|
||||
#define ADM_CH_CONF_SHADOW_EN BIT(12)
|
||||
#define ADM_CH_CONF_MPU_DISABLE BIT(11)
|
||||
#define ADM_CH_CONF_PERM_MPU_CONF BIT(9)
|
||||
#define ADM_CH_CONF_FORCE_RSLT_EN BIT(7)
|
||||
#define ADM_CH_CONF_SEC_DOMAIN(ee) ((((ee) & 0x3) << 4) | (((ee) & 0x4) << 11))
|
||||
|
||||
/* channel result conf */
|
||||
#define ADM_CH_RSLT_CONF_FLUSH_EN BIT(1)
|
||||
#define ADM_CH_RSLT_CONF_IRQ_EN BIT(0)
|
||||
|
||||
/* CRCI CTL */
|
||||
#define ADM_CRCI_CTL_MUX_SEL BIT(18)
|
||||
#define ADM_CRCI_CTL_RST BIT(17)
|
||||
|
||||
/* CI configuration */
|
||||
#define ADM_CI_RANGE_END(x) ((x) << 24)
|
||||
#define ADM_CI_RANGE_START(x) ((x) << 16)
|
||||
#define ADM_CI_BURST_4_WORDS BIT(2)
|
||||
#define ADM_CI_BURST_8_WORDS BIT(3)
|
||||
|
||||
/* GP CTL */
|
||||
#define ADM_GP_CTL_LP_EN BIT(12)
|
||||
#define ADM_GP_CTL_LP_CNT(x) ((x) << 8)
|
||||
|
||||
/* Command pointer list entry */
|
||||
#define ADM_CPLE_LP BIT(31)
|
||||
#define ADM_CPLE_CMD_PTR_LIST BIT(29)
|
||||
|
||||
/* Command list entry */
|
||||
#define ADM_CMD_LC BIT(31)
|
||||
#define ADM_CMD_DST_CRCI(n) (((n) & 0xf) << 7)
|
||||
#define ADM_CMD_SRC_CRCI(n) (((n) & 0xf) << 3)
|
||||
|
||||
#define ADM_CMD_TYPE_SINGLE 0x0
|
||||
#define ADM_CMD_TYPE_BOX 0x3
|
||||
|
||||
#define ADM_CRCI_MUX_SEL BIT(4)
|
||||
#define ADM_DESC_ALIGN 8
|
||||
#define ADM_MAX_XFER (SZ_64K - 1)
|
||||
#define ADM_MAX_ROWS (SZ_64K - 1)
|
||||
#define ADM_MAX_CHANNELS 16
|
||||
|
||||
struct adm_desc_hw_box {
|
||||
u32 cmd;
|
||||
u32 src_addr;
|
||||
u32 dst_addr;
|
||||
u32 row_len;
|
||||
u32 num_rows;
|
||||
u32 row_offset;
|
||||
};
|
||||
|
||||
struct adm_desc_hw_single {
|
||||
u32 cmd;
|
||||
u32 src_addr;
|
||||
u32 dst_addr;
|
||||
u32 len;
|
||||
};
|
||||
|
||||
struct adm_async_desc {
|
||||
struct virt_dma_desc vd;
|
||||
struct adm_device *adev;
|
||||
|
||||
size_t length;
|
||||
enum dma_transfer_direction dir;
|
||||
dma_addr_t dma_addr;
|
||||
size_t dma_len;
|
||||
|
||||
void *cpl;
|
||||
dma_addr_t cp_addr;
|
||||
u32 crci;
|
||||
u32 mux;
|
||||
u32 blk_size;
|
||||
};
|
||||
|
||||
struct adm_chan {
|
||||
struct virt_dma_chan vc;
|
||||
struct adm_device *adev;
|
||||
|
||||
/* parsed from DT */
|
||||
u32 id; /* channel id */
|
||||
|
||||
struct adm_async_desc *curr_txd;
|
||||
struct dma_slave_config slave;
|
||||
struct list_head node;
|
||||
|
||||
int error;
|
||||
int initialized;
|
||||
};
|
||||
|
||||
static inline struct adm_chan *to_adm_chan(struct dma_chan *common)
|
||||
{
|
||||
return container_of(common, struct adm_chan, vc.chan);
|
||||
}
|
||||
|
||||
struct adm_device {
|
||||
void __iomem *regs;
|
||||
struct device *dev;
|
||||
struct dma_device common;
|
||||
struct device_dma_parameters dma_parms;
|
||||
struct adm_chan *channels;
|
||||
|
||||
u32 ee;
|
||||
|
||||
struct clk *core_clk;
|
||||
struct clk *iface_clk;
|
||||
|
||||
struct reset_control *clk_reset;
|
||||
struct reset_control *c0_reset;
|
||||
struct reset_control *c1_reset;
|
||||
struct reset_control *c2_reset;
|
||||
int irq;
|
||||
};
|
||||
|
||||
/**
|
||||
* adm_free_chan - Frees dma resources associated with the specific channel
|
||||
*
|
||||
* @chan: dma channel
|
||||
*
|
||||
* Free all allocated descriptors associated with this channel
|
||||
*/
|
||||
static void adm_free_chan(struct dma_chan *chan)
|
||||
{
|
||||
/* free all queued descriptors */
|
||||
vchan_free_chan_resources(to_virt_chan(chan));
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_get_blksize - Get block size from burst value
|
||||
*
|
||||
* @burst: Burst size of transaction
|
||||
*/
|
||||
static int adm_get_blksize(unsigned int burst)
|
||||
{
|
||||
int ret;
|
||||
|
||||
switch (burst) {
|
||||
case 16:
|
||||
case 32:
|
||||
case 64:
|
||||
case 128:
|
||||
ret = ffs(burst >> 4) - 1;
|
||||
break;
|
||||
case 192:
|
||||
ret = 4;
|
||||
break;
|
||||
case 256:
|
||||
ret = 5;
|
||||
break;
|
||||
default:
|
||||
ret = -EINVAL;
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_process_fc_descriptors - Process descriptors for flow controlled xfers
|
||||
*
|
||||
* @achan: ADM channel
|
||||
* @desc: Descriptor memory pointer
|
||||
* @sg: Scatterlist entry
|
||||
* @crci: CRCI value
|
||||
* @burst: Burst size of transaction
|
||||
* @direction: DMA transfer direction
|
||||
*/
|
||||
static void *adm_process_fc_descriptors(struct adm_chan *achan, void *desc,
|
||||
struct scatterlist *sg, u32 crci,
|
||||
u32 burst,
|
||||
enum dma_transfer_direction direction)
|
||||
{
|
||||
struct adm_desc_hw_box *box_desc = NULL;
|
||||
struct adm_desc_hw_single *single_desc;
|
||||
u32 remainder = sg_dma_len(sg);
|
||||
u32 rows, row_offset, crci_cmd;
|
||||
u32 mem_addr = sg_dma_address(sg);
|
||||
u32 *incr_addr = &mem_addr;
|
||||
u32 *src, *dst;
|
||||
|
||||
if (direction == DMA_DEV_TO_MEM) {
|
||||
crci_cmd = ADM_CMD_SRC_CRCI(crci);
|
||||
row_offset = burst;
|
||||
src = &achan->slave.src_addr;
|
||||
dst = &mem_addr;
|
||||
} else {
|
||||
crci_cmd = ADM_CMD_DST_CRCI(crci);
|
||||
row_offset = burst << 16;
|
||||
src = &mem_addr;
|
||||
dst = &achan->slave.dst_addr;
|
||||
}
|
||||
|
||||
while (remainder >= burst) {
|
||||
box_desc = desc;
|
||||
box_desc->cmd = ADM_CMD_TYPE_BOX | crci_cmd;
|
||||
box_desc->row_offset = row_offset;
|
||||
box_desc->src_addr = *src;
|
||||
box_desc->dst_addr = *dst;
|
||||
|
||||
rows = remainder / burst;
|
||||
rows = min_t(u32, rows, ADM_MAX_ROWS);
|
||||
box_desc->num_rows = rows << 16 | rows;
|
||||
box_desc->row_len = burst << 16 | burst;
|
||||
|
||||
*incr_addr += burst * rows;
|
||||
remainder -= burst * rows;
|
||||
desc += sizeof(*box_desc);
|
||||
}
|
||||
|
||||
/* if leftover bytes, do one single descriptor */
|
||||
if (remainder) {
|
||||
single_desc = desc;
|
||||
single_desc->cmd = ADM_CMD_TYPE_SINGLE | crci_cmd;
|
||||
single_desc->len = remainder;
|
||||
single_desc->src_addr = *src;
|
||||
single_desc->dst_addr = *dst;
|
||||
desc += sizeof(*single_desc);
|
||||
|
||||
if (sg_is_last(sg))
|
||||
single_desc->cmd |= ADM_CMD_LC;
|
||||
} else {
|
||||
if (box_desc && sg_is_last(sg))
|
||||
box_desc->cmd |= ADM_CMD_LC;
|
||||
}
|
||||
|
||||
return desc;
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_process_non_fc_descriptors - Process descriptors for non-fc xfers
|
||||
*
|
||||
* @achan: ADM channel
|
||||
* @desc: Descriptor memory pointer
|
||||
* @sg: Scatterlist entry
|
||||
* @direction: DMA transfer direction
|
||||
*/
|
||||
static void *adm_process_non_fc_descriptors(struct adm_chan *achan, void *desc,
|
||||
struct scatterlist *sg,
|
||||
enum dma_transfer_direction direction)
|
||||
{
|
||||
struct adm_desc_hw_single *single_desc;
|
||||
u32 remainder = sg_dma_len(sg);
|
||||
u32 mem_addr = sg_dma_address(sg);
|
||||
u32 *incr_addr = &mem_addr;
|
||||
u32 *src, *dst;
|
||||
|
||||
if (direction == DMA_DEV_TO_MEM) {
|
||||
src = &achan->slave.src_addr;
|
||||
dst = &mem_addr;
|
||||
} else {
|
||||
src = &mem_addr;
|
||||
dst = &achan->slave.dst_addr;
|
||||
}
|
||||
|
||||
do {
|
||||
single_desc = desc;
|
||||
single_desc->cmd = ADM_CMD_TYPE_SINGLE;
|
||||
single_desc->src_addr = *src;
|
||||
single_desc->dst_addr = *dst;
|
||||
single_desc->len = (remainder > ADM_MAX_XFER) ?
|
||||
ADM_MAX_XFER : remainder;
|
||||
|
||||
remainder -= single_desc->len;
|
||||
*incr_addr += single_desc->len;
|
||||
desc += sizeof(*single_desc);
|
||||
} while (remainder);
|
||||
|
||||
/* set last command if this is the end of the whole transaction */
|
||||
if (sg_is_last(sg))
|
||||
single_desc->cmd |= ADM_CMD_LC;
|
||||
|
||||
return desc;
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_prep_slave_sg - Prep slave sg transaction
|
||||
*
|
||||
* @chan: dma channel
|
||||
* @sgl: scatter gather list
|
||||
* @sg_len: length of sg
|
||||
* @direction: DMA transfer direction
|
||||
* @flags: DMA flags
|
||||
* @context: transfer context (unused)
|
||||
*/
|
||||
static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan,
|
||||
struct scatterlist *sgl,
|
||||
unsigned int sg_len,
|
||||
enum dma_transfer_direction direction,
|
||||
unsigned long flags,
|
||||
void *context)
|
||||
{
|
||||
struct adm_chan *achan = to_adm_chan(chan);
|
||||
struct adm_device *adev = achan->adev;
|
||||
struct adm_async_desc *async_desc;
|
||||
struct scatterlist *sg;
|
||||
dma_addr_t cple_addr;
|
||||
u32 i, burst;
|
||||
u32 single_count = 0, box_count = 0, crci = 0;
|
||||
void *desc;
|
||||
u32 *cple;
|
||||
int blk_size = 0;
|
||||
|
||||
if (!is_slave_direction(direction)) {
|
||||
dev_err(adev->dev, "invalid dma direction\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* get burst value from slave configuration
|
||||
*/
|
||||
burst = (direction == DMA_MEM_TO_DEV) ?
|
||||
achan->slave.dst_maxburst :
|
||||
achan->slave.src_maxburst;
|
||||
|
||||
/* if using flow control, validate burst and crci values */
|
||||
if (achan->slave.device_fc) {
|
||||
blk_size = adm_get_blksize(burst);
|
||||
if (blk_size < 0) {
|
||||
dev_err(adev->dev, "invalid burst value: %d\n",
|
||||
burst);
|
||||
return ERR_PTR(-EINVAL);
|
||||
}
|
||||
|
||||
crci = achan->slave.slave_id & 0xf;
|
||||
if (!crci || achan->slave.slave_id > 0x1f) {
|
||||
dev_err(adev->dev, "invalid crci value\n");
|
||||
return ERR_PTR(-EINVAL);
|
||||
}
|
||||
}
|
||||
|
||||
/* iterate through sgs and compute allocation size of structures */
|
||||
for_each_sg(sgl, sg, sg_len, i) {
|
||||
if (achan->slave.device_fc) {
|
||||
box_count += DIV_ROUND_UP(sg_dma_len(sg) / burst,
|
||||
ADM_MAX_ROWS);
|
||||
if (sg_dma_len(sg) % burst)
|
||||
single_count++;
|
||||
} else {
|
||||
single_count += DIV_ROUND_UP(sg_dma_len(sg),
|
||||
ADM_MAX_XFER);
|
||||
}
|
||||
}
|
||||
|
||||
async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT);
|
||||
if (!async_desc)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
if (crci)
|
||||
async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ?
|
||||
ADM_CRCI_CTL_MUX_SEL : 0;
|
||||
async_desc->crci = crci;
|
||||
async_desc->blk_size = blk_size;
|
||||
async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) +
|
||||
box_count * sizeof(struct adm_desc_hw_box) +
|
||||
sizeof(*cple) + 2 * ADM_DESC_ALIGN;
|
||||
|
||||
async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT);
|
||||
if (!async_desc->cpl)
|
||||
goto free;
|
||||
|
||||
async_desc->adev = adev;
|
||||
|
||||
/* both command list entry and descriptors must be 8 byte aligned */
|
||||
cple = PTR_ALIGN(async_desc->cpl, ADM_DESC_ALIGN);
|
||||
desc = PTR_ALIGN(cple + 1, ADM_DESC_ALIGN);
|
||||
|
||||
for_each_sg(sgl, sg, sg_len, i) {
|
||||
async_desc->length += sg_dma_len(sg);
|
||||
|
||||
if (achan->slave.device_fc)
|
||||
desc = adm_process_fc_descriptors(achan, desc, sg, crci,
|
||||
burst, direction);
|
||||
else
|
||||
desc = adm_process_non_fc_descriptors(achan, desc, sg,
|
||||
direction);
|
||||
}
|
||||
|
||||
async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl,
|
||||
async_desc->dma_len,
|
||||
DMA_TO_DEVICE);
|
||||
if (dma_mapping_error(adev->dev, async_desc->dma_addr))
|
||||
goto free;
|
||||
|
||||
cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl);
|
||||
|
||||
/* init cmd list */
|
||||
dma_sync_single_for_cpu(adev->dev, cple_addr, sizeof(*cple),
|
||||
DMA_TO_DEVICE);
|
||||
*cple = ADM_CPLE_LP;
|
||||
*cple |= (async_desc->dma_addr + ADM_DESC_ALIGN) >> 3;
|
||||
dma_sync_single_for_device(adev->dev, cple_addr, sizeof(*cple),
|
||||
DMA_TO_DEVICE);
|
||||
|
||||
return vchan_tx_prep(&achan->vc, &async_desc->vd, flags);
|
||||
|
||||
free:
|
||||
kfree(async_desc);
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_terminate_all - terminate all transactions on a channel
|
||||
* @chan: dma channel
|
||||
*
|
||||
* Dequeues and frees all transactions, aborts current transaction
|
||||
* No callbacks are done
|
||||
*
|
||||
*/
|
||||
static int adm_terminate_all(struct dma_chan *chan)
|
||||
{
|
||||
struct adm_chan *achan = to_adm_chan(chan);
|
||||
struct adm_device *adev = achan->adev;
|
||||
unsigned long flags;
|
||||
LIST_HEAD(head);
|
||||
|
||||
spin_lock_irqsave(&achan->vc.lock, flags);
|
||||
vchan_get_all_descriptors(&achan->vc, &head);
|
||||
|
||||
/* send flush command to terminate current transaction */
|
||||
writel_relaxed(0x0,
|
||||
adev->regs + ADM_CH_FLUSH_STATE0(achan->id, adev->ee));
|
||||
|
||||
spin_unlock_irqrestore(&achan->vc.lock, flags);
|
||||
|
||||
vchan_dma_desc_free_list(&achan->vc, &head);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
|
||||
{
|
||||
struct adm_chan *achan = to_adm_chan(chan);
|
||||
unsigned long flag;
|
||||
|
||||
spin_lock_irqsave(&achan->vc.lock, flag);
|
||||
memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config));
|
||||
spin_unlock_irqrestore(&achan->vc.lock, flag);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_start_dma - start next transaction
|
||||
* @achan: ADM dma channel
|
||||
*/
|
||||
static void adm_start_dma(struct adm_chan *achan)
|
||||
{
|
||||
struct virt_dma_desc *vd = vchan_next_desc(&achan->vc);
|
||||
struct adm_device *adev = achan->adev;
|
||||
struct adm_async_desc *async_desc;
|
||||
|
||||
lockdep_assert_held(&achan->vc.lock);
|
||||
|
||||
if (!vd)
|
||||
return;
|
||||
|
||||
list_del(&vd->node);
|
||||
|
||||
/* write next command list out to the CMD FIFO */
|
||||
async_desc = container_of(vd, struct adm_async_desc, vd);
|
||||
achan->curr_txd = async_desc;
|
||||
|
||||
/* reset channel error */
|
||||
achan->error = 0;
|
||||
|
||||
if (!achan->initialized) {
|
||||
/* enable interrupts */
|
||||
writel(ADM_CH_CONF_SHADOW_EN |
|
||||
ADM_CH_CONF_PERM_MPU_CONF |
|
||||
ADM_CH_CONF_MPU_DISABLE |
|
||||
ADM_CH_CONF_SEC_DOMAIN(adev->ee),
|
||||
adev->regs + ADM_CH_CONF(achan->id));
|
||||
|
||||
writel(ADM_CH_RSLT_CONF_IRQ_EN | ADM_CH_RSLT_CONF_FLUSH_EN,
|
||||
adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
|
||||
|
||||
achan->initialized = 1;
|
||||
}
|
||||
|
||||
/* set the crci block size if this transaction requires CRCI */
|
||||
if (async_desc->crci) {
|
||||
writel(async_desc->mux | async_desc->blk_size,
|
||||
adev->regs + ADM_CRCI_CTL(async_desc->crci, adev->ee));
|
||||
}
|
||||
|
||||
/* make sure IRQ enable doesn't get reordered */
|
||||
wmb();
|
||||
|
||||
/* write next command list out to the CMD FIFO */
|
||||
writel(ALIGN(async_desc->dma_addr, ADM_DESC_ALIGN) >> 3,
|
||||
adev->regs + ADM_CH_CMD_PTR(achan->id, adev->ee));
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_dma_irq - irq handler for ADM controller
|
||||
* @irq: IRQ of interrupt
|
||||
* @data: callback data
|
||||
*
|
||||
* IRQ handler for the bam controller
|
||||
*/
|
||||
static irqreturn_t adm_dma_irq(int irq, void *data)
|
||||
{
|
||||
struct adm_device *adev = data;
|
||||
u32 srcs, i;
|
||||
struct adm_async_desc *async_desc;
|
||||
unsigned long flags;
|
||||
|
||||
srcs = readl_relaxed(adev->regs +
|
||||
ADM_SEC_DOMAIN_IRQ_STATUS(adev->ee));
|
||||
|
||||
for (i = 0; i < ADM_MAX_CHANNELS; i++) {
|
||||
struct adm_chan *achan = &adev->channels[i];
|
||||
u32 status, result;
|
||||
|
||||
if (srcs & BIT(i)) {
|
||||
status = readl_relaxed(adev->regs +
|
||||
ADM_CH_STATUS_SD(i, adev->ee));
|
||||
|
||||
/* if no result present, skip */
|
||||
if (!(status & ADM_CH_STATUS_VALID))
|
||||
continue;
|
||||
|
||||
result = readl_relaxed(adev->regs +
|
||||
ADM_CH_RSLT(i, adev->ee));
|
||||
|
||||
/* no valid results, skip */
|
||||
if (!(result & ADM_CH_RSLT_VALID))
|
||||
continue;
|
||||
|
||||
/* flag error if transaction was flushed or failed */
|
||||
if (result & (ADM_CH_RSLT_ERR | ADM_CH_RSLT_FLUSH))
|
||||
achan->error = 1;
|
||||
|
||||
spin_lock_irqsave(&achan->vc.lock, flags);
|
||||
async_desc = achan->curr_txd;
|
||||
|
||||
achan->curr_txd = NULL;
|
||||
|
||||
if (async_desc) {
|
||||
vchan_cookie_complete(&async_desc->vd);
|
||||
|
||||
/* kick off next DMA */
|
||||
adm_start_dma(achan);
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&achan->vc.lock, flags);
|
||||
}
|
||||
}
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_tx_status - returns status of transaction
|
||||
* @chan: dma channel
|
||||
* @cookie: transaction cookie
|
||||
* @txstate: DMA transaction state
|
||||
*
|
||||
* Return status of dma transaction
|
||||
*/
|
||||
static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
|
||||
struct dma_tx_state *txstate)
|
||||
{
|
||||
struct adm_chan *achan = to_adm_chan(chan);
|
||||
struct virt_dma_desc *vd;
|
||||
enum dma_status ret;
|
||||
unsigned long flags;
|
||||
size_t residue = 0;
|
||||
|
||||
ret = dma_cookie_status(chan, cookie, txstate);
|
||||
if (ret == DMA_COMPLETE || !txstate)
|
||||
return ret;
|
||||
|
||||
spin_lock_irqsave(&achan->vc.lock, flags);
|
||||
|
||||
vd = vchan_find_desc(&achan->vc, cookie);
|
||||
if (vd)
|
||||
residue = container_of(vd, struct adm_async_desc, vd)->length;
|
||||
|
||||
spin_unlock_irqrestore(&achan->vc.lock, flags);
|
||||
|
||||
/*
|
||||
* residue is either the full length if it is in the issued list, or 0
|
||||
* if it is in progress. We have no reliable way of determining
|
||||
* anything inbetween
|
||||
*/
|
||||
dma_set_residue(txstate, residue);
|
||||
|
||||
if (achan->error)
|
||||
return DMA_ERROR;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_issue_pending - starts pending transactions
|
||||
* @chan: dma channel
|
||||
*
|
||||
* Issues all pending transactions and starts DMA
|
||||
*/
|
||||
static void adm_issue_pending(struct dma_chan *chan)
|
||||
{
|
||||
struct adm_chan *achan = to_adm_chan(chan);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&achan->vc.lock, flags);
|
||||
|
||||
if (vchan_issue_pending(&achan->vc) && !achan->curr_txd)
|
||||
adm_start_dma(achan);
|
||||
spin_unlock_irqrestore(&achan->vc.lock, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* adm_dma_free_desc - free descriptor memory
|
||||
* @vd: virtual descriptor
|
||||
*
|
||||
*/
|
||||
static void adm_dma_free_desc(struct virt_dma_desc *vd)
|
||||
{
|
||||
struct adm_async_desc *async_desc = container_of(vd,
|
||||
struct adm_async_desc, vd);
|
||||
|
||||
dma_unmap_single(async_desc->adev->dev, async_desc->dma_addr,
|
||||
async_desc->dma_len, DMA_TO_DEVICE);
|
||||
kfree(async_desc->cpl);
|
||||
kfree(async_desc);
|
||||
}
|
||||
|
||||
static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan,
|
||||
u32 index)
|
||||
{
|
||||
achan->id = index;
|
||||
achan->adev = adev;
|
||||
|
||||
vchan_init(&achan->vc, &adev->common);
|
||||
achan->vc.desc_free = adm_dma_free_desc;
|
||||
}
|
||||
|
||||
static int adm_dma_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct adm_device *adev;
|
||||
int ret;
|
||||
u32 i;
|
||||
|
||||
adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL);
|
||||
if (!adev)
|
||||
return -ENOMEM;
|
||||
|
||||
adev->dev = &pdev->dev;
|
||||
|
||||
adev->regs = devm_platform_ioremap_resource(pdev, 0);
|
||||
if (IS_ERR(adev->regs))
|
||||
return PTR_ERR(adev->regs);
|
||||
|
||||
adev->irq = platform_get_irq(pdev, 0);
|
||||
if (adev->irq < 0)
|
||||
return adev->irq;
|
||||
|
||||
ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &adev->ee);
|
||||
if (ret) {
|
||||
dev_err(adev->dev, "Execution environment unspecified\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
adev->core_clk = devm_clk_get(adev->dev, "core");
|
||||
if (IS_ERR(adev->core_clk))
|
||||
return PTR_ERR(adev->core_clk);
|
||||
|
||||
adev->iface_clk = devm_clk_get(adev->dev, "iface");
|
||||
if (IS_ERR(adev->iface_clk))
|
||||
return PTR_ERR(adev->iface_clk);
|
||||
|
||||
adev->clk_reset = devm_reset_control_get_exclusive(&pdev->dev, "clk");
|
||||
if (IS_ERR(adev->clk_reset)) {
|
||||
dev_err(adev->dev, "failed to get ADM0 reset\n");
|
||||
return PTR_ERR(adev->clk_reset);
|
||||
}
|
||||
|
||||
adev->c0_reset = devm_reset_control_get_exclusive(&pdev->dev, "c0");
|
||||
if (IS_ERR(adev->c0_reset)) {
|
||||
dev_err(adev->dev, "failed to get ADM0 C0 reset\n");
|
||||
return PTR_ERR(adev->c0_reset);
|
||||
}
|
||||
|
||||
adev->c1_reset = devm_reset_control_get_exclusive(&pdev->dev, "c1");
|
||||
if (IS_ERR(adev->c1_reset)) {
|
||||
dev_err(adev->dev, "failed to get ADM0 C1 reset\n");
|
||||
return PTR_ERR(adev->c1_reset);
|
||||
}
|
||||
|
||||
adev->c2_reset = devm_reset_control_get_exclusive(&pdev->dev, "c2");
|
||||
if (IS_ERR(adev->c2_reset)) {
|
||||
dev_err(adev->dev, "failed to get ADM0 C2 reset\n");
|
||||
return PTR_ERR(adev->c2_reset);
|
||||
}
|
||||
|
||||
ret = clk_prepare_enable(adev->core_clk);
|
||||
if (ret) {
|
||||
dev_err(adev->dev, "failed to prepare/enable core clock\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = clk_prepare_enable(adev->iface_clk);
|
||||
if (ret) {
|
||||
dev_err(adev->dev, "failed to prepare/enable iface clock\n");
|
||||
goto err_disable_core_clk;
|
||||
}
|
||||
|
||||
reset_control_assert(adev->clk_reset);
|
||||
reset_control_assert(adev->c0_reset);
|
||||
reset_control_assert(adev->c1_reset);
|
||||
reset_control_assert(adev->c2_reset);
|
||||
|
||||
udelay(2);
|
||||
|
||||
reset_control_deassert(adev->clk_reset);
|
||||
reset_control_deassert(adev->c0_reset);
|
||||
reset_control_deassert(adev->c1_reset);
|
||||
reset_control_deassert(adev->c2_reset);
|
||||
|
||||
adev->channels = devm_kcalloc(adev->dev, ADM_MAX_CHANNELS,
|
||||
sizeof(*adev->channels), GFP_KERNEL);
|
||||
|
||||
if (!adev->channels) {
|
||||
ret = -ENOMEM;
|
||||
goto err_disable_clks;
|
||||
}
|
||||
|
||||
/* allocate and initialize channels */
|
||||
INIT_LIST_HEAD(&adev->common.channels);
|
||||
|
||||
for (i = 0; i < ADM_MAX_CHANNELS; i++)
|
||||
adm_channel_init(adev, &adev->channels[i], i);
|
||||
|
||||
/* reset CRCIs */
|
||||
for (i = 0; i < 16; i++)
|
||||
writel(ADM_CRCI_CTL_RST, adev->regs +
|
||||
ADM_CRCI_CTL(i, adev->ee));
|
||||
|
||||
/* configure client interfaces */
|
||||
writel(ADM_CI_RANGE_START(0x40) | ADM_CI_RANGE_END(0xb0) |
|
||||
ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(0));
|
||||
writel(ADM_CI_RANGE_START(0x2a) | ADM_CI_RANGE_END(0x2c) |
|
||||
ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(1));
|
||||
writel(ADM_CI_RANGE_START(0x12) | ADM_CI_RANGE_END(0x28) |
|
||||
ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(2));
|
||||
writel(ADM_GP_CTL_LP_EN | ADM_GP_CTL_LP_CNT(0xf),
|
||||
adev->regs + ADM_GP_CTL);
|
||||
|
||||
ret = devm_request_irq(adev->dev, adev->irq, adm_dma_irq,
|
||||
0, "adm_dma", adev);
|
||||
if (ret)
|
||||
goto err_disable_clks;
|
||||
|
||||
platform_set_drvdata(pdev, adev);
|
||||
|
||||
adev->common.dev = adev->dev;
|
||||
adev->common.dev->dma_parms = &adev->dma_parms;
|
||||
|
||||
/* set capabilities */
|
||||
dma_cap_zero(adev->common.cap_mask);
|
||||
dma_cap_set(DMA_SLAVE, adev->common.cap_mask);
|
||||
dma_cap_set(DMA_PRIVATE, adev->common.cap_mask);
|
||||
|
||||
/* initialize dmaengine apis */
|
||||
adev->common.directions = BIT(DMA_DEV_TO_MEM | DMA_MEM_TO_DEV);
|
||||
adev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
|
||||
adev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
adev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
adev->common.device_free_chan_resources = adm_free_chan;
|
||||
adev->common.device_prep_slave_sg = adm_prep_slave_sg;
|
||||
adev->common.device_issue_pending = adm_issue_pending;
|
||||
adev->common.device_tx_status = adm_tx_status;
|
||||
adev->common.device_terminate_all = adm_terminate_all;
|
||||
adev->common.device_config = adm_slave_config;
|
||||
|
||||
ret = dma_async_device_register(&adev->common);
|
||||
if (ret) {
|
||||
dev_err(adev->dev, "failed to register dma async device\n");
|
||||
goto err_disable_clks;
|
||||
}
|
||||
|
||||
ret = of_dma_controller_register(pdev->dev.of_node,
|
||||
of_dma_xlate_by_chan_id,
|
||||
&adev->common);
|
||||
if (ret)
|
||||
goto err_unregister_dma;
|
||||
|
||||
return 0;
|
||||
|
||||
err_unregister_dma:
|
||||
dma_async_device_unregister(&adev->common);
|
||||
err_disable_clks:
|
||||
clk_disable_unprepare(adev->iface_clk);
|
||||
err_disable_core_clk:
|
||||
clk_disable_unprepare(adev->core_clk);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int adm_dma_remove(struct platform_device *pdev)
|
||||
{
|
||||
struct adm_device *adev = platform_get_drvdata(pdev);
|
||||
struct adm_chan *achan;
|
||||
u32 i;
|
||||
|
||||
of_dma_controller_free(pdev->dev.of_node);
|
||||
dma_async_device_unregister(&adev->common);
|
||||
|
||||
for (i = 0; i < ADM_MAX_CHANNELS; i++) {
|
||||
achan = &adev->channels[i];
|
||||
|
||||
/* mask IRQs for this channel/EE pair */
|
||||
writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
|
||||
|
||||
tasklet_kill(&adev->channels[i].vc.task);
|
||||
adm_terminate_all(&adev->channels[i].vc.chan);
|
||||
}
|
||||
|
||||
devm_free_irq(adev->dev, adev->irq, adev);
|
||||
|
||||
clk_disable_unprepare(adev->core_clk);
|
||||
clk_disable_unprepare(adev->iface_clk);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct of_device_id adm_of_match[] = {
|
||||
{ .compatible = "qcom,adm", },
|
||||
{}
|
||||
};
|
||||
MODULE_DEVICE_TABLE(of, adm_of_match);
|
||||
|
||||
static struct platform_driver adm_dma_driver = {
|
||||
.probe = adm_dma_probe,
|
||||
.remove = adm_dma_remove,
|
||||
.driver = {
|
||||
.name = "adm-dma-engine",
|
||||
.of_match_table = adm_of_match,
|
||||
},
|
||||
};
|
||||
|
||||
module_platform_driver(adm_dma_driver);
|
||||
|
||||
MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
|
||||
MODULE_DESCRIPTION("QCOM ADM DMA engine driver");
|
||||
MODULE_LICENSE("GPL v2");
|
|
@ -326,10 +326,9 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
|
|||
{
|
||||
struct sf_pdma_chan *chan = dev_id;
|
||||
struct pdma_regs *regs = &chan->regs;
|
||||
unsigned long flags;
|
||||
u64 residue;
|
||||
|
||||
spin_lock_irqsave(&chan->vchan.lock, flags);
|
||||
spin_lock(&chan->vchan.lock);
|
||||
writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl);
|
||||
residue = readq(regs->residue);
|
||||
|
||||
|
@ -346,7 +345,7 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
|
|||
sf_pdma_xfer_desc(chan);
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&chan->vchan.lock, flags);
|
||||
spin_unlock(&chan->vchan.lock);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
@ -355,11 +354,10 @@ static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id)
|
|||
{
|
||||
struct sf_pdma_chan *chan = dev_id;
|
||||
struct pdma_regs *regs = &chan->regs;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&chan->lock, flags);
|
||||
spin_lock(&chan->lock);
|
||||
writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl);
|
||||
spin_unlock_irqrestore(&chan->lock, flags);
|
||||
spin_unlock(&chan->lock);
|
||||
|
||||
tasklet_schedule(&chan->err_tasklet);
|
||||
|
||||
|
@ -584,7 +582,7 @@ static struct platform_driver sf_pdma_driver = {
|
|||
.remove = sf_pdma_remove,
|
||||
.driver = {
|
||||
.name = "sf-pdma",
|
||||
.of_match_table = of_match_ptr(sf_pdma_dt_ids),
|
||||
.of_match_table = sf_pdma_dt_ids,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
|
@ -1643,13 +1643,12 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
|
|||
u32 row;
|
||||
long chan = -1;
|
||||
struct d40_chan *d40c;
|
||||
unsigned long flags;
|
||||
struct d40_base *base = data;
|
||||
u32 *regs = base->regs_interrupt;
|
||||
struct d40_interrupt_lookup *il = base->gen_dmac.il;
|
||||
u32 il_size = base->gen_dmac.il_size;
|
||||
|
||||
spin_lock_irqsave(&base->interrupt_lock, flags);
|
||||
spin_lock(&base->interrupt_lock);
|
||||
|
||||
/* Read interrupt status of both logical and physical channels */
|
||||
for (i = 0; i < il_size; i++)
|
||||
|
@ -1694,7 +1693,7 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
|
|||
spin_unlock(&d40c->lock);
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&base->interrupt_lock, flags);
|
||||
spin_unlock(&base->interrupt_lock);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
|
|
@ -264,9 +264,11 @@ static int stm32_dma_get_width(struct stm32_dma_chan *chan,
|
|||
}
|
||||
|
||||
static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,
|
||||
dma_addr_t buf_addr,
|
||||
u32 threshold)
|
||||
{
|
||||
enum dma_slave_buswidth max_width;
|
||||
u64 addr = buf_addr;
|
||||
|
||||
if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL)
|
||||
max_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
|
||||
|
@ -277,6 +279,9 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len,
|
|||
max_width > DMA_SLAVE_BUSWIDTH_1_BYTE)
|
||||
max_width = max_width >> 1;
|
||||
|
||||
if (do_div(addr, max_width))
|
||||
max_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
|
||||
|
||||
return max_width;
|
||||
}
|
||||
|
||||
|
@ -648,21 +653,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
|
|||
scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
|
||||
sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));
|
||||
|
||||
if (status & STM32_DMA_TCI) {
|
||||
stm32_dma_irq_clear(chan, STM32_DMA_TCI);
|
||||
if (scr & STM32_DMA_SCR_TCIE)
|
||||
stm32_dma_handle_chan_done(chan);
|
||||
status &= ~STM32_DMA_TCI;
|
||||
}
|
||||
if (status & STM32_DMA_HTI) {
|
||||
stm32_dma_irq_clear(chan, STM32_DMA_HTI);
|
||||
status &= ~STM32_DMA_HTI;
|
||||
}
|
||||
if (status & STM32_DMA_FEI) {
|
||||
stm32_dma_irq_clear(chan, STM32_DMA_FEI);
|
||||
status &= ~STM32_DMA_FEI;
|
||||
if (sfcr & STM32_DMA_SFCR_FEIE) {
|
||||
if (!(scr & STM32_DMA_SCR_EN))
|
||||
if (!(scr & STM32_DMA_SCR_EN) &&
|
||||
!(status & STM32_DMA_TCI))
|
||||
dev_err(chan2dev(chan), "FIFO Error\n");
|
||||
else
|
||||
dev_dbg(chan2dev(chan), "FIFO over/underrun\n");
|
||||
|
@ -674,6 +670,19 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
|
|||
if (sfcr & STM32_DMA_SCR_DMEIE)
|
||||
dev_dbg(chan2dev(chan), "Direct mode overrun\n");
|
||||
}
|
||||
|
||||
if (status & STM32_DMA_TCI) {
|
||||
stm32_dma_irq_clear(chan, STM32_DMA_TCI);
|
||||
if (scr & STM32_DMA_SCR_TCIE)
|
||||
stm32_dma_handle_chan_done(chan);
|
||||
status &= ~STM32_DMA_TCI;
|
||||
}
|
||||
|
||||
if (status & STM32_DMA_HTI) {
|
||||
stm32_dma_irq_clear(chan, STM32_DMA_HTI);
|
||||
status &= ~STM32_DMA_HTI;
|
||||
}
|
||||
|
||||
if (status) {
|
||||
stm32_dma_irq_clear(chan, status);
|
||||
dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
|
||||
|
@ -703,7 +712,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c)
|
|||
static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
|
||||
enum dma_transfer_direction direction,
|
||||
enum dma_slave_buswidth *buswidth,
|
||||
u32 buf_len)
|
||||
u32 buf_len, dma_addr_t buf_addr)
|
||||
{
|
||||
enum dma_slave_buswidth src_addr_width, dst_addr_width;
|
||||
int src_bus_width, dst_bus_width;
|
||||
|
@ -735,7 +744,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
|
|||
return dst_burst_size;
|
||||
|
||||
/* Set memory data size */
|
||||
src_addr_width = stm32_dma_get_max_width(buf_len, fifoth);
|
||||
src_addr_width = stm32_dma_get_max_width(buf_len, buf_addr,
|
||||
fifoth);
|
||||
chan->mem_width = src_addr_width;
|
||||
src_bus_width = stm32_dma_get_width(chan, src_addr_width);
|
||||
if (src_bus_width < 0)
|
||||
|
@ -784,7 +794,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
|
|||
return src_burst_size;
|
||||
|
||||
/* Set memory data size */
|
||||
dst_addr_width = stm32_dma_get_max_width(buf_len, fifoth);
|
||||
dst_addr_width = stm32_dma_get_max_width(buf_len, buf_addr,
|
||||
fifoth);
|
||||
chan->mem_width = dst_addr_width;
|
||||
dst_bus_width = stm32_dma_get_width(chan, dst_addr_width);
|
||||
if (dst_bus_width < 0)
|
||||
|
@ -872,7 +883,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg(
|
|||
|
||||
for_each_sg(sgl, sg, sg_len, i) {
|
||||
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth,
|
||||
sg_dma_len(sg));
|
||||
sg_dma_len(sg),
|
||||
sg_dma_address(sg));
|
||||
if (ret < 0)
|
||||
goto err;
|
||||
|
||||
|
@ -940,7 +952,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(
|
|||
return NULL;
|
||||
}
|
||||
|
||||
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len);
|
||||
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, period_len,
|
||||
buf_addr);
|
||||
if (ret < 0)
|
||||
return NULL;
|
||||
|
||||
|
@ -1216,6 +1229,8 @@ static void stm32_dma_free_chan_resources(struct dma_chan *c)
|
|||
pm_runtime_put(dmadev->ddev.dev);
|
||||
|
||||
vchan_free_chan_resources(to_virt_chan(c));
|
||||
stm32_dma_clear_reg(&chan->chan_reg);
|
||||
chan->threshold = 0;
|
||||
}
|
||||
|
||||
static void stm32_dma_desc_free(struct virt_dma_desc *vdesc)
|
||||
|
|
|
@ -168,7 +168,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
|
|||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
static const struct of_device_id stm32_stm32dma_master_match[] = {
|
||||
static const struct of_device_id stm32_stm32dma_master_match[] __maybe_unused = {
|
||||
{ .compatible = "st,stm32-dma", },
|
||||
{},
|
||||
};
|
||||
|
|
|
@ -339,7 +339,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
|
|||
struct stm32_mdma_desc *desc;
|
||||
int i;
|
||||
|
||||
desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT);
|
||||
desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT);
|
||||
if (!desc)
|
||||
return NULL;
|
||||
|
||||
|
@ -1346,7 +1346,7 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
|
|||
{
|
||||
struct stm32_mdma_device *dmadev = devid;
|
||||
struct stm32_mdma_chan *chan = devid;
|
||||
u32 reg, id, ien, status, flag;
|
||||
u32 reg, id, ccr, ien, status;
|
||||
|
||||
/* Find out which channel generates the interrupt */
|
||||
status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
|
||||
|
@ -1368,67 +1368,71 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
|
|||
|
||||
chan = &dmadev->chan[id];
|
||||
if (!chan) {
|
||||
dev_dbg(mdma2dev(dmadev), "MDMA channel not initialized\n");
|
||||
goto exit;
|
||||
dev_warn(mdma2dev(dmadev), "MDMA channel not initialized\n");
|
||||
return IRQ_NONE;
|
||||
}
|
||||
|
||||
/* Handle interrupt for the channel */
|
||||
spin_lock(&chan->vchan.lock);
|
||||
status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
|
||||
ien = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
|
||||
ien &= STM32_MDMA_CCR_IRQ_MASK;
|
||||
ien >>= 1;
|
||||
status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
|
||||
/* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */
|
||||
status &= ~STM32_MDMA_CISR_CRQA;
|
||||
ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));
|
||||
ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1;
|
||||
|
||||
if (!(status & ien)) {
|
||||
spin_unlock(&chan->vchan.lock);
|
||||
dev_dbg(chan2dev(chan),
|
||||
"spurious it (status=0x%04x, ien=0x%04x)\n",
|
||||
status, ien);
|
||||
dev_warn(chan2dev(chan),
|
||||
"spurious it (status=0x%04x, ien=0x%04x)\n",
|
||||
status, ien);
|
||||
return IRQ_NONE;
|
||||
}
|
||||
|
||||
flag = __ffs(status & ien);
|
||||
reg = STM32_MDMA_CIFCR(chan->id);
|
||||
reg = STM32_MDMA_CIFCR(id);
|
||||
|
||||
switch (1 << flag) {
|
||||
case STM32_MDMA_CISR_TEIF:
|
||||
id = chan->id;
|
||||
status = readl_relaxed(dmadev->base + STM32_MDMA_CESR(id));
|
||||
dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", status);
|
||||
if (status & STM32_MDMA_CISR_TEIF) {
|
||||
dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n",
|
||||
readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)));
|
||||
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
|
||||
break;
|
||||
status &= ~STM32_MDMA_CISR_TEIF;
|
||||
}
|
||||
|
||||
case STM32_MDMA_CISR_CTCIF:
|
||||
if (status & STM32_MDMA_CISR_CTCIF) {
|
||||
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);
|
||||
status &= ~STM32_MDMA_CISR_CTCIF;
|
||||
stm32_mdma_xfer_end(chan);
|
||||
break;
|
||||
}
|
||||
|
||||
case STM32_MDMA_CISR_BRTIF:
|
||||
if (status & STM32_MDMA_CISR_BRTIF) {
|
||||
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);
|
||||
break;
|
||||
status &= ~STM32_MDMA_CISR_BRTIF;
|
||||
}
|
||||
|
||||
case STM32_MDMA_CISR_BTIF:
|
||||
if (status & STM32_MDMA_CISR_BTIF) {
|
||||
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);
|
||||
status &= ~STM32_MDMA_CISR_BTIF;
|
||||
chan->curr_hwdesc++;
|
||||
if (chan->desc && chan->desc->cyclic) {
|
||||
if (chan->curr_hwdesc == chan->desc->count)
|
||||
chan->curr_hwdesc = 0;
|
||||
vchan_cyclic_callback(&chan->desc->vdesc);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case STM32_MDMA_CISR_TCIF:
|
||||
if (status & STM32_MDMA_CISR_TCIF) {
|
||||
stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);
|
||||
break;
|
||||
status &= ~STM32_MDMA_CISR_TCIF;
|
||||
}
|
||||
|
||||
default:
|
||||
dev_err(chan2dev(chan), "it %d unhandled (status=0x%04x)\n",
|
||||
1 << flag, status);
|
||||
if (status) {
|
||||
stm32_mdma_set_bits(dmadev, reg, status);
|
||||
dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
|
||||
if (!(ccr & STM32_MDMA_CCR_EN))
|
||||
dev_err(chan2dev(chan), "chan disabled by HW\n");
|
||||
}
|
||||
|
||||
spin_unlock(&chan->vchan.lock);
|
||||
|
||||
exit:
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
|
|
|
@ -1173,6 +1173,30 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = {
|
|||
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
|
||||
};
|
||||
|
||||
/*
|
||||
* TODO: Add support for more than 4g physical addressing.
|
||||
*
|
||||
* The A100 binding uses the number of dma channels from the
|
||||
* device tree node.
|
||||
*/
|
||||
static struct sun6i_dma_config sun50i_a100_dma_cfg = {
|
||||
.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
|
||||
.set_burst_length = sun6i_set_burst_length_h3,
|
||||
.set_drq = sun6i_set_drq_h6,
|
||||
.set_mode = sun6i_set_mode_h6,
|
||||
.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
|
||||
.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
|
||||
.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
|
||||
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
|
||||
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
|
||||
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
|
||||
.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
|
||||
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
|
||||
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
|
||||
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
|
||||
.has_mbus_clk = true,
|
||||
};
|
||||
|
||||
/*
|
||||
* The H6 binding uses the number of dma channels from the
|
||||
* device tree node.
|
||||
|
@ -1225,6 +1249,7 @@ static const struct of_device_id sun6i_dma_match[] = {
|
|||
{ .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
|
||||
{ .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
|
||||
{ .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg },
|
||||
{ .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg },
|
||||
{ .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg },
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
|
|
|
@ -408,19 +408,18 @@ static irqreturn_t tegra_adma_isr(int irq, void *dev_id)
|
|||
{
|
||||
struct tegra_adma_chan *tdc = dev_id;
|
||||
unsigned long status;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&tdc->vc.lock, flags);
|
||||
spin_lock(&tdc->vc.lock);
|
||||
|
||||
status = tegra_adma_irq_clear(tdc);
|
||||
if (status == 0 || !tdc->desc) {
|
||||
spin_unlock_irqrestore(&tdc->vc.lock, flags);
|
||||
spin_unlock(&tdc->vc.lock);
|
||||
return IRQ_NONE;
|
||||
}
|
||||
|
||||
vchan_cyclic_callback(&tdc->desc->vd);
|
||||
|
||||
spin_unlock_irqrestore(&tdc->vc.lock, flags);
|
||||
spin_unlock(&tdc->vc.lock);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
|
|
@ -7,5 +7,6 @@ obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o
|
|||
obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o \
|
||||
k3-psil-am654.o \
|
||||
k3-psil-j721e.o \
|
||||
k3-psil-j7200.o
|
||||
k3-psil-j7200.o \
|
||||
k3-psil-am64.o
|
||||
obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o
|
||||
|
|
|
@ -122,7 +122,7 @@ static void *ti_am335x_xbar_route_allocate(struct of_phandle_args *dma_spec,
|
|||
return map;
|
||||
}
|
||||
|
||||
static const struct of_device_id ti_am335x_master_match[] = {
|
||||
static const struct of_device_id ti_am335x_master_match[] __maybe_unused = {
|
||||
{ .compatible = "ti,edma3-tpcc", },
|
||||
{},
|
||||
};
|
||||
|
@ -292,7 +292,7 @@ static const u32 ti_dma_offset[] = {
|
|||
[TI_XBAR_SDMA_OFFSET] = 1,
|
||||
};
|
||||
|
||||
static const struct of_device_id ti_dra7_master_match[] = {
|
||||
static const struct of_device_id ti_dra7_master_match[] __maybe_unused = {
|
||||
{
|
||||
.compatible = "ti,omap4430-sdma",
|
||||
.data = &ti_dma_offset[TI_XBAR_SDMA_OFFSET],
|
||||
|
@ -460,7 +460,7 @@ static int ti_dma_xbar_probe(struct platform_device *pdev)
|
|||
static struct platform_driver ti_dma_xbar_driver = {
|
||||
.driver = {
|
||||
.name = "ti-dma-crossbar",
|
||||
.of_match_table = of_match_ptr(ti_dma_xbar_match),
|
||||
.of_match_table = ti_dma_xbar_match,
|
||||
},
|
||||
.probe = ti_dma_xbar_probe,
|
||||
};
|
||||
|
|
158
drivers/dma/ti/k3-psil-am64.c
Normal file
158
drivers/dma/ti/k3-psil-am64.c
Normal file
|
@ -0,0 +1,158 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com
|
||||
* Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
|
||||
#include "k3-psil-priv.h"
|
||||
|
||||
#define PSIL_PDMA_XY_TR(x) \
|
||||
{ \
|
||||
.thread_id = x, \
|
||||
.ep_config = { \
|
||||
.ep_type = PSIL_EP_PDMA_XY, \
|
||||
.mapped_channel_id = -1, \
|
||||
.default_flow_id = -1, \
|
||||
}, \
|
||||
}
|
||||
|
||||
#define PSIL_PDMA_XY_PKT(x) \
|
||||
{ \
|
||||
.thread_id = x, \
|
||||
.ep_config = { \
|
||||
.ep_type = PSIL_EP_PDMA_XY, \
|
||||
.mapped_channel_id = -1, \
|
||||
.default_flow_id = -1, \
|
||||
.pkt_mode = 1, \
|
||||
}, \
|
||||
}
|
||||
|
||||
#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \
|
||||
{ \
|
||||
.thread_id = x, \
|
||||
.ep_config = { \
|
||||
.ep_type = PSIL_EP_NATIVE, \
|
||||
.pkt_mode = 1, \
|
||||
.needs_epib = 1, \
|
||||
.psd_size = 16, \
|
||||
.mapped_channel_id = ch, \
|
||||
.flow_start = flow_base, \
|
||||
.flow_num = flow_cnt, \
|
||||
.default_flow_id = flow_base, \
|
||||
}, \
|
||||
}
|
||||
|
||||
#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \
|
||||
{ \
|
||||
.thread_id = x, \
|
||||
.ep_config = { \
|
||||
.ep_type = PSIL_EP_NATIVE, \
|
||||
.pkt_mode = 1, \
|
||||
.needs_epib = 1, \
|
||||
.psd_size = 64, \
|
||||
.mapped_channel_id = ch, \
|
||||
.flow_start = flow_base, \
|
||||
.flow_num = flow_cnt, \
|
||||
.default_flow_id = default_flow, \
|
||||
.notdpkt = tx, \
|
||||
}, \
|
||||
}
|
||||
|
||||
/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */
|
||||
static struct psil_ep am64_src_ep_map[] = {
|
||||
/* SAUL */
|
||||
PSIL_SAUL(0x4000, 17, 32, 8, 32, 0),
|
||||
PSIL_SAUL(0x4001, 18, 32, 8, 33, 0),
|
||||
PSIL_SAUL(0x4002, 19, 40, 8, 40, 0),
|
||||
PSIL_SAUL(0x4003, 20, 40, 8, 41, 0),
|
||||
/* ICSS_G0 */
|
||||
PSIL_ETHERNET(0x4100, 21, 48, 16),
|
||||
PSIL_ETHERNET(0x4101, 22, 64, 16),
|
||||
PSIL_ETHERNET(0x4102, 23, 80, 16),
|
||||
PSIL_ETHERNET(0x4103, 24, 96, 16),
|
||||
/* ICSS_G1 */
|
||||
PSIL_ETHERNET(0x4200, 25, 112, 16),
|
||||
PSIL_ETHERNET(0x4201, 26, 128, 16),
|
||||
PSIL_ETHERNET(0x4202, 27, 144, 16),
|
||||
PSIL_ETHERNET(0x4203, 28, 160, 16),
|
||||
/* PDMA_MAIN0 - SPI0-3 */
|
||||
PSIL_PDMA_XY_PKT(0x4300),
|
||||
PSIL_PDMA_XY_PKT(0x4301),
|
||||
PSIL_PDMA_XY_PKT(0x4302),
|
||||
PSIL_PDMA_XY_PKT(0x4303),
|
||||
PSIL_PDMA_XY_PKT(0x4304),
|
||||
PSIL_PDMA_XY_PKT(0x4305),
|
||||
PSIL_PDMA_XY_PKT(0x4306),
|
||||
PSIL_PDMA_XY_PKT(0x4307),
|
||||
PSIL_PDMA_XY_PKT(0x4308),
|
||||
PSIL_PDMA_XY_PKT(0x4309),
|
||||
PSIL_PDMA_XY_PKT(0x430a),
|
||||
PSIL_PDMA_XY_PKT(0x430b),
|
||||
PSIL_PDMA_XY_PKT(0x430c),
|
||||
PSIL_PDMA_XY_PKT(0x430d),
|
||||
PSIL_PDMA_XY_PKT(0x430e),
|
||||
PSIL_PDMA_XY_PKT(0x430f),
|
||||
/* PDMA_MAIN0 - USART0-1 */
|
||||
PSIL_PDMA_XY_PKT(0x4310),
|
||||
PSIL_PDMA_XY_PKT(0x4311),
|
||||
/* PDMA_MAIN1 - SPI4 */
|
||||
PSIL_PDMA_XY_PKT(0x4400),
|
||||
PSIL_PDMA_XY_PKT(0x4401),
|
||||
PSIL_PDMA_XY_PKT(0x4402),
|
||||
PSIL_PDMA_XY_PKT(0x4403),
|
||||
/* PDMA_MAIN1 - USART2-6 */
|
||||
PSIL_PDMA_XY_PKT(0x4404),
|
||||
PSIL_PDMA_XY_PKT(0x4405),
|
||||
PSIL_PDMA_XY_PKT(0x4406),
|
||||
PSIL_PDMA_XY_PKT(0x4407),
|
||||
PSIL_PDMA_XY_PKT(0x4408),
|
||||
/* PDMA_MAIN1 - ADCs */
|
||||
PSIL_PDMA_XY_TR(0x440f),
|
||||
PSIL_PDMA_XY_TR(0x4410),
|
||||
/* CPSW2 */
|
||||
PSIL_ETHERNET(0x4500, 16, 16, 16),
|
||||
};
|
||||
|
||||
/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */
|
||||
static struct psil_ep am64_dst_ep_map[] = {
|
||||
/* SAUL */
|
||||
PSIL_SAUL(0xc000, 24, 80, 8, 80, 1),
|
||||
PSIL_SAUL(0xc001, 25, 88, 8, 88, 1),
|
||||
/* ICSS_G0 */
|
||||
PSIL_ETHERNET(0xc100, 26, 96, 1),
|
||||
PSIL_ETHERNET(0xc101, 27, 97, 1),
|
||||
PSIL_ETHERNET(0xc102, 28, 98, 1),
|
||||
PSIL_ETHERNET(0xc103, 29, 99, 1),
|
||||
PSIL_ETHERNET(0xc104, 30, 100, 1),
|
||||
PSIL_ETHERNET(0xc105, 31, 101, 1),
|
||||
PSIL_ETHERNET(0xc106, 32, 102, 1),
|
||||
PSIL_ETHERNET(0xc107, 33, 103, 1),
|
||||
/* ICSS_G1 */
|
||||
PSIL_ETHERNET(0xc200, 34, 104, 1),
|
||||
PSIL_ETHERNET(0xc201, 35, 105, 1),
|
||||
PSIL_ETHERNET(0xc202, 36, 106, 1),
|
||||
PSIL_ETHERNET(0xc203, 37, 107, 1),
|
||||
PSIL_ETHERNET(0xc204, 38, 108, 1),
|
||||
PSIL_ETHERNET(0xc205, 39, 109, 1),
|
||||
PSIL_ETHERNET(0xc206, 40, 110, 1),
|
||||
PSIL_ETHERNET(0xc207, 41, 111, 1),
|
||||
/* CPSW2 */
|
||||
PSIL_ETHERNET(0xc500, 16, 16, 8),
|
||||
PSIL_ETHERNET(0xc501, 17, 24, 8),
|
||||
PSIL_ETHERNET(0xc502, 18, 32, 8),
|
||||
PSIL_ETHERNET(0xc503, 19, 40, 8),
|
||||
PSIL_ETHERNET(0xc504, 20, 48, 8),
|
||||
PSIL_ETHERNET(0xc505, 21, 56, 8),
|
||||
PSIL_ETHERNET(0xc506, 22, 64, 8),
|
||||
PSIL_ETHERNET(0xc507, 23, 72, 8),
|
||||
};
|
||||
|
||||
struct psil_ep_map am64_ep_map = {
|
||||
.name = "am64",
|
||||
.src = am64_src_ep_map,
|
||||
.src_count = ARRAY_SIZE(am64_src_ep_map),
|
||||
.dst = am64_dst_ep_map,
|
||||
.dst_count = ARRAY_SIZE(am64_dst_ep_map),
|
||||
};
|
|
@ -40,5 +40,6 @@ struct psil_endpoint_config *psil_get_ep_config(u32 thread_id);
|
|||
extern struct psil_ep_map am654_ep_map;
|
||||
extern struct psil_ep_map j721e_ep_map;
|
||||
extern struct psil_ep_map j7200_ep_map;
|
||||
extern struct psil_ep_map am64_ep_map;
|
||||
|
||||
#endif /* K3_PSIL_PRIV_H_ */
|
||||
|
|
|
@ -20,6 +20,7 @@ static const struct soc_device_attribute k3_soc_devices[] = {
|
|||
{ .family = "AM65X", .data = &am654_ep_map },
|
||||
{ .family = "J721E", .data = &j721e_ep_map },
|
||||
{ .family = "J7200", .data = &j7200_ep_map },
|
||||
{ .family = "AM64X", .data = &am64_ep_map },
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
|
||||
|
|
|
@ -22,6 +22,7 @@
|
|||
|
||||
struct k3_udma_glue_common {
|
||||
struct device *dev;
|
||||
struct device chan_dev;
|
||||
struct udma_dev *udmax;
|
||||
const struct udma_tisci_rm *tisci_rm;
|
||||
struct k3_ringacc *ringacc;
|
||||
|
@ -32,7 +33,8 @@ struct k3_udma_glue_common {
|
|||
bool epib;
|
||||
u32 psdata_size;
|
||||
u32 swdata_size;
|
||||
u32 atype;
|
||||
u32 atype_asel;
|
||||
struct psil_endpoint_config *ep_config;
|
||||
};
|
||||
|
||||
struct k3_udma_glue_tx_channel {
|
||||
|
@ -53,6 +55,8 @@ struct k3_udma_glue_tx_channel {
|
|||
bool tx_filt_einfo;
|
||||
bool tx_filt_pswords;
|
||||
bool tx_supr_tdpkt;
|
||||
|
||||
int udma_tflow_id;
|
||||
};
|
||||
|
||||
struct k3_udma_glue_rx_flow {
|
||||
|
@ -81,20 +85,26 @@ struct k3_udma_glue_rx_channel {
|
|||
u32 flows_ready;
|
||||
};
|
||||
|
||||
static void k3_udma_chan_dev_release(struct device *dev)
|
||||
{
|
||||
/* The struct containing the device is devm managed */
|
||||
}
|
||||
|
||||
static struct class k3_udma_glue_devclass = {
|
||||
.name = "k3_udma_glue_chan",
|
||||
.dev_release = k3_udma_chan_dev_release,
|
||||
};
|
||||
|
||||
#define K3_UDMAX_TDOWN_TIMEOUT_US 1000
|
||||
|
||||
static int of_k3_udma_glue_parse(struct device_node *udmax_np,
|
||||
struct k3_udma_glue_common *common)
|
||||
{
|
||||
common->ringacc = of_k3_ringacc_get_by_phandle(udmax_np,
|
||||
"ti,ringacc");
|
||||
if (IS_ERR(common->ringacc))
|
||||
return PTR_ERR(common->ringacc);
|
||||
|
||||
common->udmax = of_xudma_dev_get(udmax_np, NULL);
|
||||
if (IS_ERR(common->udmax))
|
||||
return PTR_ERR(common->udmax);
|
||||
|
||||
common->ringacc = xudma_get_ringacc(common->udmax);
|
||||
common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
|
||||
|
||||
return 0;
|
||||
|
@ -104,7 +114,6 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
|
|||
const char *name, struct k3_udma_glue_common *common,
|
||||
bool tx_chn)
|
||||
{
|
||||
struct psil_endpoint_config *ep_config;
|
||||
struct of_phandle_args dma_spec;
|
||||
u32 thread_id;
|
||||
int ret = 0;
|
||||
|
@ -121,15 +130,26 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
|
|||
&dma_spec))
|
||||
return -ENOENT;
|
||||
|
||||
ret = of_k3_udma_glue_parse(dma_spec.np, common);
|
||||
if (ret)
|
||||
goto out_put_spec;
|
||||
|
||||
thread_id = dma_spec.args[0];
|
||||
if (dma_spec.args_count == 2) {
|
||||
if (dma_spec.args[1] > 2) {
|
||||
if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
|
||||
dev_err(common->dev, "Invalid channel atype: %u\n",
|
||||
dma_spec.args[1]);
|
||||
ret = -EINVAL;
|
||||
goto out_put_spec;
|
||||
}
|
||||
common->atype = dma_spec.args[1];
|
||||
if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
|
||||
dev_err(common->dev, "Invalid channel asel: %u\n",
|
||||
dma_spec.args[1]);
|
||||
ret = -EINVAL;
|
||||
goto out_put_spec;
|
||||
}
|
||||
|
||||
common->atype_asel = dma_spec.args[1];
|
||||
}
|
||||
|
||||
if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
|
||||
|
@ -143,25 +163,23 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
|
|||
}
|
||||
|
||||
/* get psil endpoint config */
|
||||
ep_config = psil_get_ep_config(thread_id);
|
||||
if (IS_ERR(ep_config)) {
|
||||
common->ep_config = psil_get_ep_config(thread_id);
|
||||
if (IS_ERR(common->ep_config)) {
|
||||
dev_err(common->dev,
|
||||
"No configuration for psi-l thread 0x%04x\n",
|
||||
thread_id);
|
||||
ret = PTR_ERR(ep_config);
|
||||
ret = PTR_ERR(common->ep_config);
|
||||
goto out_put_spec;
|
||||
}
|
||||
|
||||
common->epib = ep_config->needs_epib;
|
||||
common->psdata_size = ep_config->psd_size;
|
||||
common->epib = common->ep_config->needs_epib;
|
||||
common->psdata_size = common->ep_config->psd_size;
|
||||
|
||||
if (tx_chn)
|
||||
common->dst_thread = thread_id;
|
||||
else
|
||||
common->src_thread = thread_id;
|
||||
|
||||
ret = of_k3_udma_glue_parse(dma_spec.np, common);
|
||||
|
||||
out_put_spec:
|
||||
of_node_put(dma_spec.np);
|
||||
return ret;
|
||||
|
@ -227,7 +245,7 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
|
|||
req.tx_supr_tdpkt = 1;
|
||||
req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
|
||||
req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
|
||||
req.tx_atype = tx_chn->common.atype;
|
||||
req.tx_atype = tx_chn->common.atype_asel;
|
||||
|
||||
return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
|
||||
}
|
||||
|
@ -259,8 +277,14 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
|
|||
tx_chn->common.psdata_size,
|
||||
tx_chn->common.swdata_size);
|
||||
|
||||
if (xudma_is_pktdma(tx_chn->common.udmax))
|
||||
tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
|
||||
else
|
||||
tx_chn->udma_tchan_id = -1;
|
||||
|
||||
/* request and cfg UDMAP TX channel */
|
||||
tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax, -1);
|
||||
tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
|
||||
tx_chn->udma_tchan_id);
|
||||
if (IS_ERR(tx_chn->udma_tchanx)) {
|
||||
ret = PTR_ERR(tx_chn->udma_tchanx);
|
||||
dev_err(dev, "UDMAX tchanx get err %d\n", ret);
|
||||
|
@ -268,11 +292,34 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
|
|||
}
|
||||
tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
|
||||
|
||||
tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
|
||||
tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
|
||||
dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
|
||||
tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
|
||||
ret = device_register(&tx_chn->common.chan_dev);
|
||||
if (ret) {
|
||||
dev_err(dev, "Channel Device registration failed %d\n", ret);
|
||||
tx_chn->common.chan_dev.parent = NULL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (xudma_is_pktdma(tx_chn->common.udmax)) {
|
||||
/* prepare the channel device as coherent */
|
||||
tx_chn->common.chan_dev.dma_coherent = true;
|
||||
dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
|
||||
DMA_BIT_MASK(48));
|
||||
}
|
||||
|
||||
atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
|
||||
|
||||
if (xudma_is_pktdma(tx_chn->common.udmax))
|
||||
tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
|
||||
else
|
||||
tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
|
||||
|
||||
/* request and cfg rings */
|
||||
ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
|
||||
tx_chn->udma_tchan_id, -1,
|
||||
tx_chn->udma_tflow_id, -1,
|
||||
&tx_chn->ringtx,
|
||||
&tx_chn->ringtxcq);
|
||||
if (ret) {
|
||||
|
@ -280,6 +327,16 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
|
|||
goto err;
|
||||
}
|
||||
|
||||
/* Set the dma_dev for the rings to be configured */
|
||||
cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
|
||||
cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
|
||||
|
||||
/* Set the ASEL value for DMA rings of PKTDMA */
|
||||
if (xudma_is_pktdma(tx_chn->common.udmax)) {
|
||||
cfg->tx_cfg.asel = tx_chn->common.atype_asel;
|
||||
cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
|
||||
}
|
||||
|
||||
ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
|
||||
if (ret) {
|
||||
dev_err(dev, "Failed to cfg ringtx %d\n", ret);
|
||||
|
@ -303,19 +360,6 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
|
|||
goto err;
|
||||
}
|
||||
|
||||
ret = xudma_navss_psil_pair(tx_chn->common.udmax,
|
||||
tx_chn->common.src_thread,
|
||||
tx_chn->common.dst_thread);
|
||||
if (ret) {
|
||||
dev_err(dev, "PSI-L request err %d\n", ret);
|
||||
goto err;
|
||||
}
|
||||
|
||||
tx_chn->psil_paired = true;
|
||||
|
||||
/* reset TX RT registers */
|
||||
k3_udma_glue_disable_tx_chn(tx_chn);
|
||||
|
||||
k3_udma_glue_dump_tx_chn(tx_chn);
|
||||
|
||||
return tx_chn;
|
||||
|
@ -344,6 +388,11 @@ void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
|
|||
|
||||
if (tx_chn->ringtx)
|
||||
k3_ringacc_ring_free(tx_chn->ringtx);
|
||||
|
||||
if (tx_chn->common.chan_dev.parent) {
|
||||
device_unregister(&tx_chn->common.chan_dev);
|
||||
tx_chn->common.chan_dev.parent = NULL;
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
|
||||
|
||||
|
@ -378,6 +427,18 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
|
|||
|
||||
int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = xudma_navss_psil_pair(tx_chn->common.udmax,
|
||||
tx_chn->common.src_thread,
|
||||
tx_chn->common.dst_thread);
|
||||
if (ret) {
|
||||
dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
tx_chn->psil_paired = true;
|
||||
|
||||
xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
|
||||
UDMA_PEER_RT_EN_ENABLE);
|
||||
|
||||
|
@ -398,6 +459,13 @@ void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
|
|||
xudma_tchanrt_write(tx_chn->udma_tchanx,
|
||||
UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
|
||||
k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
|
||||
|
||||
if (tx_chn->psil_paired) {
|
||||
xudma_navss_psil_unpair(tx_chn->common.udmax,
|
||||
tx_chn->common.src_thread,
|
||||
tx_chn->common.dst_thread);
|
||||
tx_chn->psil_paired = false;
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
|
||||
|
||||
|
@ -437,13 +505,10 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
|
|||
void *data,
|
||||
void (*cleanup)(void *data, dma_addr_t desc_dma))
|
||||
{
|
||||
struct device *dev = tx_chn->common.dev;
|
||||
dma_addr_t desc_dma;
|
||||
int occ_tx, i, ret;
|
||||
|
||||
/* reset TXCQ as it is not input for udma - expected to be empty */
|
||||
if (tx_chn->ringtxcq)
|
||||
k3_ringacc_ring_reset(tx_chn->ringtxcq);
|
||||
|
||||
/*
|
||||
* TXQ reset need to be special way as it is input for udma and its
|
||||
* state cached by udma, so:
|
||||
|
@ -452,17 +517,20 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
|
|||
* 3) reset TXQ in a special way
|
||||
*/
|
||||
occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
|
||||
dev_dbg(tx_chn->common.dev, "TX reset occ_tx %u\n", occ_tx);
|
||||
dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
|
||||
|
||||
for (i = 0; i < occ_tx; i++) {
|
||||
ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
|
||||
if (ret) {
|
||||
dev_err(tx_chn->common.dev, "TX reset pop %d\n", ret);
|
||||
if (ret != -ENODATA)
|
||||
dev_err(dev, "TX reset pop %d\n", ret);
|
||||
break;
|
||||
}
|
||||
cleanup(data, desc_dma);
|
||||
}
|
||||
|
||||
/* reset TXCQ as it is not input for udma - expected to be empty */
|
||||
k3_ringacc_ring_reset(tx_chn->ringtxcq);
|
||||
k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
|
||||
|
@ -481,12 +549,50 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
|
|||
|
||||
int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
|
||||
{
|
||||
tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
|
||||
if (xudma_is_pktdma(tx_chn->common.udmax)) {
|
||||
tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
|
||||
tx_chn->udma_tflow_id);
|
||||
} else {
|
||||
tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
|
||||
}
|
||||
|
||||
return tx_chn->virq;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
|
||||
|
||||
struct device *
|
||||
k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
|
||||
{
|
||||
if (xudma_is_pktdma(tx_chn->common.udmax) &&
|
||||
(tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
|
||||
return &tx_chn->common.chan_dev;
|
||||
|
||||
return xudma_get_device(tx_chn->common.udmax);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
|
||||
|
||||
void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
|
||||
dma_addr_t *addr)
|
||||
{
|
||||
if (!xudma_is_pktdma(tx_chn->common.udmax) ||
|
||||
!tx_chn->common.atype_asel)
|
||||
return;
|
||||
|
||||
*addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
|
||||
|
||||
void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
|
||||
dma_addr_t *addr)
|
||||
{
|
||||
if (!xudma_is_pktdma(tx_chn->common.udmax) ||
|
||||
!tx_chn->common.atype_asel)
|
||||
return;
|
||||
|
||||
*addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
|
||||
|
||||
static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
|
||||
{
|
||||
const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
|
||||
|
@ -498,8 +604,6 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
|
|||
req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
|
||||
TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
|
||||
TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
|
||||
TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
|
||||
TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |
|
||||
TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
|
||||
|
||||
req.nav_id = tisci_rm->tisci_dev_id;
|
||||
|
@ -511,13 +615,16 @@ static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
|
|||
* req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
|
||||
*/
|
||||
req.rxcq_qnum = 0xFFFF;
|
||||
if (rx_chn->flow_num && rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
|
||||
if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
|
||||
rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
|
||||
/* Default flow + extra ones */
|
||||
req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
|
||||
TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
|
||||
req.flowid_start = rx_chn->flow_id_base;
|
||||
req.flowid_cnt = rx_chn->flow_num;
|
||||
}
|
||||
req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
|
||||
req.rx_atype = rx_chn->common.atype;
|
||||
req.rx_atype = rx_chn->common.atype_asel;
|
||||
|
||||
ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
|
||||
if (ret)
|
||||
|
@ -571,10 +678,18 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
|
|||
goto err_rflow_put;
|
||||
}
|
||||
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax)) {
|
||||
rx_ringfdq_id = flow->udma_rflow_id +
|
||||
xudma_get_rflow_ring_offset(rx_chn->common.udmax);
|
||||
rx_ring_id = 0;
|
||||
} else {
|
||||
rx_ring_id = flow_cfg->ring_rxq_id;
|
||||
rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
|
||||
}
|
||||
|
||||
/* request and cfg rings */
|
||||
ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
|
||||
flow_cfg->ring_rxfdq0_id,
|
||||
flow_cfg->ring_rxq_id,
|
||||
rx_ringfdq_id, rx_ring_id,
|
||||
&flow->ringrxfdq,
|
||||
&flow->ringrx);
|
||||
if (ret) {
|
||||
|
@ -582,6 +697,16 @@ static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
|
|||
goto err_rflow_put;
|
||||
}
|
||||
|
||||
/* Set the dma_dev for the rings to be configured */
|
||||
flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
|
||||
flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
|
||||
|
||||
/* Set the ASEL value for DMA rings of PKTDMA */
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax)) {
|
||||
flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
|
||||
flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
|
||||
}
|
||||
|
||||
ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
|
||||
if (ret) {
|
||||
dev_err(dev, "Failed to cfg ringrx %d\n", ret);
|
||||
|
@ -740,6 +865,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
|
|||
struct k3_udma_glue_rx_channel_cfg *cfg)
|
||||
{
|
||||
struct k3_udma_glue_rx_channel *rx_chn;
|
||||
struct psil_endpoint_config *ep_cfg;
|
||||
int ret, i;
|
||||
|
||||
if (cfg->flow_id_num <= 0)
|
||||
|
@ -767,8 +893,16 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
|
|||
rx_chn->common.psdata_size,
|
||||
rx_chn->common.swdata_size);
|
||||
|
||||
ep_cfg = rx_chn->common.ep_config;
|
||||
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax))
|
||||
rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
|
||||
else
|
||||
rx_chn->udma_rchan_id = -1;
|
||||
|
||||
/* request and cfg UDMAP RX channel */
|
||||
rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, -1);
|
||||
rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
|
||||
rx_chn->udma_rchan_id);
|
||||
if (IS_ERR(rx_chn->udma_rchanx)) {
|
||||
ret = PTR_ERR(rx_chn->udma_rchanx);
|
||||
dev_err(dev, "UDMAX rchanx get err %d\n", ret);
|
||||
|
@ -776,12 +910,48 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
|
|||
}
|
||||
rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
|
||||
|
||||
rx_chn->flow_num = cfg->flow_id_num;
|
||||
rx_chn->flow_id_base = cfg->flow_id_base;
|
||||
rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
|
||||
rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
|
||||
dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
|
||||
rx_chn->udma_rchan_id, rx_chn->common.src_thread);
|
||||
ret = device_register(&rx_chn->common.chan_dev);
|
||||
if (ret) {
|
||||
dev_err(dev, "Channel Device registration failed %d\n", ret);
|
||||
rx_chn->common.chan_dev.parent = NULL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
/* Use RX channel id as flow id: target dev can't generate flow_id */
|
||||
if (cfg->flow_id_use_rxchan_id)
|
||||
rx_chn->flow_id_base = rx_chn->udma_rchan_id;
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax)) {
|
||||
/* prepare the channel device as coherent */
|
||||
rx_chn->common.chan_dev.dma_coherent = true;
|
||||
dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
|
||||
DMA_BIT_MASK(48));
|
||||
}
|
||||
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax)) {
|
||||
int flow_start = cfg->flow_id_base;
|
||||
int flow_end;
|
||||
|
||||
if (flow_start == -1)
|
||||
flow_start = ep_cfg->flow_start;
|
||||
|
||||
flow_end = flow_start + cfg->flow_id_num - 1;
|
||||
if (flow_start < ep_cfg->flow_start ||
|
||||
flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
|
||||
dev_err(dev, "Invalid flow range requested\n");
|
||||
ret = -EINVAL;
|
||||
goto err;
|
||||
}
|
||||
rx_chn->flow_id_base = flow_start;
|
||||
} else {
|
||||
rx_chn->flow_id_base = cfg->flow_id_base;
|
||||
|
||||
/* Use RX channel id as flow id: target dev can't generate flow_id */
|
||||
if (cfg->flow_id_use_rxchan_id)
|
||||
rx_chn->flow_id_base = rx_chn->udma_rchan_id;
|
||||
}
|
||||
|
||||
rx_chn->flow_num = cfg->flow_id_num;
|
||||
|
||||
rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
|
||||
sizeof(*rx_chn->flows), GFP_KERNEL);
|
||||
|
@ -815,19 +985,6 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
|
|||
goto err;
|
||||
}
|
||||
|
||||
ret = xudma_navss_psil_pair(rx_chn->common.udmax,
|
||||
rx_chn->common.src_thread,
|
||||
rx_chn->common.dst_thread);
|
||||
if (ret) {
|
||||
dev_err(dev, "PSI-L request err %d\n", ret);
|
||||
goto err;
|
||||
}
|
||||
|
||||
rx_chn->psil_paired = true;
|
||||
|
||||
/* reset RX RT registers */
|
||||
k3_udma_glue_disable_rx_chn(rx_chn);
|
||||
|
||||
k3_udma_glue_dump_rx_chn(rx_chn);
|
||||
|
||||
return rx_chn;
|
||||
|
@ -884,6 +1041,24 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
|
|||
goto err;
|
||||
}
|
||||
|
||||
rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
|
||||
rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
|
||||
dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
|
||||
rx_chn->common.src_thread);
|
||||
ret = device_register(&rx_chn->common.chan_dev);
|
||||
if (ret) {
|
||||
dev_err(dev, "Channel Device registration failed %d\n", ret);
|
||||
rx_chn->common.chan_dev.parent = NULL;
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax)) {
|
||||
/* prepare the channel device as coherent */
|
||||
rx_chn->common.chan_dev.dma_coherent = true;
|
||||
dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
|
||||
DMA_BIT_MASK(48));
|
||||
}
|
||||
|
||||
ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
|
||||
if (ret)
|
||||
goto err;
|
||||
|
@ -936,6 +1111,11 @@ void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
|
|||
if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
|
||||
xudma_rchan_put(rx_chn->common.udmax,
|
||||
rx_chn->udma_rchanx);
|
||||
|
||||
if (rx_chn->common.chan_dev.parent) {
|
||||
device_unregister(&rx_chn->common.chan_dev);
|
||||
rx_chn->common.chan_dev.parent = NULL;
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
|
||||
|
||||
|
@ -1052,12 +1232,24 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
|
|||
|
||||
int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (rx_chn->remote)
|
||||
return -EINVAL;
|
||||
|
||||
if (rx_chn->flows_ready < rx_chn->flow_num)
|
||||
return -EINVAL;
|
||||
|
||||
ret = xudma_navss_psil_pair(rx_chn->common.udmax,
|
||||
rx_chn->common.src_thread,
|
||||
rx_chn->common.dst_thread);
|
||||
if (ret) {
|
||||
dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
rx_chn->psil_paired = true;
|
||||
|
||||
xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
|
||||
UDMA_CHAN_RT_CTL_EN);
|
||||
|
||||
|
@ -1078,6 +1270,13 @@ void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
|
|||
xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
|
||||
|
||||
k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
|
||||
|
||||
if (rx_chn->psil_paired) {
|
||||
xudma_navss_psil_unpair(rx_chn->common.udmax,
|
||||
rx_chn->common.src_thread,
|
||||
rx_chn->common.dst_thread);
|
||||
rx_chn->psil_paired = false;
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
|
||||
|
||||
|
@ -1128,12 +1327,10 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
|
|||
/* reset RXCQ as it is not input for udma - expected to be empty */
|
||||
occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
|
||||
dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
|
||||
if (flow->ringrx)
|
||||
k3_ringacc_ring_reset(flow->ringrx);
|
||||
|
||||
/* Skip RX FDQ in case one FDQ is used for the set of flows */
|
||||
if (skip_fdq)
|
||||
return;
|
||||
goto do_reset;
|
||||
|
||||
/*
|
||||
* RX FDQ reset need to be special way as it is input for udma and its
|
||||
|
@ -1148,13 +1345,17 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
|
|||
for (i = 0; i < occ_rx; i++) {
|
||||
ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
|
||||
if (ret) {
|
||||
dev_err(dev, "RX reset pop %d\n", ret);
|
||||
if (ret != -ENODATA)
|
||||
dev_err(dev, "RX reset pop %d\n", ret);
|
||||
break;
|
||||
}
|
||||
cleanup(data, desc_dma);
|
||||
}
|
||||
|
||||
k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
|
||||
|
||||
do_reset:
|
||||
k3_ringacc_ring_reset(flow->ringrx);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
|
||||
|
||||
|
@ -1184,8 +1385,52 @@ int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
|
|||
|
||||
flow = &rx_chn->flows[flow_num];
|
||||
|
||||
flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax)) {
|
||||
flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
|
||||
flow->udma_rflow_id);
|
||||
} else {
|
||||
flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
|
||||
}
|
||||
|
||||
return flow->virq;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
|
||||
|
||||
struct device *
|
||||
k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
|
||||
{
|
||||
if (xudma_is_pktdma(rx_chn->common.udmax) &&
|
||||
(rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
|
||||
return &rx_chn->common.chan_dev;
|
||||
|
||||
return xudma_get_device(rx_chn->common.udmax);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
|
||||
|
||||
void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
|
||||
dma_addr_t *addr)
|
||||
{
|
||||
if (!xudma_is_pktdma(rx_chn->common.udmax) ||
|
||||
!rx_chn->common.atype_asel)
|
||||
return;
|
||||
|
||||
*addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
|
||||
|
||||
void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
|
||||
dma_addr_t *addr)
|
||||
{
|
||||
if (!xudma_is_pktdma(rx_chn->common.udmax) ||
|
||||
!rx_chn->common.atype_asel)
|
||||
return;
|
||||
|
||||
*addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
|
||||
|
||||
static int __init k3_udma_glue_class_init(void)
|
||||
{
|
||||
return class_register(&k3_udma_glue_devclass);
|
||||
}
|
||||
arch_initcall(k3_udma_glue_class_init);
|
||||
|
|
|
@ -50,6 +50,18 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property)
|
|||
}
|
||||
EXPORT_SYMBOL(of_xudma_dev_get);
|
||||
|
||||
struct device *xudma_get_device(struct udma_dev *ud)
|
||||
{
|
||||
return ud->dev;
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_get_device);
|
||||
|
||||
struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud)
|
||||
{
|
||||
return ud->ringacc;
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_get_ringacc);
|
||||
|
||||
u32 xudma_dev_get_psil_base(struct udma_dev *ud)
|
||||
{
|
||||
return ud->psil_base;
|
||||
|
@ -76,6 +88,9 @@ EXPORT_SYMBOL(xudma_free_gp_rflow_range);
|
|||
|
||||
bool xudma_rflow_is_gp(struct udma_dev *ud, int id)
|
||||
{
|
||||
if (!ud->rflow_gp_map)
|
||||
return false;
|
||||
|
||||
return !test_bit(id, ud->rflow_gp_map);
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_rflow_is_gp);
|
||||
|
@ -107,6 +122,12 @@ void xudma_rflow_put(struct udma_dev *ud, struct udma_rflow *p)
|
|||
}
|
||||
EXPORT_SYMBOL(xudma_rflow_put);
|
||||
|
||||
int xudma_get_rflow_ring_offset(struct udma_dev *ud)
|
||||
{
|
||||
return ud->tflow_cnt;
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_get_rflow_ring_offset);
|
||||
|
||||
#define XUDMA_GET_RESOURCE_ID(res) \
|
||||
int xudma_##res##_get_id(struct udma_##res *p) \
|
||||
{ \
|
||||
|
@ -136,3 +157,27 @@ void xudma_##res##rt_write(struct udma_##res *p, int reg, u32 val) \
|
|||
EXPORT_SYMBOL(xudma_##res##rt_write)
|
||||
XUDMA_RT_IO_FUNCTIONS(tchan);
|
||||
XUDMA_RT_IO_FUNCTIONS(rchan);
|
||||
|
||||
int xudma_is_pktdma(struct udma_dev *ud)
|
||||
{
|
||||
return ud->match_data->type == DMA_TYPE_PKTDMA;
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_is_pktdma);
|
||||
|
||||
int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id)
|
||||
{
|
||||
const struct udma_oes_offsets *oes = &ud->soc_data->oes;
|
||||
|
||||
return ti_sci_inta_msi_get_virq(ud->dev, udma_tflow_id +
|
||||
oes->pktdma_tchan_flow);
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_pktdma_tflow_get_irq);
|
||||
|
||||
int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id)
|
||||
{
|
||||
const struct udma_oes_offsets *oes = &ud->soc_data->oes;
|
||||
|
||||
return ti_sci_inta_msi_get_virq(ud->dev, udma_rflow_id +
|
||||
oes->pktdma_rchan_flow);
|
||||
}
|
||||
EXPORT_SYMBOL(xudma_pktdma_rflow_get_irq);
|
||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -18,7 +18,7 @@
|
|||
#define UDMA_RX_FLOW_ID_FW_OES_REG 0x80
|
||||
#define UDMA_RX_FLOW_ID_FW_STATUS_REG 0x88
|
||||
|
||||
/* TCHANRT/RCHANRT registers */
|
||||
/* BCHANRT/TCHANRT/RCHANRT registers */
|
||||
#define UDMA_CHAN_RT_CTL_REG 0x0
|
||||
#define UDMA_CHAN_RT_SWTRIG_REG 0x8
|
||||
#define UDMA_CHAN_RT_STDATA_REG 0x80
|
||||
|
@ -45,6 +45,18 @@
|
|||
#define UDMA_CAP3_HCHAN_CNT(val) (((val) >> 14) & 0x1ff)
|
||||
#define UDMA_CAP3_UCHAN_CNT(val) (((val) >> 23) & 0x1ff)
|
||||
|
||||
#define BCDMA_CAP2_BCHAN_CNT(val) ((val) & 0x1ff)
|
||||
#define BCDMA_CAP2_TCHAN_CNT(val) (((val) >> 9) & 0x1ff)
|
||||
#define BCDMA_CAP2_RCHAN_CNT(val) (((val) >> 18) & 0x1ff)
|
||||
#define BCDMA_CAP3_HBCHAN_CNT(val) (((val) >> 14) & 0x1ff)
|
||||
#define BCDMA_CAP3_UBCHAN_CNT(val) (((val) >> 23) & 0x1ff)
|
||||
#define BCDMA_CAP4_HRCHAN_CNT(val) ((val) & 0xff)
|
||||
#define BCDMA_CAP4_URCHAN_CNT(val) (((val) >> 8) & 0xff)
|
||||
#define BCDMA_CAP4_HTCHAN_CNT(val) (((val) >> 16) & 0xff)
|
||||
#define BCDMA_CAP4_UTCHAN_CNT(val) (((val) >> 24) & 0xff)
|
||||
|
||||
#define PKTDMA_CAP4_TFLOW_CNT(val) ((val) & 0x3fff)
|
||||
|
||||
/* UDMA_CHAN_RT_CTL_REG */
|
||||
#define UDMA_CHAN_RT_CTL_EN BIT(31)
|
||||
#define UDMA_CHAN_RT_CTL_TDOWN BIT(30)
|
||||
|
@ -82,15 +94,20 @@
|
|||
*/
|
||||
#define PDMA_STATIC_TR_Z(x, mask) ((x) & (mask))
|
||||
|
||||
/* Address Space Select */
|
||||
#define K3_ADDRESS_ASEL_SHIFT 48
|
||||
|
||||
struct udma_dev;
|
||||
struct udma_tchan;
|
||||
struct udma_rchan;
|
||||
struct udma_rflow;
|
||||
|
||||
enum udma_rm_range {
|
||||
RM_RANGE_TCHAN = 0,
|
||||
RM_RANGE_BCHAN = 0,
|
||||
RM_RANGE_TCHAN,
|
||||
RM_RANGE_RCHAN,
|
||||
RM_RANGE_RFLOW,
|
||||
RM_RANGE_TFLOW,
|
||||
RM_RANGE_LAST,
|
||||
};
|
||||
|
||||
|
@ -112,6 +129,8 @@ int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
|
|||
u32 dst_thread);
|
||||
|
||||
struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property);
|
||||
struct device *xudma_get_device(struct udma_dev *ud);
|
||||
struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud);
|
||||
void xudma_dev_put(struct udma_dev *ud);
|
||||
u32 xudma_dev_get_psil_base(struct udma_dev *ud);
|
||||
struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud);
|
||||
|
@ -136,5 +155,10 @@ void xudma_tchanrt_write(struct udma_tchan *tchan, int reg, u32 val);
|
|||
u32 xudma_rchanrt_read(struct udma_rchan *rchan, int reg);
|
||||
void xudma_rchanrt_write(struct udma_rchan *rchan, int reg, u32 val);
|
||||
bool xudma_rflow_is_gp(struct udma_dev *ud, int id);
|
||||
int xudma_get_rflow_ring_offset(struct udma_dev *ud);
|
||||
|
||||
int xudma_is_pktdma(struct udma_dev *ud);
|
||||
|
||||
int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id);
|
||||
int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id);
|
||||
#endif /* K3_UDMA_H_ */
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
#include <linux/of.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/sys_soc.h>
|
||||
#include <linux/dma/ti-cppi5.h>
|
||||
#include <linux/soc/ti/k3-ringacc.h>
|
||||
#include <linux/soc/ti/ti_sci_protocol.h>
|
||||
#include <linux/soc/ti/ti_sci_inta_msi.h>
|
||||
|
@ -21,6 +22,7 @@ static LIST_HEAD(k3_ringacc_list);
|
|||
static DEFINE_MUTEX(k3_ringacc_list_lock);
|
||||
|
||||
#define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
|
||||
#define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0)
|
||||
|
||||
/**
|
||||
* struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
|
||||
|
@ -43,7 +45,13 @@ struct k3_ring_rt_regs {
|
|||
u32 hwindx;
|
||||
};
|
||||
|
||||
#define K3_RINGACC_RT_REGS_STEP 0x1000
|
||||
#define K3_RINGACC_RT_REGS_STEP 0x1000
|
||||
#define K3_DMARING_RT_REGS_STEP 0x2000
|
||||
#define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000
|
||||
#define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0)
|
||||
#define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31)
|
||||
#define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0)
|
||||
#define K3_DMARING_RT_DB_TDOWN_ACK BIT(31)
|
||||
|
||||
/**
|
||||
* struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
|
||||
|
@ -122,6 +130,7 @@ struct k3_ring_state {
|
|||
u32 occ;
|
||||
u32 windex;
|
||||
u32 rindex;
|
||||
u32 tdown_complete:1;
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -143,6 +152,7 @@ struct k3_ring_state {
|
|||
* @use_count: Use count for shared rings
|
||||
* @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
|
||||
* @dma_dev: device to be used for DMA API (allocation, mapping)
|
||||
* @asel: Address Space Select value for physical addresses
|
||||
*/
|
||||
struct k3_ring {
|
||||
struct k3_ring_rt_regs __iomem *rt;
|
||||
|
@ -157,12 +167,15 @@ struct k3_ring {
|
|||
u32 flags;
|
||||
#define K3_RING_FLAG_BUSY BIT(1)
|
||||
#define K3_RING_FLAG_SHARED BIT(2)
|
||||
#define K3_RING_FLAG_REVERSE BIT(3)
|
||||
struct k3_ring_state state;
|
||||
u32 ring_id;
|
||||
struct k3_ringacc *parent;
|
||||
u32 use_count;
|
||||
int proxy_id;
|
||||
struct device *dma_dev;
|
||||
u32 asel;
|
||||
#define K3_ADDRESS_ASEL_SHIFT 48
|
||||
};
|
||||
|
||||
struct k3_ringacc_ops {
|
||||
|
@ -188,6 +201,7 @@ struct k3_ringacc_ops {
|
|||
* @tisci_ring_ops: ti-sci rings ops
|
||||
* @tisci_dev_id: ti-sci device id
|
||||
* @ops: SoC specific ringacc operation
|
||||
* @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
|
||||
*/
|
||||
struct k3_ringacc {
|
||||
struct device *dev;
|
||||
|
@ -210,6 +224,7 @@ struct k3_ringacc {
|
|||
u32 tisci_dev_id;
|
||||
|
||||
const struct k3_ringacc_ops *ops;
|
||||
bool dma_rings;
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -221,6 +236,21 @@ struct k3_ringacc_soc_data {
|
|||
unsigned dma_ring_reset_quirk:1;
|
||||
};
|
||||
|
||||
static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
|
||||
{
|
||||
return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
|
||||
}
|
||||
|
||||
static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
|
||||
{
|
||||
u32 val;
|
||||
|
||||
val = readl(&ring->rt->occ);
|
||||
|
||||
ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
|
||||
ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
|
||||
}
|
||||
|
||||
static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
|
||||
{
|
||||
return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
|
||||
|
@ -234,12 +264,24 @@ static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
|
|||
|
||||
static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
|
||||
static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
|
||||
static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
|
||||
static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
|
||||
|
||||
static struct k3_ring_ops k3_ring_mode_ring_ops = {
|
||||
.push_tail = k3_ringacc_ring_push_mem,
|
||||
.pop_head = k3_ringacc_ring_pop_mem,
|
||||
};
|
||||
|
||||
static struct k3_ring_ops k3_dmaring_fwd_ops = {
|
||||
.push_tail = k3_ringacc_ring_push_mem,
|
||||
.pop_head = k3_dmaring_fwd_pop,
|
||||
};
|
||||
|
||||
static struct k3_ring_ops k3_dmaring_reverse_ops = {
|
||||
/* Reverse side of the DMA ring can only be popped by SW */
|
||||
.pop_head = k3_dmaring_reverse_pop,
|
||||
};
|
||||
|
||||
static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
|
||||
static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
|
||||
static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
|
||||
|
@ -342,6 +384,40 @@ struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
|
||||
|
||||
static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
|
||||
struct k3_ring **fwd_ring,
|
||||
struct k3_ring **compl_ring)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
/*
|
||||
* DMA rings must be requested by ID, completion ring is the reverse
|
||||
* side of the forward ring
|
||||
*/
|
||||
if (fwd_id < 0)
|
||||
return -EINVAL;
|
||||
|
||||
mutex_lock(&ringacc->req_lock);
|
||||
|
||||
if (test_bit(fwd_id, ringacc->rings_inuse)) {
|
||||
ret = -EBUSY;
|
||||
goto error;
|
||||
}
|
||||
|
||||
*fwd_ring = &ringacc->rings[fwd_id];
|
||||
*compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
|
||||
set_bit(fwd_id, ringacc->rings_inuse);
|
||||
ringacc->rings[fwd_id].use_count++;
|
||||
dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
|
||||
|
||||
mutex_unlock(&ringacc->req_lock);
|
||||
return 0;
|
||||
|
||||
error:
|
||||
mutex_unlock(&ringacc->req_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
|
||||
int fwd_id, int compl_id,
|
||||
struct k3_ring **fwd_ring,
|
||||
|
@ -352,6 +428,10 @@ int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
|
|||
if (!fwd_ring || !compl_ring)
|
||||
return -EINVAL;
|
||||
|
||||
if (ringacc->dma_rings)
|
||||
return k3_dmaring_request_dual_ring(ringacc, fwd_id,
|
||||
fwd_ring, compl_ring);
|
||||
|
||||
*fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
|
||||
if (!(*fwd_ring))
|
||||
return -ENODEV;
|
||||
|
@ -421,7 +501,7 @@ void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
|
|||
goto reset;
|
||||
|
||||
if (!occ)
|
||||
occ = readl(&ring->rt->occ);
|
||||
occ = k3_ringacc_ring_read_occ(ring);
|
||||
|
||||
if (occ) {
|
||||
u32 db_ring_cnt, db_ring_cnt_cur;
|
||||
|
@ -496,6 +576,13 @@ int k3_ringacc_ring_free(struct k3_ring *ring)
|
|||
|
||||
ringacc = ring->parent;
|
||||
|
||||
/*
|
||||
* DMA rings: rings shared memory and configuration, only forward ring
|
||||
* is configured and reverse ring considered as slave.
|
||||
*/
|
||||
if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
|
||||
return 0;
|
||||
|
||||
dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
|
||||
|
||||
if (!test_bit(ring->ring_id, ringacc->rings_inuse))
|
||||
|
@ -517,6 +604,8 @@ int k3_ringacc_ring_free(struct k3_ring *ring)
|
|||
ring->flags = 0;
|
||||
ring->ops = NULL;
|
||||
ring->dma_dev = NULL;
|
||||
ring->asel = 0;
|
||||
|
||||
if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
|
||||
clear_bit(ring->proxy_id, ringacc->proxy_inuse);
|
||||
ring->proxy = NULL;
|
||||
|
@ -581,6 +670,7 @@ static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
|
|||
ring_cfg.count = ring->size;
|
||||
ring_cfg.mode = ring->mode;
|
||||
ring_cfg.size = ring->elm_size;
|
||||
ring_cfg.asel = ring->asel;
|
||||
|
||||
ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
|
||||
if (ret)
|
||||
|
@ -590,6 +680,90 @@ static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
|
||||
{
|
||||
struct k3_ringacc *ringacc;
|
||||
struct k3_ring *reverse_ring;
|
||||
int ret = 0;
|
||||
|
||||
if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
|
||||
cfg->mode != K3_RINGACC_RING_MODE_RING ||
|
||||
cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
|
||||
return -EINVAL;
|
||||
|
||||
ringacc = ring->parent;
|
||||
|
||||
/*
|
||||
* DMA rings: rings shared memory and configuration, only forward ring
|
||||
* is configured and reverse ring considered as slave.
|
||||
*/
|
||||
if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
|
||||
return 0;
|
||||
|
||||
if (!test_bit(ring->ring_id, ringacc->rings_inuse))
|
||||
return -EINVAL;
|
||||
|
||||
ring->size = cfg->size;
|
||||
ring->elm_size = cfg->elm_size;
|
||||
ring->mode = cfg->mode;
|
||||
ring->asel = cfg->asel;
|
||||
ring->dma_dev = cfg->dma_dev;
|
||||
if (!ring->dma_dev) {
|
||||
dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
|
||||
ring->ring_id);
|
||||
ring->dma_dev = ringacc->dev;
|
||||
}
|
||||
|
||||
memset(&ring->state, 0, sizeof(ring->state));
|
||||
|
||||
ring->ops = &k3_dmaring_fwd_ops;
|
||||
|
||||
ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
|
||||
ring->size * (4 << ring->elm_size),
|
||||
&ring->ring_mem_dma, GFP_KERNEL);
|
||||
if (!ring->ring_mem_virt) {
|
||||
dev_err(ringacc->dev, "Failed to alloc ring mem\n");
|
||||
ret = -ENOMEM;
|
||||
goto err_free_ops;
|
||||
}
|
||||
|
||||
ret = k3_ringacc_ring_cfg_sci(ring);
|
||||
if (ret)
|
||||
goto err_free_mem;
|
||||
|
||||
ring->flags |= K3_RING_FLAG_BUSY;
|
||||
|
||||
k3_ringacc_ring_dump(ring);
|
||||
|
||||
/* DMA rings: configure reverse ring */
|
||||
reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
|
||||
reverse_ring->size = cfg->size;
|
||||
reverse_ring->elm_size = cfg->elm_size;
|
||||
reverse_ring->mode = cfg->mode;
|
||||
reverse_ring->asel = cfg->asel;
|
||||
memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
|
||||
reverse_ring->ops = &k3_dmaring_reverse_ops;
|
||||
|
||||
reverse_ring->ring_mem_virt = ring->ring_mem_virt;
|
||||
reverse_ring->ring_mem_dma = ring->ring_mem_dma;
|
||||
reverse_ring->flags |= K3_RING_FLAG_BUSY;
|
||||
k3_ringacc_ring_dump(reverse_ring);
|
||||
|
||||
return 0;
|
||||
|
||||
err_free_mem:
|
||||
dma_free_coherent(ring->dma_dev,
|
||||
ring->size * (4 << ring->elm_size),
|
||||
ring->ring_mem_virt,
|
||||
ring->ring_mem_dma);
|
||||
err_free_ops:
|
||||
ring->ops = NULL;
|
||||
ring->proxy = NULL;
|
||||
ring->dma_dev = NULL;
|
||||
ring->asel = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
|
||||
{
|
||||
struct k3_ringacc *ringacc;
|
||||
|
@ -597,8 +771,12 @@ int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
|
|||
|
||||
if (!ring || !cfg)
|
||||
return -EINVAL;
|
||||
|
||||
ringacc = ring->parent;
|
||||
|
||||
if (ringacc->dma_rings)
|
||||
return k3_dmaring_cfg(ring, cfg);
|
||||
|
||||
if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
|
||||
cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
|
||||
cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
|
||||
|
@ -705,7 +883,7 @@ u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
|
|||
return -EINVAL;
|
||||
|
||||
if (!ring->state.free)
|
||||
ring->state.free = ring->size - readl(&ring->rt->occ);
|
||||
ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
|
||||
|
||||
return ring->state.free;
|
||||
}
|
||||
|
@ -716,7 +894,7 @@ u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
|
|||
if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
|
||||
return -EINVAL;
|
||||
|
||||
return readl(&ring->rt->occ);
|
||||
return k3_ringacc_ring_read_occ(ring);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
|
||||
|
||||
|
@ -892,6 +1070,72 @@ static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
|
|||
K3_RINGACC_ACCESS_MODE_POP_HEAD);
|
||||
}
|
||||
|
||||
/*
|
||||
* The element is 48 bits of address + ASEL bits in the ring.
|
||||
* ASEL is used by the DMAs and should be removed for the kernel as it is not
|
||||
* part of the physical memory address.
|
||||
*/
|
||||
static void k3_dmaring_remove_asel_from_elem(u64 *elem)
|
||||
{
|
||||
*elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
|
||||
}
|
||||
|
||||
static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
|
||||
{
|
||||
void *elem_ptr;
|
||||
u32 elem_idx;
|
||||
|
||||
/*
|
||||
* DMA rings: forward ring is always tied DMA channel and HW does not
|
||||
* maintain any state data required for POP operation and its unknown
|
||||
* how much elements were consumed by HW. So, to actually
|
||||
* do POP, the read pointer has to be recalculated every time.
|
||||
*/
|
||||
ring->state.occ = k3_ringacc_ring_read_occ(ring);
|
||||
if (ring->state.windex >= ring->state.occ)
|
||||
elem_idx = ring->state.windex - ring->state.occ;
|
||||
else
|
||||
elem_idx = ring->size - (ring->state.occ - ring->state.windex);
|
||||
|
||||
elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
|
||||
memcpy(elem, elem_ptr, (4 << ring->elm_size));
|
||||
k3_dmaring_remove_asel_from_elem(elem);
|
||||
|
||||
ring->state.occ--;
|
||||
writel(-1, &ring->rt->db);
|
||||
|
||||
dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
|
||||
__func__, ring->state.occ, ring->state.windex, elem_idx,
|
||||
elem_ptr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
|
||||
{
|
||||
void *elem_ptr;
|
||||
|
||||
elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
|
||||
|
||||
if (ring->state.occ) {
|
||||
memcpy(elem, elem_ptr, (4 << ring->elm_size));
|
||||
k3_dmaring_remove_asel_from_elem(elem);
|
||||
|
||||
ring->state.rindex = (ring->state.rindex + 1) % ring->size;
|
||||
ring->state.occ--;
|
||||
writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
|
||||
} else if (ring->state.tdown_complete) {
|
||||
dma_addr_t *value = elem;
|
||||
|
||||
*value = CPPI5_TDCM_MARKER;
|
||||
writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
|
||||
ring->state.tdown_complete = false;
|
||||
}
|
||||
|
||||
dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
|
||||
__func__, ring->state.occ, ring->state.rindex, elem_ptr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
|
||||
{
|
||||
void *elem_ptr;
|
||||
|
@ -899,6 +1143,11 @@ static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
|
|||
elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
|
||||
|
||||
memcpy(elem_ptr, elem, (4 << ring->elm_size));
|
||||
if (ring->parent->dma_rings) {
|
||||
u64 *addr = elem_ptr;
|
||||
|
||||
*addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
|
||||
}
|
||||
|
||||
ring->state.windex = (ring->state.windex + 1) % ring->size;
|
||||
ring->state.free--;
|
||||
|
@ -975,12 +1224,12 @@ int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
|
|||
return -EINVAL;
|
||||
|
||||
if (!ring->state.occ)
|
||||
ring->state.occ = k3_ringacc_ring_get_occ(ring);
|
||||
k3_ringacc_ring_update_occ(ring);
|
||||
|
||||
dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
|
||||
ring->state.rindex);
|
||||
|
||||
if (!ring->state.occ)
|
||||
if (!ring->state.occ && !ring->state.tdown_complete)
|
||||
return -ENODATA;
|
||||
|
||||
if (ring->ops && ring->ops->pop_head)
|
||||
|
@ -998,7 +1247,7 @@ int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
|
|||
return -EINVAL;
|
||||
|
||||
if (!ring->state.occ)
|
||||
ring->state.occ = k3_ringacc_ring_get_occ(ring);
|
||||
k3_ringacc_ring_update_occ(ring);
|
||||
|
||||
dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
|
||||
ring->state.occ, ring->state.rindex);
|
||||
|
@ -1203,6 +1452,68 @@ static const struct of_device_id k3_ringacc_of_match[] = {
|
|||
{},
|
||||
};
|
||||
|
||||
struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
|
||||
struct k3_ringacc_init_data *data)
|
||||
{
|
||||
struct device *dev = &pdev->dev;
|
||||
struct k3_ringacc *ringacc;
|
||||
void __iomem *base_rt;
|
||||
struct resource *res;
|
||||
int i;
|
||||
|
||||
ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
|
||||
if (!ringacc)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
ringacc->dev = dev;
|
||||
ringacc->dma_rings = true;
|
||||
ringacc->num_rings = data->num_rings;
|
||||
ringacc->tisci = data->tisci;
|
||||
ringacc->tisci_dev_id = data->tisci_dev_id;
|
||||
|
||||
mutex_init(&ringacc->req_lock);
|
||||
|
||||
res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt");
|
||||
base_rt = devm_ioremap_resource(dev, res);
|
||||
if (IS_ERR(base_rt))
|
||||
return ERR_CAST(base_rt);
|
||||
|
||||
ringacc->rings = devm_kzalloc(dev,
|
||||
sizeof(*ringacc->rings) *
|
||||
ringacc->num_rings * 2,
|
||||
GFP_KERNEL);
|
||||
ringacc->rings_inuse = devm_kcalloc(dev,
|
||||
BITS_TO_LONGS(ringacc->num_rings),
|
||||
sizeof(unsigned long), GFP_KERNEL);
|
||||
|
||||
if (!ringacc->rings || !ringacc->rings_inuse)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
for (i = 0; i < ringacc->num_rings; i++) {
|
||||
struct k3_ring *ring = &ringacc->rings[i];
|
||||
|
||||
ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
|
||||
ring->parent = ringacc;
|
||||
ring->ring_id = i;
|
||||
ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
|
||||
|
||||
ring = &ringacc->rings[ringacc->num_rings + i];
|
||||
ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
|
||||
K3_DMARING_RT_REGS_REVERSE_OFS;
|
||||
ring->parent = ringacc;
|
||||
ring->ring_id = i;
|
||||
ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
|
||||
ring->flags = K3_RING_FLAG_REVERSE;
|
||||
}
|
||||
|
||||
ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
|
||||
|
||||
dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
|
||||
|
||||
return ringacc;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
|
||||
|
||||
static int k3_ringacc_probe(struct platform_device *pdev)
|
||||
{
|
||||
const struct ringacc_match_data *match_data;
|
||||
|
|
44
include/dt-bindings/dma/jz4775-dma.h
Normal file
44
include/dt-bindings/dma/jz4775-dma.h
Normal file
|
@ -0,0 +1,44 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* This header provides macros for JZ4775 DMA bindings.
|
||||
*
|
||||
* Copyright (c) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
|
||||
*/
|
||||
|
||||
#ifndef __DT_BINDINGS_DMA_JZ4775_DMA_H__
|
||||
#define __DT_BINDINGS_DMA_JZ4775_DMA_H__
|
||||
|
||||
/*
|
||||
* Request type numbers for the JZ4775 DMA controller (written to the DRTn
|
||||
* register for the channel).
|
||||
*/
|
||||
#define JZ4775_DMA_I2S0_TX 0x6
|
||||
#define JZ4775_DMA_I2S0_RX 0x7
|
||||
#define JZ4775_DMA_AUTO 0x8
|
||||
#define JZ4775_DMA_SADC_RX 0x9
|
||||
#define JZ4775_DMA_UART3_TX 0x0e
|
||||
#define JZ4775_DMA_UART3_RX 0x0f
|
||||
#define JZ4775_DMA_UART2_TX 0x10
|
||||
#define JZ4775_DMA_UART2_RX 0x11
|
||||
#define JZ4775_DMA_UART1_TX 0x12
|
||||
#define JZ4775_DMA_UART1_RX 0x13
|
||||
#define JZ4775_DMA_UART0_TX 0x14
|
||||
#define JZ4775_DMA_UART0_RX 0x15
|
||||
#define JZ4775_DMA_SSI0_TX 0x16
|
||||
#define JZ4775_DMA_SSI0_RX 0x17
|
||||
#define JZ4775_DMA_MSC0_TX 0x1a
|
||||
#define JZ4775_DMA_MSC0_RX 0x1b
|
||||
#define JZ4775_DMA_MSC1_TX 0x1c
|
||||
#define JZ4775_DMA_MSC1_RX 0x1d
|
||||
#define JZ4775_DMA_MSC2_TX 0x1e
|
||||
#define JZ4775_DMA_MSC2_RX 0x1f
|
||||
#define JZ4775_DMA_PCM0_TX 0x20
|
||||
#define JZ4775_DMA_PCM0_RX 0x21
|
||||
#define JZ4775_DMA_SMB0_TX 0x24
|
||||
#define JZ4775_DMA_SMB0_RX 0x25
|
||||
#define JZ4775_DMA_SMB1_TX 0x26
|
||||
#define JZ4775_DMA_SMB1_RX 0x27
|
||||
#define JZ4775_DMA_SMB2_TX 0x28
|
||||
#define JZ4775_DMA_SMB2_RX 0x29
|
||||
|
||||
#endif /* __DT_BINDINGS_DMA_JZ4775_DMA_H__ */
|
11
include/dt-bindings/dma/qcom-gpi.h
Normal file
11
include/dt-bindings/dma/qcom-gpi.h
Normal file
|
@ -0,0 +1,11 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
|
||||
/* Copyright (c) 2020, Linaro Ltd. */
|
||||
|
||||
#ifndef __DT_BINDINGS_DMA_QCOM_GPI_H__
|
||||
#define __DT_BINDINGS_DMA_QCOM_GPI_H__
|
||||
|
||||
#define QCOM_GPI_SPI 1
|
||||
#define QCOM_GPI_UART 2
|
||||
#define QCOM_GPI_I2C 3
|
||||
|
||||
#endif /* __DT_BINDINGS_DMA_QCOM_GPI_H__ */
|
54
include/dt-bindings/dma/x2000-dma.h
Normal file
54
include/dt-bindings/dma/x2000-dma.h
Normal file
|
@ -0,0 +1,54 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* This header provides macros for X2000 DMA bindings.
|
||||
*
|
||||
* Copyright (c) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
|
||||
*/
|
||||
|
||||
#ifndef __DT_BINDINGS_DMA_X2000_DMA_H__
|
||||
#define __DT_BINDINGS_DMA_X2000_DMA_H__
|
||||
|
||||
/*
|
||||
* Request type numbers for the X2000 DMA controller (written to the DRTn
|
||||
* register for the channel).
|
||||
*/
|
||||
#define X2000_DMA_AUTO 0x8
|
||||
#define X2000_DMA_UART5_TX 0xa
|
||||
#define X2000_DMA_UART5_RX 0xb
|
||||
#define X2000_DMA_UART4_TX 0xc
|
||||
#define X2000_DMA_UART4_RX 0xd
|
||||
#define X2000_DMA_UART3_TX 0xe
|
||||
#define X2000_DMA_UART3_RX 0xf
|
||||
#define X2000_DMA_UART2_TX 0x10
|
||||
#define X2000_DMA_UART2_RX 0x11
|
||||
#define X2000_DMA_UART1_TX 0x12
|
||||
#define X2000_DMA_UART1_RX 0x13
|
||||
#define X2000_DMA_UART0_TX 0x14
|
||||
#define X2000_DMA_UART0_RX 0x15
|
||||
#define X2000_DMA_SSI0_TX 0x16
|
||||
#define X2000_DMA_SSI0_RX 0x17
|
||||
#define X2000_DMA_SSI1_TX 0x18
|
||||
#define X2000_DMA_SSI1_RX 0x19
|
||||
#define X2000_DMA_I2C0_TX 0x24
|
||||
#define X2000_DMA_I2C0_RX 0x25
|
||||
#define X2000_DMA_I2C1_TX 0x26
|
||||
#define X2000_DMA_I2C1_RX 0x27
|
||||
#define X2000_DMA_I2C2_TX 0x28
|
||||
#define X2000_DMA_I2C2_RX 0x29
|
||||
#define X2000_DMA_I2C3_TX 0x2a
|
||||
#define X2000_DMA_I2C3_RX 0x2b
|
||||
#define X2000_DMA_I2C4_TX 0x2c
|
||||
#define X2000_DMA_I2C4_RX 0x2d
|
||||
#define X2000_DMA_I2C5_TX 0x2e
|
||||
#define X2000_DMA_I2C5_RX 0x2f
|
||||
#define X2000_DMA_UART6_TX 0x30
|
||||
#define X2000_DMA_UART6_RX 0x31
|
||||
#define X2000_DMA_UART7_TX 0x32
|
||||
#define X2000_DMA_UART7_RX 0x33
|
||||
#define X2000_DMA_UART8_TX 0x34
|
||||
#define X2000_DMA_UART8_RX 0x35
|
||||
#define X2000_DMA_UART9_TX 0x36
|
||||
#define X2000_DMA_UART9_RX 0x37
|
||||
#define X2000_DMA_SADC_RX 0x38
|
||||
|
||||
#endif /* __DT_BINDINGS_DMA_X2000_DMA_H__ */
|
16
include/linux/dma/k3-event-router.h
Normal file
16
include/linux/dma/k3-event-router.h
Normal file
|
@ -0,0 +1,16 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com
|
||||
*/
|
||||
|
||||
#ifndef K3_EVENT_ROUTER_
|
||||
#define K3_EVENT_ROUTER_
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
struct k3_event_route_data {
|
||||
void *priv;
|
||||
int (*set_event)(void *priv, u32 event);
|
||||
};
|
||||
|
||||
#endif /* K3_EVENT_ROUTER_ */
|
|
@ -50,6 +50,15 @@ enum psil_endpoint_type {
|
|||
* @channel_tpl: Desired throughput level for the channel
|
||||
* @pdma_acc32: ACC32 must be enabled on the PDMA side
|
||||
* @pdma_burst: BURST must be enabled on the PDMA side
|
||||
* @mapped_channel_id: PKTDMA thread to channel mapping for mapped channels.
|
||||
* The thread must be serviced by the specified channel if
|
||||
* mapped_channel_id is >= 0 in case of PKTDMA
|
||||
* @flow_start: PKDMA flow range start of mapped channel. Unmapped
|
||||
* channels use flow_id == chan_id
|
||||
* @flow_num: PKDMA flow count of mapped channel. Unmapped channels
|
||||
* use flow_id == chan_id
|
||||
* @default_flow_id: PKDMA default (r)flow index of mapped channel.
|
||||
* Must be within the flow range of the mapped channel.
|
||||
*/
|
||||
struct psil_endpoint_config {
|
||||
enum psil_endpoint_type ep_type;
|
||||
|
@ -63,6 +72,13 @@ struct psil_endpoint_config {
|
|||
/* PDMA properties, valid for PSIL_EP_PDMA_* */
|
||||
unsigned pdma_acc32:1;
|
||||
unsigned pdma_burst:1;
|
||||
|
||||
/* PKDMA mapped channel */
|
||||
int mapped_channel_id;
|
||||
/* PKTDMA tflow and rflow ranges for mapped channel */
|
||||
u16 flow_start;
|
||||
u16 flow_num;
|
||||
u16 default_flow_id;
|
||||
};
|
||||
|
||||
int psil_set_new_ep_config(struct device *dev, const char *name,
|
||||
|
|
|
@ -41,6 +41,12 @@ void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
|
|||
u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn);
|
||||
u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn);
|
||||
int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn);
|
||||
struct device *
|
||||
k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn);
|
||||
void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
|
||||
dma_addr_t *addr);
|
||||
void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
|
||||
dma_addr_t *addr);
|
||||
|
||||
enum {
|
||||
K3_UDMA_GLUE_SRC_TAG_LO_KEEP = 0,
|
||||
|
@ -130,5 +136,11 @@ int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
|
|||
u32 flow_idx);
|
||||
int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
|
||||
u32 flow_idx);
|
||||
struct device *
|
||||
k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn);
|
||||
void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
|
||||
dma_addr_t *addr);
|
||||
void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
|
||||
dma_addr_t *addr);
|
||||
|
||||
#endif /* K3_UDMA_GLUE_H_ */
|
||||
|
|
83
include/linux/dma/qcom-gpi-dma.h
Normal file
83
include/linux/dma/qcom-gpi-dma.h
Normal file
|
@ -0,0 +1,83 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Copyright (c) 2020, Linaro Limited
|
||||
*/
|
||||
|
||||
#ifndef QCOM_GPI_DMA_H
|
||||
#define QCOM_GPI_DMA_H
|
||||
|
||||
/**
|
||||
* enum spi_transfer_cmd - spi transfer commands
|
||||
*/
|
||||
enum spi_transfer_cmd {
|
||||
SPI_TX = 1,
|
||||
SPI_RX,
|
||||
SPI_DUPLEX,
|
||||
};
|
||||
|
||||
/**
|
||||
* struct gpi_spi_config - spi config for peripheral
|
||||
*
|
||||
* @loopback_en: spi loopback enable when set
|
||||
* @clock_pol_high: clock polarity
|
||||
* @data_pol_high: data polarity
|
||||
* @pack_en: process tx/rx buffers as packed
|
||||
* @word_len: spi word length
|
||||
* @clk_div: source clock divider
|
||||
* @clk_src: serial clock
|
||||
* @cmd: spi cmd
|
||||
* @fragmentation: keep CS assserted at end of sequence
|
||||
* @cs: chip select toggle
|
||||
* @set_config: set peripheral config
|
||||
* @rx_len: receive length for buffer
|
||||
*/
|
||||
struct gpi_spi_config {
|
||||
u8 set_config;
|
||||
u8 loopback_en;
|
||||
u8 clock_pol_high;
|
||||
u8 data_pol_high;
|
||||
u8 pack_en;
|
||||
u8 word_len;
|
||||
u8 fragmentation;
|
||||
u8 cs;
|
||||
u32 clk_div;
|
||||
u32 clk_src;
|
||||
enum spi_transfer_cmd cmd;
|
||||
u32 rx_len;
|
||||
};
|
||||
|
||||
enum i2c_op {
|
||||
I2C_WRITE = 1,
|
||||
I2C_READ,
|
||||
};
|
||||
|
||||
/**
|
||||
* struct gpi_i2c_config - i2c config for peripheral
|
||||
*
|
||||
* @pack_enable: process tx/rx buffers as packed
|
||||
* @cycle_count: clock cycles to be sent
|
||||
* @high_count: high period of clock
|
||||
* @low_count: low period of clock
|
||||
* @clk_div: source clock divider
|
||||
* @addr: i2c bus address
|
||||
* @stretch: stretch the clock at eot
|
||||
* @set_config: set peripheral config
|
||||
* @rx_len: receive length for buffer
|
||||
* @op: i2c cmd
|
||||
* @muli-msg: is part of multi i2c r-w msgs
|
||||
*/
|
||||
struct gpi_i2c_config {
|
||||
u8 set_config;
|
||||
u8 pack_enable;
|
||||
u8 cycle_count;
|
||||
u8 high_count;
|
||||
u8 low_count;
|
||||
u8 addr;
|
||||
u8 stretch;
|
||||
u16 clk_div;
|
||||
u32 rx_len;
|
||||
enum i2c_op op;
|
||||
bool multi_msg;
|
||||
};
|
||||
|
||||
#endif /* QCOM_GPI_DMA_H */
|
|
@ -357,11 +357,14 @@ struct dma_chan {
|
|||
* @chan: driver channel device
|
||||
* @device: sysfs device
|
||||
* @dev_id: parent dma_device dev_id
|
||||
* @chan_dma_dev: The channel is using custom/different dma-mapping
|
||||
* compared to the parent dma_device
|
||||
*/
|
||||
struct dma_chan_dev {
|
||||
struct dma_chan *chan;
|
||||
struct device device;
|
||||
int dev_id;
|
||||
bool chan_dma_dev;
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -418,6 +421,9 @@ enum dma_slave_buswidth {
|
|||
* @slave_id: Slave requester id. Only valid for slave channels. The dma
|
||||
* slave peripheral will have unique id as dma requester which need to be
|
||||
* pass as slave config.
|
||||
* @peripheral_config: peripheral configuration for programming peripheral
|
||||
* for dmaengine transfer
|
||||
* @peripheral_size: peripheral configuration buffer size
|
||||
*
|
||||
* This struct is passed in as configuration data to a DMA engine
|
||||
* in order to set up a certain channel for DMA transport at runtime.
|
||||
|
@ -443,6 +449,8 @@ struct dma_slave_config {
|
|||
u32 dst_port_window_size;
|
||||
bool device_fc;
|
||||
unsigned int slave_id;
|
||||
void *peripheral_config;
|
||||
size_t peripheral_size;
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -800,6 +808,7 @@ struct dma_filter {
|
|||
* by tx_status
|
||||
* @device_alloc_chan_resources: allocate resources and return the
|
||||
* number of allocated descriptors
|
||||
* @device_router_config: optional callback for DMA router configuration
|
||||
* @device_free_chan_resources: release DMA channel's resources
|
||||
* @device_prep_dma_memcpy: prepares a memcpy operation
|
||||
* @device_prep_dma_xor: prepares a xor operation
|
||||
|
@ -874,6 +883,7 @@ struct dma_device {
|
|||
enum dma_residue_granularity residue_granularity;
|
||||
|
||||
int (*device_alloc_chan_resources)(struct dma_chan *chan);
|
||||
int (*device_router_config)(struct dma_chan *chan);
|
||||
void (*device_free_chan_resources)(struct dma_chan *chan);
|
||||
|
||||
struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
|
||||
|
@ -1611,4 +1621,13 @@ dmaengine_get_direction_text(enum dma_transfer_direction dir)
|
|||
return "invalid";
|
||||
}
|
||||
}
|
||||
|
||||
static inline struct device *dmaengine_get_dma_device(struct dma_chan *chan)
|
||||
{
|
||||
if (chan->dev->chan_dma_dev)
|
||||
return &chan->dev->device;
|
||||
|
||||
return chan->device->dev;
|
||||
}
|
||||
|
||||
#endif /* DMAENGINE_H */
|
||||
|
|
|
@ -70,6 +70,7 @@ struct k3_ring;
|
|||
* @dma_dev: Master device which is using and accessing to the ring
|
||||
* memory when the mode is K3_RINGACC_RING_MODE_RING. Memory allocations
|
||||
* should be done using this device.
|
||||
* @asel: Address Space Select value for physical addresses
|
||||
*/
|
||||
struct k3_ring_cfg {
|
||||
u32 size;
|
||||
|
@ -79,6 +80,7 @@ struct k3_ring_cfg {
|
|||
u32 flags;
|
||||
|
||||
struct device *dma_dev;
|
||||
u32 asel;
|
||||
};
|
||||
|
||||
#define K3_RINGACC_RING_ID_ANY (-1)
|
||||
|
@ -250,4 +252,19 @@ int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem);
|
|||
|
||||
u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring);
|
||||
|
||||
/* DMA ring support */
|
||||
struct ti_sci_handle;
|
||||
|
||||
/**
|
||||
* struct struct k3_ringacc_init_data - Initialization data for DMA rings
|
||||
*/
|
||||
struct k3_ringacc_init_data {
|
||||
const struct ti_sci_handle *tisci;
|
||||
u32 tisci_dev_id;
|
||||
u32 num_rings;
|
||||
};
|
||||
|
||||
struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
|
||||
struct k3_ringacc_init_data *data);
|
||||
|
||||
#endif /* __SOC_TI_K3_RINGACC_API_H_ */
|
||||
|
|
|
@ -26,6 +26,9 @@
|
|||
#define IDXD_OP_FLAG_DRDBK 0x4000
|
||||
#define IDXD_OP_FLAG_DSTS 0x8000
|
||||
|
||||
/* IAX */
|
||||
#define IDXD_OP_FLAG_RD_SRC2_AECS 0x010000
|
||||
|
||||
/* Opcode */
|
||||
enum dsa_opcode {
|
||||
DSA_OPCODE_NOOP = 0,
|
||||
|
@ -47,6 +50,14 @@ enum dsa_opcode {
|
|||
DSA_OPCODE_CFLUSH = 0x20,
|
||||
};
|
||||
|
||||
enum iax_opcode {
|
||||
IAX_OPCODE_NOOP = 0,
|
||||
IAX_OPCODE_DRAIN = 2,
|
||||
IAX_OPCODE_MEMMOVE,
|
||||
IAX_OPCODE_DECOMPRESS = 0x42,
|
||||
IAX_OPCODE_COMPRESS,
|
||||
};
|
||||
|
||||
/* Completion record status */
|
||||
enum dsa_completion_status {
|
||||
DSA_COMP_NONE = 0,
|
||||
|
@ -80,6 +91,33 @@ enum dsa_completion_status {
|
|||
DSA_COMP_TRANSLATION_FAIL,
|
||||
};
|
||||
|
||||
enum iax_completion_status {
|
||||
IAX_COMP_NONE = 0,
|
||||
IAX_COMP_SUCCESS,
|
||||
IAX_COMP_PAGE_FAULT_IR = 0x04,
|
||||
IAX_COMP_OUTBUF_OVERFLOW,
|
||||
IAX_COMP_BAD_OPCODE = 0x10,
|
||||
IAX_COMP_INVALID_FLAGS,
|
||||
IAX_COMP_NOZERO_RESERVE,
|
||||
IAX_COMP_INVALID_SIZE,
|
||||
IAX_COMP_OVERLAP_BUFFERS = 0x16,
|
||||
IAX_COMP_INT_HANDLE_INVAL = 0x19,
|
||||
IAX_COMP_CRA_XLAT,
|
||||
IAX_COMP_CRA_ALIGN,
|
||||
IAX_COMP_ADDR_ALIGN,
|
||||
IAX_COMP_PRIV_BAD,
|
||||
IAX_COMP_TRAFFIC_CLASS_CONF,
|
||||
IAX_COMP_PFAULT_RDBA,
|
||||
IAX_COMP_HW_ERR1,
|
||||
IAX_COMP_HW_ERR_DRB,
|
||||
IAX_COMP_TRANSLATION_FAIL,
|
||||
IAX_COMP_PRS_TIMEOUT,
|
||||
IAX_COMP_WATCHDOG,
|
||||
IAX_COMP_INVALID_COMP_FLAG = 0x30,
|
||||
IAX_COMP_INVALID_FILTER_FLAG,
|
||||
IAX_COMP_INVALID_NUM_ELEMS = 0x33,
|
||||
};
|
||||
|
||||
#define DSA_COMP_STATUS_MASK 0x7f
|
||||
#define DSA_COMP_STATUS_WRITE 0x80
|
||||
|
||||
|
@ -163,6 +201,28 @@ struct dsa_hw_desc {
|
|||
};
|
||||
} __attribute__((packed));
|
||||
|
||||
struct iax_hw_desc {
|
||||
uint32_t pasid:20;
|
||||
uint32_t rsvd:11;
|
||||
uint32_t priv:1;
|
||||
uint32_t flags:24;
|
||||
uint32_t opcode:8;
|
||||
uint64_t completion_addr;
|
||||
uint64_t src1_addr;
|
||||
uint64_t dst_addr;
|
||||
uint32_t src1_size;
|
||||
uint16_t int_handle;
|
||||
union {
|
||||
uint16_t compr_flags;
|
||||
uint16_t decompr_flags;
|
||||
};
|
||||
uint64_t src2_addr;
|
||||
uint32_t max_dst_size;
|
||||
uint32_t src2_size;
|
||||
uint32_t filter_flags;
|
||||
uint32_t num_inputs;
|
||||
} __attribute__((packed));
|
||||
|
||||
struct dsa_raw_desc {
|
||||
uint64_t field[8];
|
||||
} __attribute__((packed));
|
||||
|
@ -223,4 +283,23 @@ struct dsa_raw_completion_record {
|
|||
uint64_t field[4];
|
||||
} __attribute__((packed));
|
||||
|
||||
struct iax_completion_record {
|
||||
volatile uint8_t status;
|
||||
uint8_t error_code;
|
||||
uint16_t rsvd;
|
||||
uint32_t bytes_completed;
|
||||
uint64_t fault_addr;
|
||||
uint32_t invalid_flags;
|
||||
uint32_t rsvd2;
|
||||
uint32_t output_size;
|
||||
uint8_t output_bits;
|
||||
uint8_t rsvd3;
|
||||
uint16_t rsvd4;
|
||||
uint64_t rsvd5[4];
|
||||
} __attribute__((packed));
|
||||
|
||||
struct iax_raw_completion_record {
|
||||
uint64_t field[8];
|
||||
} __attribute__((packed));
|
||||
|
||||
#endif
|
||||
|
|
Loading…
Reference in a new issue