Kernel/USB: Add support for async & interrupt transfers

Add support for async transfers by using a separate kernel task to poll
a list of active async transfers on a set time interval, and invoke
their user-provided callback function when they are complete. Also add
support for the interrupt class of transfers, building off of this async
functionality.
This commit is contained in:
b14ckcat 2022-10-17 01:22:54 -04:00 committed by Andrew Kaster
parent 1aa16b4dd4
commit 7400eb3640
8 changed files with 181 additions and 62 deletions

View file

@ -5,6 +5,7 @@
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Find.h>
#include <AK/Platform.h>
#include <Kernel/Arch/Delay.h>
#include <Kernel/Bus/PCI/API.h>
@ -78,6 +79,7 @@ ErrorOr<void> UHCIController::initialize()
dmesgln("UHCI: I/O base {}", m_registers_io_window);
dmesgln("UHCI: Interrupt line: {}", interrupt_number());
TRY(spawn_async_poll_process());
TRY(spawn_port_process());
TRY(reset());
@ -88,6 +90,7 @@ UNMAP_AFTER_INIT UHCIController::UHCIController(PCI::DeviceIdentifier const& pci
: PCI::Device(pci_device_identifier.address())
, IRQHandler(pci_device_identifier.interrupt_line().value())
, m_registers_io_window(move(registers_io_window))
, m_async_lock(LockRank::None)
, m_schedule_lock(LockRank::None)
{
}
@ -132,14 +135,11 @@ UNMAP_AFTER_INIT ErrorOr<void> UHCIController::create_structures()
{
m_queue_head_pool = TRY(UHCIDescriptorPool<QueueHead>::try_create("Queue Head Pool"sv));
// Used as a sentinel value to loop back to the beginning of the list
// Doesn't do anything other than give interrupt transfer queues something to set as prev QH so that we don't have to handle that as an extra edge case
m_schedule_begin_anchor = allocate_queue_head();
// Each interrupt QH anchor in the array is linked into the schedule so that
// it is executed once every (2^i) milliseconds, where i is it's index
for (int i = 0; i < NUMBER_OF_INTERRUPT_QHS; i++) {
m_interrupt_qh_anchor_arr[i] = allocate_queue_head();
}
// Create the Full Speed, Low Speed Control and Bulk Queue Heads
// Create the Interrupt, Full Speed, Low Speed Control and Bulk Queue Heads
m_interrupt_qh_anchor = allocate_queue_head();
m_ls_control_qh_anchor = allocate_queue_head();
m_fs_control_qh_anchor = allocate_queue_head();
m_bulk_qh_anchor = allocate_queue_head();
@ -201,15 +201,11 @@ UNMAP_AFTER_INIT void UHCIController::setup_schedule()
// Not specified in the datasheet, however, is another Queue Head with an "inactive" Transfer Descriptor. This
// is to circumvent a bug in the silicon of the PIIX4's UHCI controller.
// https://github.com/openbsd/src/blob/master/sys/dev/usb/uhci.c#L390
m_schedule_begin_anchor->link_next_queue_head(m_interrupt_qh_anchor_arr[0]);
m_schedule_begin_anchor->link_next_queue_head(m_interrupt_qh_anchor);
m_schedule_begin_anchor->terminate_element_link_ptr();
for (int i = 0; i < NUMBER_OF_INTERRUPT_QHS - 1; i++) {
m_interrupt_qh_anchor_arr[i]->link_next_queue_head(m_interrupt_qh_anchor_arr[i + 1]);
m_interrupt_qh_anchor_arr[i]->terminate_element_link_ptr();
}
m_interrupt_qh_anchor_arr[NUMBER_OF_INTERRUPT_QHS - 1]->link_next_queue_head(m_ls_control_qh_anchor);
m_interrupt_qh_anchor_arr[NUMBER_OF_INTERRUPT_QHS - 1]->terminate_element_link_ptr();
m_interrupt_qh_anchor->link_next_queue_head(m_ls_control_qh_anchor);
m_interrupt_qh_anchor->terminate_element_link_ptr();
m_ls_control_qh_anchor->link_next_queue_head(m_fs_control_qh_anchor);
m_ls_control_qh_anchor->terminate_element_link_ptr();
@ -227,20 +223,12 @@ UNMAP_AFTER_INIT void UHCIController::setup_schedule()
u32* framelist = reinterpret_cast<u32*>(m_framelist->vaddr().as_ptr());
for (int frame_num = 0; frame_num < UHCI_NUMBER_OF_FRAMES; frame_num++) {
auto frame_iso_td = m_iso_td_list.at(frame_num % UHCI_NUMBER_OF_ISOCHRONOUS_TDS);
// Each frame pointer points to iso_td % NUM_ISO_TDS
for (int i = NUMBER_OF_INTERRUPT_QHS - 1; i >= 0; i--) {
if (frame_num % (1 << i) == 0) {
frame_iso_td->link_queue_head(m_interrupt_qh_anchor_arr[i]->paddr());
break;
}
}
auto& frame_iso_td = m_iso_td_list.at(frame_num % UHCI_NUMBER_OF_ISOCHRONOUS_TDS);
frame_iso_td->link_queue_head(m_schedule_begin_anchor->paddr());
framelist[frame_num] = frame_iso_td->paddr();
}
for (int i = 0; i < NUMBER_OF_INTERRUPT_QHS; i++) {
m_interrupt_qh_anchor_arr[i]->print();
}
m_interrupt_qh_anchor->print();
m_ls_control_qh_anchor->print();
m_fs_control_qh_anchor->print();
m_bulk_qh_anchor->print();
@ -387,6 +375,70 @@ void UHCIController::dequeue_qh(QueueHead* transfer_queue)
transfer_queue->prev_qh()->link_next_queue_head(transfer_queue->next_qh());
}
ErrorOr<QueueHead*> UHCIController::create_transfer_queue(Transfer& transfer)
{
Pipe& pipe = transfer.pipe();
// Create a new descriptor chain
TransferDescriptor* last_data_descriptor;
TransferDescriptor* data_descriptor_chain;
auto buffer_address = Ptr32<u8>(transfer.buffer_physical().as_ptr());
TRY(create_chain(pipe, transfer.pipe().direction() == Pipe::Direction::In ? PacketID::IN : PacketID::OUT, buffer_address, pipe.max_packet_size(), transfer.transfer_data_size(), &data_descriptor_chain, &last_data_descriptor));
last_data_descriptor->terminate();
if constexpr (UHCI_VERBOSE_DEBUG) {
if (data_descriptor_chain) {
dbgln("Data TD");
data_descriptor_chain->print();
}
}
QueueHead* transfer_queue = allocate_queue_head();
if (!transfer_queue) {
free_descriptor_chain(data_descriptor_chain);
return ENOMEM;
}
transfer_queue->attach_transfer_descriptor_chain(data_descriptor_chain);
transfer_queue->set_transfer(&transfer);
return transfer_queue;
}
ErrorOr<void> UHCIController::submit_async_transfer(NonnullOwnPtr<AsyncTransferHandle> async_handle, QueueHead* anchor, QueueHead* transfer_queue)
{
{
SpinlockLocker locker { m_async_lock };
auto iter = find_if(m_active_async_transfers.begin(), m_active_async_transfers.end(), [](auto& handle) { return handle == nullptr; });
if (iter == m_active_async_transfers.end())
return ENOMEM;
*iter = move(async_handle);
}
enqueue_qh(transfer_queue, anchor);
return {};
}
void UHCIController::cancel_async_transfer(NonnullLockRefPtr<Transfer> transfer)
{
SpinlockLocker locker { m_async_lock };
auto iter = find_if(m_active_async_transfers.begin(), m_active_async_transfers.end(), [transfer](auto& handle) { return handle != nullptr && handle->transfer.ptr() == transfer.ptr(); });
if (iter == m_active_async_transfers.end()) {
dbgln("Error: couldn't cancel supplied async transfer");
return; // We can't really do anything here, so just give up
}
auto& transfer_queue = (*iter)->qh;
dequeue_qh(transfer_queue);
free_descriptor_chain(transfer_queue->get_first_td());
transfer_queue->free();
m_queue_head_pool->release_to_pool(transfer_queue);
*iter = nullptr;
}
ErrorOr<size_t> UHCIController::submit_control_transfer(Transfer& transfer)
{
Pipe& pipe = transfer.pipe(); // Short circuit the pipe related to this transfer
@ -467,35 +519,11 @@ ErrorOr<size_t> UHCIController::submit_control_transfer(Transfer& transfer)
ErrorOr<size_t> UHCIController::submit_bulk_transfer(Transfer& transfer)
{
Pipe& pipe = transfer.pipe();
dbgln_if(UHCI_DEBUG, "UHCI: Received bulk transfer for address {}. Root Hub is at address {}.", pipe.device_address(), m_root_hub->device_address());
// Create a new descriptor chain
TransferDescriptor* last_data_descriptor;
TransferDescriptor* data_descriptor_chain;
auto buffer_address = Ptr32<u8>(transfer.buffer_physical().as_ptr());
TRY(create_chain(pipe, transfer.pipe().direction() == Pipe::Direction::In ? PacketID::IN : PacketID::OUT, buffer_address, pipe.max_packet_size(), transfer.transfer_data_size(), &data_descriptor_chain, &last_data_descriptor));
last_data_descriptor->terminate();
if constexpr (UHCI_VERBOSE_DEBUG) {
if (data_descriptor_chain) {
dbgln("Data TD");
data_descriptor_chain->print();
}
}
QueueHead* transfer_queue = allocate_queue_head();
if (!transfer_queue) {
free_descriptor_chain(data_descriptor_chain);
return ENOMEM;
}
transfer_queue->attach_transfer_descriptor_chain(data_descriptor_chain);
transfer_queue->set_transfer(&transfer);
auto transfer_queue = TRY(create_transfer_queue(transfer));
enqueue_qh(transfer_queue, m_bulk_qh_anchor);
dbgln_if(UHCI_DEBUG, "UHCI: Received bulk transfer for address {}. Root Hub is at address {}.", transfer.pipe().device_address(), m_root_hub->device_address());
size_t transfer_size = 0;
while (!transfer.complete()) {
transfer_size = poll_transfer_queue(*transfer_queue);
@ -510,6 +538,21 @@ ErrorOr<size_t> UHCIController::submit_bulk_transfer(Transfer& transfer)
return transfer_size;
}
ErrorOr<void> UHCIController::submit_async_interrupt_transfer(NonnullLockRefPtr<Transfer> transfer, u16 ms_interval)
{
dbgln_if(UHCI_DEBUG, "UHCI: Received interrupt transfer for address {}. Root Hub is at address {}.", transfer->pipe().device_address(), m_root_hub->device_address());
if (ms_interval == 0) {
return EINVAL;
}
auto transfer_queue = TRY(create_transfer_queue(*transfer));
auto async_transfer_handle = TRY(adopt_nonnull_own_or_enomem(new (nothrow) AsyncTransferHandle { transfer, transfer_queue, ms_interval }));
TRY(submit_async_transfer(move(async_transfer_handle), m_interrupt_qh_anchor, transfer_queue));
return {};
}
size_t UHCIController::poll_transfer_queue(QueueHead& transfer_queue)
{
Transfer* transfer = transfer_queue.transfer();
@ -556,6 +599,33 @@ ErrorOr<void> UHCIController::spawn_port_process()
return {};
}
ErrorOr<void> UHCIController::spawn_async_poll_process()
{
LockRefPtr<Thread> async_poll_thread;
(void)Process::create_kernel_process(async_poll_thread, TRY(KString::try_create("UHCI Async Poll Task"sv)), [&] {
u16 poll_interval_ms = 1024;
for (;;) {
{
SpinlockLocker locker { m_async_lock };
for (OwnPtr<AsyncTransferHandle>& handle : m_active_async_transfers) {
if (handle != nullptr) {
poll_interval_ms = min(poll_interval_ms, handle->ms_poll_interval);
QueueHead* qh = handle->qh;
for (auto td = qh->get_first_td(); td != nullptr && !td->active(); td = td->next_td()) {
if (td->next_td() == nullptr) { // Finished QH
handle->transfer->invoke_async_callback();
qh->reinitialize(); // Set the QH to be active again
}
}
}
}
}
(void)Thread::current()->sleep(Time::from_milliseconds(poll_interval_ms));
}
});
return {};
}
bool UHCIController::handle_irq(RegisterState const&)
{
u32 status = read_usbsts();

View file

@ -31,7 +31,6 @@ class UHCIController final
static constexpr u8 MAXIMUM_NUMBER_OF_TDS = 128; // Upper pool limit. This consumes the second page we have allocated
static constexpr u8 MAXIMUM_NUMBER_OF_QHS = 64;
static constexpr u8 NUMBER_OF_INTERRUPT_QHS = 11;
public:
static constexpr u8 NUMBER_OF_ROOT_PORTS = 2;
@ -44,10 +43,16 @@ public:
virtual ErrorOr<void> reset() override;
virtual ErrorOr<void> stop() override;
virtual ErrorOr<void> start() override;
ErrorOr<void> spawn_async_poll_process();
ErrorOr<void> spawn_port_process();
ErrorOr<QueueHead*> create_transfer_queue(Transfer& transfer);
ErrorOr<void> submit_async_transfer(NonnullOwnPtr<AsyncTransferHandle> async_handle, QueueHead* anchor, QueueHead* transfer_queue);
virtual void cancel_async_transfer(NonnullLockRefPtr<Transfer> transfer) override;
virtual ErrorOr<size_t> submit_control_transfer(Transfer& transfer) override;
virtual ErrorOr<size_t> submit_bulk_transfer(Transfer& transfer) override;
virtual ErrorOr<void> submit_async_interrupt_transfer(NonnullLockRefPtr<Transfer> transfer, u16 ms_interval) override;
void get_port_status(Badge<UHCIRootHub>, u8, HubStatus&);
ErrorOr<void> set_port_feature(Badge<UHCIRootHub>, u8, HubFeatureSelector);
@ -95,15 +100,17 @@ private:
NonnullOwnPtr<IOWindow> m_registers_io_window;
Spinlock m_async_lock;
Spinlock m_schedule_lock;
OwnPtr<UHCIRootHub> m_root_hub;
OwnPtr<UHCIDescriptorPool<QueueHead>> m_queue_head_pool;
OwnPtr<UHCIDescriptorPool<TransferDescriptor>> m_transfer_descriptor_pool;
Vector<TransferDescriptor*> m_iso_td_list;
Array<OwnPtr<AsyncTransferHandle>, MAXIMUM_NUMBER_OF_QHS> m_active_async_transfers;
QueueHead* m_schedule_begin_anchor;
Array<QueueHead*, NUMBER_OF_INTERRUPT_QHS> m_interrupt_qh_anchor_arr;
QueueHead* m_interrupt_qh_anchor;
QueueHead* m_ls_control_qh_anchor;
QueueHead* m_fs_control_qh_anchor;
// Always final queue in the schedule, may loop back to previous QH for bandwidth

View file

@ -356,6 +356,15 @@ struct alignas(16) QueueHead {
m_bookkeeping->in_use = false;
}
void reinitialize()
{
for (TransferDescriptor* iter = get_first_td(); iter != nullptr; iter = iter->next_td()) {
iter->set_active();
}
attach_transfer_descriptor_chain(get_first_td());
}
private:
u32 m_link_ptr { 0 }; // Pointer to the next horizontal object that the controller will execute after this one
volatile u32 m_element_link_ptr { 0 }; // Pointer to the first data object in the queue (can be modified by hw)
@ -366,4 +375,11 @@ private:
};
static_assert(AssertSize<QueueHead, 32>()); // Queue Head is always 8 Dwords
struct AsyncTransferHandle {
NonnullLockRefPtr<Transfer> transfer;
QueueHead* qh;
u16 ms_poll_interval;
};
}

View file

@ -23,8 +23,10 @@ public:
virtual ErrorOr<void> stop() = 0;
virtual ErrorOr<void> start() = 0;
virtual void cancel_async_transfer(NonnullLockRefPtr<Transfer> transfer) = 0;
virtual ErrorOr<size_t> submit_control_transfer(Transfer&) = 0;
virtual ErrorOr<size_t> submit_bulk_transfer(Transfer& transfer) = 0;
virtual ErrorOr<void> submit_async_interrupt_transfer(NonnullLockRefPtr<Transfer> transfer, u16 ms_interval) = 0;
u8 allocate_address();

View file

@ -50,7 +50,7 @@ ErrorOr<size_t> ControlPipe::control_transfer(u8 request_type, u8 request, u16 v
usb_request.index = index;
usb_request.length = length;
auto transfer = TRY(Transfer::try_create(*this, length, *m_dma_buffer));
auto transfer = TRY(Transfer::create(*this, length, *m_dma_buffer));
transfer->set_setup_packet(usb_request);
dbgln_if(USB_DEBUG, "ControlPipe: Transfer allocated @ {}", transfer->buffer_physical());
@ -84,7 +84,7 @@ ErrorOr<size_t> BulkInPipe::bulk_in_transfer(size_t length, void* data)
size_t transfer_length = 0;
auto transfer = TRY(Transfer::try_create(*this, length, *m_dma_buffer));
auto transfer = TRY(Transfer::create(*this, length, *m_dma_buffer));
dbgln_if(USB_DEBUG, "Pipe: Bulk in transfer allocated @ {}", transfer->buffer_physical());
transfer_length = TRY(m_controller->submit_bulk_transfer(*transfer));
@ -114,7 +114,7 @@ ErrorOr<size_t> BulkOutPipe::bulk_out_transfer(size_t length, void* data)
MutexLocker lock(m_dma_buffer_lock);
size_t transfer_length = 0;
auto transfer = TRY(Transfer::try_create(*this, length, *m_dma_buffer));
auto transfer = TRY(Transfer::create(*this, length, *m_dma_buffer));
TRY(transfer->write_buffer(length, data));
dbgln_if(USB_DEBUG, "Pipe: Bulk out transfer allocated @ {}", transfer->buffer_physical());
@ -137,6 +137,16 @@ InterruptInPipe::InterruptInPipe(USBController const& controller, u8 endpoint_ad
{
}
ErrorOr<NonnullLockRefPtr<Transfer>> InterruptInPipe::interrupt_in_transfer(size_t length, u16 ms_interval, USBAsyncCallback callback)
{
VERIFY(length <= m_dma_buffer->size());
auto transfer = TRY(Transfer::create(*this, length, *m_dma_buffer, move(callback)));
dbgln_if(USB_DEBUG, "Pipe: Interrupt in transfer allocated @ {}", transfer->buffer_physical());
TRY(m_controller->submit_async_interrupt_transfer(transfer, ms_interval));
return transfer;
}
ErrorOr<NonnullOwnPtr<InterruptOutPipe>> InterruptOutPipe::create(USBController const& controller, u8 endpoint_address, u16 max_packet_size, i8 device_address, u16 poll_interval, size_t buffer_size)
{
VERIFY(buffer_size >= max_packet_size);

View file

@ -18,6 +18,8 @@ namespace Kernel::USB {
class USBController;
class Transfer;
using USBAsyncCallback = Function<void(Transfer* transfer)>;
//
// A pipe is the logical connection between a memory buffer on the PC (host) and
// an endpoint on the device. In this implementation, the data buffer the pipe connects
@ -111,6 +113,8 @@ class InterruptInPipe : public Pipe {
public:
static ErrorOr<NonnullOwnPtr<InterruptInPipe>> create(USBController const& controller, u8 endpoint_address, u16 max_packet_size, i8 device_address, u16 poll_interval, size_t buffer_size = PAGE_SIZE);
ErrorOr<NonnullLockRefPtr<Transfer>> interrupt_in_transfer(size_t length, u16 ms_interval, USBAsyncCallback callback);
u16 poll_interval() const { return m_poll_interval; }
private:

View file

@ -9,15 +9,16 @@
namespace Kernel::USB {
ErrorOr<NonnullLockRefPtr<Transfer>> Transfer::try_create(Pipe& pipe, u16 length, Memory::Region& dma_buffer)
ErrorOr<NonnullLockRefPtr<Transfer>> Transfer::create(Pipe& pipe, u16 length, Memory::Region& dma_buffer, USBAsyncCallback callback)
{
return adopt_nonnull_lock_ref_or_enomem(new (nothrow) Transfer(pipe, length, dma_buffer));
return adopt_nonnull_lock_ref_or_enomem(new (nothrow) Transfer(pipe, length, dma_buffer, move(callback)));
}
Transfer::Transfer(Pipe& pipe, u16 len, Memory::Region& dma_buffer)
Transfer::Transfer(Pipe& pipe, u16 len, Memory::Region& dma_buffer, USBAsyncCallback callback)
: m_pipe(pipe)
, m_dma_buffer(dma_buffer)
, m_transfer_data_size(len)
, m_callback(move(callback))
{
}
@ -49,4 +50,10 @@ ErrorOr<void> Transfer::write_buffer(u16 len, void* data)
return {};
}
void Transfer::invoke_async_callback()
{
if (m_callback)
m_callback(this);
}
}

View file

@ -20,7 +20,7 @@ namespace Kernel::USB {
class Transfer final : public AtomicRefCounted<Transfer> {
public:
static ErrorOr<NonnullLockRefPtr<Transfer>> try_create(Pipe&, u16 length, Memory::Region& dma_buffer);
static ErrorOr<NonnullLockRefPtr<Transfer>> create(Pipe&, u16 length, Memory::Region& dma_buffer, USBAsyncCallback callback = nullptr);
Transfer() = delete;
~Transfer();
@ -41,14 +41,17 @@ public:
bool complete() const { return m_complete; }
bool error_occurred() const { return m_error_occurred; }
void invoke_async_callback();
private:
Transfer(Pipe& pipe, u16 len, Memory::Region& dma_buffer);
Transfer(Pipe& pipe, u16 len, Memory::Region& dma_buffer, USBAsyncCallback callback);
Pipe& m_pipe; // Pipe that initiated this transfer
Memory::Region& m_dma_buffer; // DMA buffer
USBRequestData m_request; // USB request
u16 m_transfer_data_size { 0 }; // Size of the transfer's data stage
bool m_complete { false }; // Has this transfer been completed?
bool m_error_occurred { false }; // Did an error occur during this transfer?
USBAsyncCallback m_callback { nullptr };
};
}