HelenOS sources
This source file includes following definitions.
- isoch_init
- isoch_reset
- isoch_reset_no_timer
- isoch_reset_timer
- timer_schedule_reset
- isoch_fini
- isoch_alloc_transfers
- schedule_isochronous_trb
- get_system_time
- get_current_microframe
- calc_next_mfindex
- window_decide
- isoch_feed_out
- isoch_feed_out_timer
- isoch_feed_in
- isoch_feed_in_timer
- isoch_schedule_out
- isoch_schedule_in
- isoch_handle_transfer_event
#include <str_error.h>
#include <macros.h>
#include "endpoint.h"
#include "hw_struct/trb.h"
#include "hw_struct/regs.h"
#include "trb_ring.h"
#include "hc.h"
#include "bus.h"
#include "isoch.h"
void isoch_init(xhci_endpoint_t *ep, const usb_endpoint_descriptors_t *desc)
{
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
fibril_mutex_initialize(&isoch->guard);
fibril_condvar_initialize(&isoch->avail);
const xhci_hc_t *hc = bus_to_xhci_bus(ep->base.device->bus)->hc;
isoch->buffer_count = (2 * hc->ist) / ep->interval;
isoch->buffer_count = max(2, isoch->buffer_count);
usb_log_debug("[isoch] isoch setup with %zu buffers", isoch->buffer_count);
}
static void isoch_reset(xhci_endpoint_t *ep)
{
xhci_isoch_t *const isoch = ep->isoch;
assert(fibril_mutex_is_locked(&isoch->guard));
isoch->dequeue = isoch->enqueue = isoch->hw_enqueue = 0;
for (size_t i = 0; i < isoch->buffer_count; ++i) {
isoch->transfers[i].state = ISOCH_EMPTY;
}
fibril_timer_clear_locked(isoch->feeding_timer);
isoch->last_mf = -1U;
usb_log_info("[isoch] Endpoint" XHCI_EP_FMT ": Data flow reset.",
XHCI_EP_ARGS(*ep));
}
static void isoch_reset_no_timer(xhci_endpoint_t *ep)
{
xhci_isoch_t *const isoch = ep->isoch;
assert(fibril_mutex_is_locked(&isoch->guard));
fibril_timer_clear_locked(isoch->reset_timer);
isoch_reset(ep);
}
static void isoch_reset_timer(void *ep)
{
xhci_isoch_t *const isoch = xhci_endpoint_get(ep)->isoch;
fibril_mutex_lock(&isoch->guard);
isoch_reset(ep);
fibril_mutex_unlock(&isoch->guard);
}
#define RESET_TIMER_DELAY 100000
static void timer_schedule_reset(xhci_endpoint_t *ep)
{
xhci_isoch_t *const isoch = ep->isoch;
const usec_t delay = isoch->buffer_count * ep->interval * 125 +
RESET_TIMER_DELAY;
fibril_timer_clear_locked(isoch->reset_timer);
fibril_timer_set_locked(isoch->reset_timer, delay,
isoch_reset_timer, ep);
}
void isoch_fini(xhci_endpoint_t *ep)
{
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
if (isoch->feeding_timer) {
fibril_timer_clear(isoch->feeding_timer);
fibril_timer_destroy(isoch->feeding_timer);
fibril_timer_clear(isoch->reset_timer);
fibril_timer_destroy(isoch->reset_timer);
}
if (isoch->transfers) {
for (size_t i = 0; i < isoch->buffer_count; ++i)
dma_buffer_free(&isoch->transfers[i].data);
free(isoch->transfers);
}
}
errno_t isoch_alloc_transfers(xhci_endpoint_t *ep)
{
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
isoch->feeding_timer = fibril_timer_create(&isoch->guard);
isoch->reset_timer = fibril_timer_create(&isoch->guard);
if (!isoch->feeding_timer)
return ENOMEM;
isoch->transfers = calloc(isoch->buffer_count, sizeof(xhci_isoch_transfer_t));
if (!isoch->transfers)
goto err;
for (size_t i = 0; i < isoch->buffer_count; ++i) {
xhci_isoch_transfer_t *transfer = &isoch->transfers[i];
if (dma_buffer_alloc(&transfer->data, ep->base.max_transfer_size)) {
goto err;
}
}
fibril_mutex_lock(&isoch->guard);
isoch_reset_no_timer(ep);
fibril_mutex_unlock(&isoch->guard);
return EOK;
err:
isoch_fini(ep);
return ENOMEM;
}
static errno_t schedule_isochronous_trb(xhci_endpoint_t *ep, xhci_isoch_transfer_t *it)
{
xhci_trb_t trb;
xhci_trb_clean(&trb);
trb.parameter = host2xhci(64, dma_buffer_phys_base(&it->data));
TRB_CTRL_SET_XFER_LEN(trb, it->size);
TRB_CTRL_SET_TD_SIZE(trb, 0);
TRB_CTRL_SET_IOC(trb, 1);
TRB_CTRL_SET_TRB_TYPE(trb, XHCI_TRB_TYPE_ISOCH);
size_t tdpc = it->size / 1024 + ((it->size % 1024) ? 1 : 0);
size_t tbc = tdpc / ep->max_burst;
if (!tdpc % ep->max_burst)
--tbc;
size_t bsp = tdpc % ep->max_burst;
size_t tlbpc = (bsp ? bsp : ep->max_burst) - 1;
TRB_ISOCH_SET_TBC(trb, tbc);
TRB_ISOCH_SET_TLBPC(trb, tlbpc);
TRB_ISOCH_SET_FRAMEID(trb, (it->mfindex / 8) % 2048);
const errno_t err = xhci_trb_ring_enqueue(&ep->ring, &trb, &it->interrupt_trb_phys);
return err;
}
#define EPOCH_BITS 14
#define EPOCH_DELAY 500000
#define EPOCH_LOW_MFINDEX 8 * 100
static inline uint64_t get_system_time()
{
struct timespec ts;
getuptime(&ts);
return SEC2USEC(ts.tv_sec) + NSEC2USEC(ts.tv_nsec);
}
static inline uint64_t get_current_microframe(const xhci_hc_t *hc)
{
const uint32_t reg_mfindex = XHCI_REG_RD(hc->rt_regs, XHCI_RT_MFINDEX);
uint64_t epoch = hc->wrap_count;
if (reg_mfindex < EPOCH_LOW_MFINDEX &&
get_system_time() - hc->wrap_time > EPOCH_DELAY) {
++epoch;
}
return (epoch << EPOCH_BITS) + reg_mfindex;
}
static inline void calc_next_mfindex(xhci_endpoint_t *ep, xhci_isoch_transfer_t *it)
{
xhci_isoch_t *const isoch = ep->isoch;
if (isoch->last_mf == -1U) {
const xhci_bus_t *bus = bus_to_xhci_bus(ep->base.device->bus);
const xhci_hc_t *hc = bus->hc;
const uint64_t delay = min(isoch->buffer_count * ep->interval, 10 * 8);
it->mfindex = get_current_microframe(hc) + 1 + delay + hc->ist;
it->mfindex += ep->interval - 1;
it->mfindex &= ~(ep->interval - 1);
} else {
it->mfindex = isoch->last_mf + ep->interval;
}
}
#define END_FRAME_DELAY (895000 / 125)
typedef enum {
WINDOW_TOO_SOON,
WINDOW_INSIDE,
WINDOW_TOO_LATE,
} window_position_t;
typedef struct {
window_position_t position;
uint64_t offset;
} window_decision_t;
static inline void window_decide(window_decision_t *res, xhci_hc_t *hc,
uint64_t mfindex)
{
const uint64_t current_mf = get_current_microframe(hc);
const uint64_t start = current_mf + hc->ist + 1;
const uint64_t end = current_mf + END_FRAME_DELAY;
if (mfindex < start) {
res->position = WINDOW_TOO_LATE;
res->offset = start - mfindex;
} else if (mfindex <= end) {
res->position = WINDOW_INSIDE;
} else {
res->position = WINDOW_TOO_SOON;
res->offset = mfindex - end;
}
}
static void isoch_feed_out_timer(void *);
static void isoch_feed_in_timer(void *);
static void isoch_feed_out(xhci_endpoint_t *ep)
{
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
assert(fibril_mutex_is_locked(&isoch->guard));
xhci_bus_t *bus = bus_to_xhci_bus(ep->base.device->bus);
xhci_hc_t *hc = bus->hc;
bool fed = false;
while (isoch->transfers[isoch->hw_enqueue].state == ISOCH_FILLED) {
xhci_isoch_transfer_t *const it = &isoch->transfers[isoch->hw_enqueue];
usec_t delay;
assert(it->state == ISOCH_FILLED);
window_decision_t wd;
window_decide(&wd, hc, it->mfindex);
switch (wd.position) {
case WINDOW_TOO_SOON:
delay = wd.offset * 125;
usb_log_debug("[isoch] delaying feeding buffer %zu for %lldus",
it - isoch->transfers, delay);
fibril_timer_set_locked(isoch->feeding_timer, delay,
isoch_feed_out_timer, ep);
goto out;
case WINDOW_INSIDE:
usb_log_debug("[isoch] feeding buffer %zu at 0x%" PRIx64,
it - isoch->transfers, it->mfindex);
it->error = schedule_isochronous_trb(ep, it);
if (it->error) {
it->state = ISOCH_COMPLETE;
} else {
it->state = ISOCH_FED;
fed = true;
}
isoch->hw_enqueue = (isoch->hw_enqueue + 1) % isoch->buffer_count;
break;
case WINDOW_TOO_LATE:
usb_log_debug("[isoch] missed feeding buffer %zu at 0x%" PRIx64 " by "
"%" PRIu64 " uframes", it - isoch->transfers, it->mfindex, wd.offset);
it->state = ISOCH_COMPLETE;
it->error = EOK;
it->size = 0;
isoch->hw_enqueue = (isoch->hw_enqueue + 1) % isoch->buffer_count;
break;
}
}
out:
if (fed) {
hc_ring_ep_doorbell(ep, 0);
timer_schedule_reset(ep);
}
}
static void isoch_feed_out_timer(void *ep)
{
xhci_isoch_t *const isoch = xhci_endpoint_get(ep)->isoch;
fibril_mutex_lock(&isoch->guard);
isoch_feed_out(ep);
fibril_mutex_unlock(&isoch->guard);
}
static void isoch_feed_in(xhci_endpoint_t *ep)
{
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
assert(fibril_mutex_is_locked(&isoch->guard));
xhci_bus_t *bus = bus_to_xhci_bus(ep->base.device->bus);
xhci_hc_t *hc = bus->hc;
bool fed = false;
while (isoch->transfers[isoch->enqueue].state <= ISOCH_FILLED) {
xhci_isoch_transfer_t *const it = &isoch->transfers[isoch->enqueue];
usec_t delay;
if (it->state == ISOCH_EMPTY) {
it->size = ep->base.max_transfer_size;
it->state = ISOCH_FILLED;
calc_next_mfindex(ep, it);
}
window_decision_t wd;
window_decide(&wd, hc, it->mfindex);
switch (wd.position) {
case WINDOW_TOO_SOON:
delay = wd.offset * 125;
usb_log_debug("[isoch] delaying feeding buffer %zu for %lldus",
it - isoch->transfers, delay);
fibril_timer_set_locked(isoch->feeding_timer, delay,
isoch_feed_in_timer, ep);
goto out;
case WINDOW_TOO_LATE:
usb_log_debug("[isoch] missed feeding buffer %zu at 0x%" PRIx64 " by"
"%" PRIu64 " uframes", it - isoch->transfers, it->mfindex, wd.offset);
it->mfindex += wd.offset;
it->mfindex += ep->interval - 1;
it->mfindex &= ~(ep->interval - 1);
case WINDOW_INSIDE:
isoch->enqueue = (isoch->enqueue + 1) % isoch->buffer_count;
isoch->last_mf = it->mfindex;
usb_log_debug("[isoch] feeding buffer %zu at 0x%" PRIx64,
it - isoch->transfers, it->mfindex);
it->error = schedule_isochronous_trb(ep, it);
if (it->error) {
it->state = ISOCH_COMPLETE;
} else {
it->state = ISOCH_FED;
fed = true;
}
break;
}
}
out:
if (fed) {
hc_ring_ep_doorbell(ep, 0);
timer_schedule_reset(ep);
}
}
static void isoch_feed_in_timer(void *ep)
{
xhci_isoch_t *const isoch = xhci_endpoint_get(ep)->isoch;
fibril_mutex_lock(&isoch->guard);
isoch_feed_in(ep);
fibril_mutex_unlock(&isoch->guard);
}
errno_t isoch_schedule_out(xhci_transfer_t *transfer)
{
errno_t err = EOK;
xhci_endpoint_t *ep = xhci_endpoint_get(transfer->batch.ep);
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
assert(transfer->batch.size <= ep->base.max_transfer_size);
fibril_mutex_lock(&isoch->guard);
xhci_isoch_transfer_t *it = &isoch->transfers[isoch->enqueue];
while (it->state == ISOCH_FED || it->state == ISOCH_FILLED) {
fibril_condvar_wait(&isoch->avail, &isoch->guard);
it = &isoch->transfers[isoch->enqueue];
}
isoch->enqueue = (isoch->enqueue + 1) % isoch->buffer_count;
transfer->batch.transferred_size = 0;
xhci_isoch_transfer_t *res = &isoch->transfers[isoch->dequeue];
while (res->state == ISOCH_COMPLETE) {
isoch->dequeue = (isoch->dequeue + 1) % isoch->buffer_count;
res->state = ISOCH_EMPTY;
transfer->batch.transferred_size += res->size;
transfer->batch.error = res->error;
if (res->error)
break;
res = &isoch->transfers[isoch->dequeue];
}
assert(it->state == ISOCH_EMPTY);
calc_next_mfindex(ep, it);
isoch->last_mf = it->mfindex;
usb_log_debug("[isoch] buffer %zu will be on schedule at 0x%" PRIx64,
it - isoch->transfers, it->mfindex);
it->size = transfer->batch.size;
memcpy(it->data.virt, transfer->batch.dma_buffer.virt, it->size);
it->state = ISOCH_FILLED;
fibril_timer_clear_locked(isoch->feeding_timer);
isoch_feed_out(ep);
fibril_mutex_unlock(&isoch->guard);
usb_transfer_batch_finish(&transfer->batch);
return err;
}
errno_t isoch_schedule_in(xhci_transfer_t *transfer)
{
xhci_endpoint_t *ep = xhci_endpoint_get(transfer->batch.ep);
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
if (transfer->batch.size < ep->base.max_transfer_size) {
usb_log_error("Cannot schedule an undersized isochronous transfer.");
return ELIMIT;
}
fibril_mutex_lock(&isoch->guard);
xhci_isoch_transfer_t *it = &isoch->transfers[isoch->dequeue];
while (it->state != ISOCH_COMPLETE) {
fibril_timer_clear_locked(isoch->feeding_timer);
isoch_feed_in(ep);
usb_log_debug("[isoch] waiting for buffer %zu to be completed",
it - isoch->transfers);
fibril_condvar_wait(&isoch->avail, &isoch->guard);
it = &isoch->transfers[isoch->dequeue];
}
isoch->dequeue = (isoch->dequeue + 1) % isoch->buffer_count;
if (!it->error) {
memcpy(transfer->batch.dma_buffer.virt, it->data.virt, it->size);
transfer->batch.transferred_size = it->size;
transfer->batch.error = it->error;
}
it->state = ISOCH_EMPTY;
fibril_mutex_unlock(&isoch->guard);
usb_transfer_batch_finish(&transfer->batch);
return EOK;
}
void isoch_handle_transfer_event(xhci_hc_t *hc, xhci_endpoint_t *ep,
xhci_trb_t *trb)
{
assert(ep->base.transfer_type == USB_TRANSFER_ISOCHRONOUS);
xhci_isoch_t *const isoch = ep->isoch;
fibril_mutex_lock(&ep->isoch->guard);
errno_t err;
const xhci_trb_completion_code_t completion_code = TRB_COMPLETION_CODE(*trb);
switch (completion_code) {
case XHCI_TRBC_RING_OVERRUN:
case XHCI_TRBC_RING_UNDERRUN:
usb_log_warning("Ring over/underrun.");
isoch_reset_no_timer(ep);
fibril_condvar_broadcast(&ep->isoch->avail);
fibril_mutex_unlock(&ep->isoch->guard);
goto out;
case XHCI_TRBC_SHORT_PACKET:
case XHCI_TRBC_SUCCESS:
err = EOK;
break;
default:
usb_log_warning("Transfer not successfull: %u", completion_code);
err = EIO;
break;
}
bool found_mine = false;
for (size_t i = 0, di = isoch->dequeue; i < isoch->buffer_count; ++i, ++di) {
if (di == isoch->buffer_count) {
di = 0;
}
xhci_isoch_transfer_t *const it = &isoch->transfers[di];
if (it->state == ISOCH_FED && it->interrupt_trb_phys == trb->parameter) {
usb_log_debug("[isoch] buffer %zu completed", it - isoch->transfers);
it->state = ISOCH_COMPLETE;
it->size -= TRB_TRANSFER_LENGTH(*trb);
it->error = err;
found_mine = true;
break;
}
}
if (!found_mine) {
usb_log_warning("[isoch] A transfer event occured for unknown transfer.");
}
timer_schedule_reset(ep);
out:
fibril_condvar_broadcast(&ep->isoch->avail);
fibril_mutex_unlock(&ep->isoch->guard);
}
HelenOS homepage, sources at GitHub