iPXE
xhci.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <unistd.h>
29 #include <string.h>
30 #include <strings.h>
31 #include <errno.h>
32 #include <byteswap.h>
33 #include <ipxe/malloc.h>
34 #include <ipxe/pci.h>
35 #include <ipxe/usb.h>
36 #include <ipxe/init.h>
37 #include <ipxe/profile.h>
38 #include <ipxe/xhci.h>
39 
40 /** @file
41  *
42  * USB eXtensible Host Controller Interface (xHCI) driver
43  *
44  */
45 
46 /** Message transfer profiler */
47 static struct profiler xhci_message_profiler __profiler =
48  { .name = "xhci.message" };
49 
50 /** Stream transfer profiler */
51 static struct profiler xhci_stream_profiler __profiler =
52  { .name = "xhci.stream" };
53 
54 /** Event ring profiler */
55 static struct profiler xhci_event_profiler __profiler =
56  { .name = "xhci.event" };
57 
58 /** Transfer event profiler */
59 static struct profiler xhci_transfer_profiler __profiler =
60  { .name = "xhci.transfer" };
61 
62 /* Disambiguate the various error causes */
63 #define EIO_DATA \
64  __einfo_error ( EINFO_EIO_DATA )
65 #define EINFO_EIO_DATA \
66  __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \
67  "Data buffer error" )
68 #define EIO_BABBLE \
69  __einfo_error ( EINFO_EIO_BABBLE )
70 #define EINFO_EIO_BABBLE \
71  __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \
72  "Babble detected" )
73 #define EIO_USB \
74  __einfo_error ( EINFO_EIO_USB )
75 #define EINFO_EIO_USB \
76  __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \
77  "USB transaction error" )
78 #define EIO_TRB \
79  __einfo_error ( EINFO_EIO_TRB )
80 #define EINFO_EIO_TRB \
81  __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \
82  "TRB error" )
83 #define EIO_STALL \
84  __einfo_error ( EINFO_EIO_STALL )
85 #define EINFO_EIO_STALL \
86  __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \
87  "Stall error" )
88 #define EIO_RESOURCE \
89  __einfo_error ( EINFO_EIO_RESOURCE )
90 #define EINFO_EIO_RESOURCE \
91  __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \
92  "Resource error" )
93 #define EIO_BANDWIDTH \
94  __einfo_error ( EINFO_EIO_BANDWIDTH )
95 #define EINFO_EIO_BANDWIDTH \
96  __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \
97  "Bandwidth error" )
98 #define EIO_NO_SLOTS \
99  __einfo_error ( EINFO_EIO_NO_SLOTS )
100 #define EINFO_EIO_NO_SLOTS \
101  __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \
102  "No slots available" )
103 #define EIO_STREAM_TYPE \
104  __einfo_error ( EINFO_EIO_STREAM_TYPE )
105 #define EINFO_EIO_STREAM_TYPE \
106  __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \
107  "Invalid stream type" )
108 #define EIO_SLOT \
109  __einfo_error ( EINFO_EIO_SLOT )
110 #define EINFO_EIO_SLOT \
111  __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \
112  "Slot not enabled" )
113 #define EIO_ENDPOINT \
114  __einfo_error ( EINFO_EIO_ENDPOINT )
115 #define EINFO_EIO_ENDPOINT \
116  __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \
117  "Endpoint not enabled" )
118 #define EIO_SHORT \
119  __einfo_error ( EINFO_EIO_SHORT )
120 #define EINFO_EIO_SHORT \
121  __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \
122  "Short packet" )
123 #define EIO_UNDERRUN \
124  __einfo_error ( EINFO_EIO_UNDERRUN )
125 #define EINFO_EIO_UNDERRUN \
126  __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \
127  "Ring underrun" )
128 #define EIO_OVERRUN \
129  __einfo_error ( EINFO_EIO_OVERRUN )
130 #define EINFO_EIO_OVERRUN \
131  __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \
132  "Ring overrun" )
133 #define EIO_VF_RING_FULL \
134  __einfo_error ( EINFO_EIO_VF_RING_FULL )
135 #define EINFO_EIO_VF_RING_FULL \
136  __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \
137  "Virtual function event ring full" )
138 #define EIO_PARAMETER \
139  __einfo_error ( EINFO_EIO_PARAMETER )
140 #define EINFO_EIO_PARAMETER \
141  __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \
142  "Parameter error" )
143 #define EIO_BANDWIDTH_OVERRUN \
144  __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
145 #define EINFO_EIO_BANDWIDTH_OVERRUN \
146  __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \
147  "Bandwidth overrun" )
148 #define EIO_CONTEXT \
149  __einfo_error ( EINFO_EIO_CONTEXT )
150 #define EINFO_EIO_CONTEXT \
151  __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \
152  "Context state error" )
153 #define EIO_NO_PING \
154  __einfo_error ( EINFO_EIO_NO_PING )
155 #define EINFO_EIO_NO_PING \
156  __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \
157  "No ping response" )
158 #define EIO_RING_FULL \
159  __einfo_error ( EINFO_EIO_RING_FULL )
160 #define EINFO_EIO_RING_FULL \
161  __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \
162  "Event ring full" )
163 #define EIO_INCOMPATIBLE \
164  __einfo_error ( EINFO_EIO_INCOMPATIBLE )
165 #define EINFO_EIO_INCOMPATIBLE \
166  __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \
167  "Incompatible device" )
168 #define EIO_MISSED \
169  __einfo_error ( EINFO_EIO_MISSED )
170 #define EINFO_EIO_MISSED \
171  __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \
172  "Missed service error" )
173 #define EIO_CMD_STOPPED \
174  __einfo_error ( EINFO_EIO_CMD_STOPPED )
175 #define EINFO_EIO_CMD_STOPPED \
176  __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \
177  "Command ring stopped" )
178 #define EIO_CMD_ABORTED \
179  __einfo_error ( EINFO_EIO_CMD_ABORTED )
180 #define EINFO_EIO_CMD_ABORTED \
181  __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \
182  "Command aborted" )
183 #define EIO_STOP \
184  __einfo_error ( EINFO_EIO_STOP )
185 #define EINFO_EIO_STOP \
186  __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \
187  "Stopped" )
188 #define EIO_STOP_LEN \
189  __einfo_error ( EINFO_EIO_STOP_LEN )
190 #define EINFO_EIO_STOP_LEN \
191  __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \
192  "Stopped - length invalid" )
193 #define EIO_STOP_SHORT \
194  __einfo_error ( EINFO_EIO_STOP_SHORT )
195 #define EINFO_EIO_STOP_SHORT \
196  __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \
197  "Stopped - short packet" )
198 #define EIO_LATENCY \
199  __einfo_error ( EINFO_EIO_LATENCY )
200 #define EINFO_EIO_LATENCY \
201  __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \
202  "Maximum exit latency too large" )
203 #define EIO_ISOCH \
204  __einfo_error ( EINFO_EIO_ISOCH )
205 #define EINFO_EIO_ISOCH \
206  __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \
207  "Isochronous buffer overrun" )
208 #define EPROTO_LOST \
209  __einfo_error ( EINFO_EPROTO_LOST )
210 #define EINFO_EPROTO_LOST \
211  __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \
212  "Event lost" )
213 #define EPROTO_UNDEFINED \
214  __einfo_error ( EINFO_EPROTO_UNDEFINED )
215 #define EINFO_EPROTO_UNDEFINED \
216  __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \
217  "Undefined error" )
218 #define EPROTO_STREAM_ID \
219  __einfo_error ( EINFO_EPROTO_STREAM_ID )
220 #define EINFO_EPROTO_STREAM_ID \
221  __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \
222  "Invalid stream ID" )
223 #define EPROTO_SECONDARY \
224  __einfo_error ( EINFO_EPROTO_SECONDARY )
225 #define EINFO_EPROTO_SECONDARY \
226  __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \
227  "Secondary bandwidth error" )
228 #define EPROTO_SPLIT \
229  __einfo_error ( EINFO_EPROTO_SPLIT )
230 #define EINFO_EPROTO_SPLIT \
231  __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \
232  "Split transaction error" )
233 #define ECODE(code) \
234  ( ( (code) < 32 ) ? \
235  EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \
236  EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \
237  EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \
238  EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \
239  EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \
240  EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \
241  EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \
242  EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \
243  EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \
244  EIO_ISOCH ) : \
245  ( (code) < 64 ) ? \
246  EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \
247  EPROTO_UNDEFINED, EPROTO_STREAM_ID, \
248  EPROTO_SECONDARY, EPROTO_SPLIT ) : \
249  EFAULT )
250 
251 /******************************************************************************
252  *
253  * Register access
254  *
255  ******************************************************************************
256  */
257 
258 /**
259  * Initialise device
260  *
261  * @v xhci xHCI device
262  */
263 void xhci_init ( struct xhci_device *xhci ) {
264  uint32_t hcsparams1;
265  uint32_t hcsparams2;
266  uint32_t hccparams1;
267  uint32_t pagesize;
268  size_t caplength;
269  size_t rtsoff;
270  size_t dboff;
271 
272  /* Set device name */
273  xhci->name = xhci->dev->name;
274 
275  /* Locate capability, operational, runtime, and doorbell registers */
276  xhci->cap = xhci->regs;
277  caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
278  rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
279  dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
280  xhci->op = ( xhci->cap + caplength );
281  xhci->run = ( xhci->cap + rtsoff );
282  xhci->db = ( xhci->cap + dboff );
283  DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n",
284  xhci->name, virt_to_phys ( xhci->cap ),
285  virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ),
286  virt_to_phys ( xhci->db ) );
287 
288  /* Read structural parameters 1 */
289  hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
290  xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
291  xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
292  xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
293  DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n",
294  xhci->name, xhci->slots, xhci->intrs, xhci->ports );
295 
296  /* Read structural parameters 2 */
297  hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
298  xhci->scratch.count = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
299  DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
300  xhci->name, xhci->scratch.count );
301 
302  /* Read capability parameters 1 */
303  hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
304  xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
305  xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
306  xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
307 
308  /* Read page size */
309  pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
310  xhci->pagesize = XHCI_PAGESIZE ( pagesize );
311  assert ( xhci->pagesize != 0 );
312  assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
313  DBGC2 ( xhci, "XHCI %s page size %zd bytes\n",
314  xhci->name, xhci->pagesize );
315 
316  /* Configure DMA device */
317  if ( xhci->dma && xhci->addr64 )
318  dma_set_mask_64bit ( xhci->dma );
319 }
320 
321 /**
322  * Find extended capability
323  *
324  * @v xhci xHCI device
325  * @v id Capability ID
326  * @v offset Offset to previous extended capability instance, or zero
327  * @ret offset Offset to extended capability, or zero if not found
328  */
329 static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
330  unsigned int id,
331  unsigned int offset ) {
332  uint32_t xecp;
333  unsigned int next;
334 
335  /* Locate the extended capability */
336  while ( 1 ) {
337 
338  /* Locate first or next capability as applicable */
339  if ( offset ) {
340  xecp = readl ( xhci->cap + offset );
341  next = XHCI_XECP_NEXT ( xecp );
342  } else {
343  next = xhci->xecp;
344  }
345  if ( ! next )
346  return 0;
347  offset += next;
348 
349  /* Check if this is the requested capability */
350  xecp = readl ( xhci->cap + offset );
351  if ( XHCI_XECP_ID ( xecp ) == id )
352  return offset;
353  }
354 }
355 
356 /**
357  * Write potentially 64-bit register
358  *
359  * @v xhci xHCI device
360  * @v value Value
361  * @v reg Register address
362  * @ret rc Return status code
363  */
364 static inline __attribute__ (( always_inline )) int
365 xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
366 
367  /* If this is a 32-bit build, then this can never fail
368  * (allowing the compiler to optimise out the error path).
369  */
370  if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
371  writel ( value, reg );
372  writel ( 0, ( reg + sizeof ( uint32_t ) ) );
373  return 0;
374  }
375 
376  /* If the device does not support 64-bit addresses and this
377  * address is outside the 32-bit address space, then fail.
378  */
379  if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
380  DBGC ( xhci, "XHCI %s cannot access address %lx\n",
381  xhci->name, value );
382  return -ENOTSUP;
383  }
384 
385  /* If this is a 64-bit build, then writeq() is available */
386  writeq ( value, reg );
387  return 0;
388 }
389 
390 /**
391  * Calculate buffer alignment
392  *
393  * @v len Length
394  * @ret align Buffer alignment
395  *
396  * Determine alignment required for a buffer which must be aligned to
397  * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
398  */
399 static inline size_t xhci_align ( size_t len ) {
400  size_t align;
401 
402  /* Align to own length (rounded up to a power of two) */
403  align = ( 1 << fls ( len - 1 ) );
404 
405  /* Round up to XHCI_MIN_ALIGN if needed */
406  if ( align < XHCI_MIN_ALIGN )
407  align = XHCI_MIN_ALIGN;
408 
409  return align;
410 }
411 
412 /**
413  * Calculate device context offset
414  *
415  * @v xhci xHCI device
416  * @v ctx Context index
417  */
418 static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
419  unsigned int ctx ) {
420 
421  return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
422 }
423 
424 /**
425  * Calculate input context offset
426  *
427  * @v xhci xHCI device
428  * @v ctx Context index
429  */
430 static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
431  unsigned int ctx ) {
432 
433  return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
434 }
435 
436 /******************************************************************************
437  *
438  * Diagnostics
439  *
440  ******************************************************************************
441  */
442 
443 /**
444  * Dump host controller registers
445  *
446  * @v xhci xHCI device
447  */
448 static inline void xhci_dump ( struct xhci_device *xhci ) {
449  uint32_t usbcmd;
450  uint32_t usbsts;
451  uint32_t pagesize;
452  uint32_t dnctrl;
453  uint32_t config;
454 
455  /* Do nothing unless debugging is enabled */
456  if ( ! DBG_LOG )
457  return;
458 
459  /* Dump USBCMD */
460  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
461  DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd,
462  ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
463  ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
464 
465  /* Dump USBSTS */
466  usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
467  DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts,
468  ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
469 
470  /* Dump PAGESIZE */
471  pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
472  DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize );
473 
474  /* Dump DNCTRL */
475  dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
476  DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl );
477 
478  /* Dump CONFIG */
479  config = readl ( xhci->op + XHCI_OP_CONFIG );
480  DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config );
481 }
482 
483 /**
484  * Dump port registers
485  *
486  * @v xhci xHCI device
487  * @v port Port number
488  */
489 static inline void xhci_dump_port ( struct xhci_device *xhci,
490  unsigned int port ) {
491  uint32_t portsc;
492  uint32_t portpmsc;
493  uint32_t portli;
494  uint32_t porthlpmc;
495 
496  /* Do nothing unless debugging is enabled */
497  if ( ! DBG_LOG )
498  return;
499 
500  /* Dump PORTSC */
501  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
502  DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n",
503  xhci->name, port, portsc,
504  ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
505  ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
506  ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
507  ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
508  XHCI_PORTSC_PSIV ( portsc ) );
509 
510  /* Dump PORTPMSC */
511  portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
512  DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc );
513 
514  /* Dump PORTLI */
515  portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
516  DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli );
517 
518  /* Dump PORTHLPMC */
519  porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
520  DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n",
521  xhci->name, port, porthlpmc );
522 }
523 
524 /******************************************************************************
525  *
526  * USB legacy support
527  *
528  ******************************************************************************
529  */
530 
531 /** Prevent the release of ownership back to BIOS */
532 static int xhci_legacy_prevent_release;
533 
534 /**
535  * Initialise USB legacy support
536  *
537  * @v xhci xHCI device
538  */
539 static void xhci_legacy_init ( struct xhci_device *xhci ) {
540  unsigned int legacy;
541  uint8_t bios;
542 
543  /* Locate USB legacy support capability (if present) */
544  legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
545  if ( ! legacy ) {
546  /* Not an error; capability may not be present */
547  DBGC ( xhci, "XHCI %s has no USB legacy support capability\n",
548  xhci->name );
549  return;
550  }
551 
552  /* Check if legacy USB support is enabled */
553  bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
554  if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
555  /* Not an error; already owned by OS */
556  DBGC ( xhci, "XHCI %s USB legacy support already disabled\n",
557  xhci->name );
558  return;
559  }
560 
561  /* Record presence of USB legacy support capability */
562  xhci->legacy = legacy;
563 }
564 
565 /**
566  * Claim ownership from BIOS
567  *
568  * @v xhci xHCI device
569  */
570 static void xhci_legacy_claim ( struct xhci_device *xhci ) {
571  uint32_t ctlsts;
572  uint8_t bios;
573  unsigned int i;
574 
575  /* Do nothing unless legacy support capability is present */
576  if ( ! xhci->legacy )
577  return;
578 
579  /* Claim ownership */
580  writeb ( XHCI_USBLEGSUP_OS_OWNED,
581  xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
582 
583  /* Wait for BIOS to release ownership */
584  for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
585 
586  /* Check if BIOS has released ownership */
587  bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
588  if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
589  DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n",
590  xhci->name );
591  ctlsts = readl ( xhci->cap + xhci->legacy +
592  XHCI_USBLEGSUP_CTLSTS );
593  if ( ctlsts ) {
594  DBGC ( xhci, "XHCI %s warning: BIOS retained "
595  "SMIs: %08x\n", xhci->name, ctlsts );
596  }
597  return;
598  }
599 
600  /* Delay */
601  mdelay ( 1 );
602  }
603 
604  /* BIOS did not release ownership. Claim it forcibly by
605  * disabling all SMIs.
606  */
607  DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly "
608  "disabling SMIs\n", xhci->name );
609  writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS );
610 }
611 
612 /**
613  * Release ownership back to BIOS
614  *
615  * @v xhci xHCI device
616  */
617 static void xhci_legacy_release ( struct xhci_device *xhci ) {
618 
619  /* Do nothing unless legacy support capability is present */
620  if ( ! xhci->legacy )
621  return;
622 
623  /* Do nothing if releasing ownership is prevented */
624  if ( xhci_legacy_prevent_release ) {
625  DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n",
626  xhci->name );
627  return;
628  }
629 
630  /* Release ownership */
631  writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
632  DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name );
633 }
634 
635 /******************************************************************************
636  *
637  * Supported protocols
638  *
639  ******************************************************************************
640  */
641 
642 /**
643  * Transcribe port speed (for debugging)
644  *
645  * @v psi Protocol speed ID
646  * @ret speed Transcribed speed
647  */
648 static inline const char * xhci_speed_name ( uint32_t psi ) {
649  static const char *exponents[4] = { "", "k", "M", "G" };
650  static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
651  unsigned int mantissa;
652  unsigned int exponent;
653 
654  /* Extract mantissa and exponent */
655  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
656  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
657 
658  /* Transcribe speed */
659  snprintf ( buf, sizeof ( buf ), "%d%sbps",
660  mantissa, exponents[exponent] );
661  return buf;
662 }
663 
664 /**
665  * Find supported protocol extended capability for a port
666  *
667  * @v xhci xHCI device
668  * @v port Port number
669  * @ret supported Offset to extended capability, or zero if not found
670  */
671 static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
672  unsigned int port ) {
673  unsigned int supported = 0;
674  unsigned int offset;
675  unsigned int count;
676  uint32_t ports;
677 
678  /* Iterate over all supported protocol structures */
679  while ( ( supported = xhci_extended_capability ( xhci,
680  XHCI_XECP_ID_SUPPORTED,
681  supported ) ) ) {
682 
683  /* Determine port range */
684  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
685  offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
686  count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
687 
688  /* Check if port lies within this range */
689  if ( ( port - offset ) < count )
690  return supported;
691  }
692 
693  DBGC ( xhci, "XHCI %s-%d has no supported protocol\n",
694  xhci->name, port );
695  return 0;
696 }
697 
698 /**
699  * Find port protocol
700  *
701  * @v xhci xHCI device
702  * @v port Port number
703  * @ret protocol USB protocol, or zero if not found
704  */
705 static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
706  unsigned int port ) {
707  unsigned int supported = xhci_supported_protocol ( xhci, port );
708  union {
709  uint32_t raw;
710  char text[5];
711  } name;
712  unsigned int protocol;
713  unsigned int type;
714  unsigned int psic;
715  unsigned int psiv;
716  unsigned int i;
717  uint32_t revision;
718  uint32_t ports;
719  uint32_t slot;
720  uint32_t psi;
721 
722  /* Fail if there is no supported protocol */
723  if ( ! supported )
724  return 0;
725 
726  /* Determine protocol version */
727  revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
728  protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
729 
730  /* Describe port protocol */
731  if ( DBG_EXTRA ) {
732  name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
733  XHCI_SUPPORTED_NAME ) );
734  name.text[4] = '\0';
735  slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
736  type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
737  DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d",
738  xhci->name, port, name.text, protocol, type );
739  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
740  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
741  if ( psic ) {
742  DBGC2 ( xhci, " speeds" );
743  for ( i = 0 ; i < psic ; i++ ) {
744  psi = readl ( xhci->cap + supported +
745  XHCI_SUPPORTED_PSI ( i ) );
746  psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
747  DBGC2 ( xhci, " %d:%s", psiv,
748  xhci_speed_name ( psi ) );
749  }
750  }
751  if ( xhci->quirks & XHCI_BAD_PSIV )
752  DBGC2 ( xhci, " (ignored)" );
753  DBGC2 ( xhci, "\n" );
754  }
755 
756  return protocol;
757 }
758 
759 /**
760  * Find port slot type
761  *
762  * @v xhci xHCI device
763  * @v port Port number
764  * @ret type Slot type, or negative error
765  */
766 static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
767  unsigned int supported = xhci_supported_protocol ( xhci, port );
768  unsigned int type;
769  uint32_t slot;
770 
771  /* Fail if there is no supported protocol */
772  if ( ! supported )
773  return -ENOTSUP;
774 
775  /* Get slot type */
776  slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
777  type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
778 
779  return type;
780 }
781 
782 /**
783  * Find port speed
784  *
785  * @v xhci xHCI device
786  * @v port Port number
787  * @v psiv Protocol speed ID value
788  * @ret speed Port speed, or negative error
789  */
790 static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
791  unsigned int psiv ) {
792  unsigned int supported = xhci_supported_protocol ( xhci, port );
793  unsigned int psic;
794  unsigned int mantissa;
795  unsigned int exponent;
796  unsigned int speed;
797  unsigned int i;
798  uint32_t ports;
799  uint32_t psi;
800 
801  /* Fail if there is no supported protocol */
802  if ( ! supported )
803  return -ENOTSUP;
804 
805  /* Get protocol speed ID count */
806  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
807  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
808 
809  /* Use protocol speed ID table unless device is known to be faulty */
810  if ( ! ( xhci->quirks & XHCI_BAD_PSIV ) ) {
811 
812  /* Iterate over PSI dwords looking for a match */
813  for ( i = 0 ; i < psic ; i++ ) {
814  psi = readl ( xhci->cap + supported +
815  XHCI_SUPPORTED_PSI ( i ) );
816  if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
817  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
818  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
819  speed = USB_SPEED ( mantissa, exponent );
820  return speed;
821  }
822  }
823 
824  /* Record device as faulty if no match is found */
825  if ( psic != 0 ) {
826  DBGC ( xhci, "XHCI %s-%d spurious PSI value %d: "
827  "assuming PSI table is invalid\n",
828  xhci->name, port, psiv );
829  xhci->quirks |= XHCI_BAD_PSIV;
830  }
831  }
832 
833  /* Use the default mappings */
834  switch ( psiv ) {
835  case XHCI_SPEED_LOW : return USB_SPEED_LOW;
836  case XHCI_SPEED_FULL : return USB_SPEED_FULL;
837  case XHCI_SPEED_HIGH : return USB_SPEED_HIGH;
838  case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
839  default:
840  DBGC ( xhci, "XHCI %s-%d unrecognised PSI value %d\n",
841  xhci->name, port, psiv );
842  return -ENOTSUP;
843  }
844 }
845 
846 /**
847  * Find protocol speed ID value
848  *
849  * @v xhci xHCI device
850  * @v port Port number
851  * @v speed USB speed
852  * @ret psiv Protocol speed ID value, or negative error
853  */
854 static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
855  unsigned int speed ) {
856  unsigned int supported = xhci_supported_protocol ( xhci, port );
857  unsigned int psic;
858  unsigned int mantissa;
859  unsigned int exponent;
860  unsigned int psiv;
861  unsigned int i;
862  uint32_t ports;
863  uint32_t psi;
864 
865  /* Fail if there is no supported protocol */
866  if ( ! supported )
867  return -ENOTSUP;
868 
869  /* Get protocol speed ID count */
870  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
871  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
872 
873  /* Use the default mappings if applicable */
874  if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
875  switch ( speed ) {
876  case USB_SPEED_LOW : return XHCI_SPEED_LOW;
877  case USB_SPEED_FULL : return XHCI_SPEED_FULL;
878  case USB_SPEED_HIGH : return XHCI_SPEED_HIGH;
879  case USB_SPEED_SUPER : return XHCI_SPEED_SUPER;
880  default:
881  DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n",
882  xhci->name, port, speed );
883  return -ENOTSUP;
884  }
885  }
886 
887  /* Iterate over PSI dwords looking for a match */
888  for ( i = 0 ; i < psic ; i++ ) {
889  psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
890  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
891  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
892  if ( speed == USB_SPEED ( mantissa, exponent ) ) {
893  psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
894  return psiv;
895  }
896  }
897 
898  DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n",
899  xhci->name, port, speed );
900  return -ENOENT;
901 }
902 
903 /******************************************************************************
904  *
905  * Device context base address array
906  *
907  ******************************************************************************
908  */
909 
910 /**
911  * Allocate device context base address array
912  *
913  * @v xhci xHCI device
914  * @ret rc Return status code
915  */
916 static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
917  size_t len;
918  physaddr_t dcbaap;
919  int rc;
920 
921  /* Allocate and initialise structure. Must be at least
922  * 64-byte aligned and must not cross a page boundary, so
923  * align on its own size (rounded up to a power of two and
924  * with a minimum of 64 bytes).
925  */
926  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
927  xhci->dcbaa.context = dma_alloc ( xhci->dma, &xhci->dcbaa.map, len,
928  xhci_align ( len ) );
929  if ( ! xhci->dcbaa.context ) {
930  DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
931  rc = -ENOMEM;
932  goto err_alloc;
933  }
934  memset ( xhci->dcbaa.context, 0, len );
935 
936  /* Program DCBAA pointer */
937  dcbaap = dma ( &xhci->dcbaa.map, xhci->dcbaa.context );
938  if ( ( rc = xhci_writeq ( xhci, dcbaap,
939  xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
940  goto err_writeq;
941 
942  DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n", xhci->name,
943  virt_to_phys ( xhci->dcbaa.context ),
944  ( virt_to_phys ( xhci->dcbaa.context ) + len ) );
945  return 0;
946 
947  err_writeq:
948  dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
949  err_alloc:
950  return rc;
951 }
952 
953 /**
954  * Free device context base address array
955  *
956  * @v xhci xHCI device
957  */
958 static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
959  size_t len;
960  unsigned int i;
961 
962  /* Sanity check */
963  for ( i = 0 ; i <= xhci->slots ; i++ )
964  assert ( xhci->dcbaa.context[i] == 0 );
965 
966  /* Clear DCBAA pointer */
967  xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
968 
969  /* Free DCBAA */
970  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
971  dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
972 }
973 
974 /******************************************************************************
975  *
976  * Scratchpad buffers
977  *
978  ******************************************************************************
979  */
980 
981 /**
982  * Allocate scratchpad buffers
983  *
984  * @v xhci xHCI device
985  * @ret rc Return status code
986  */
987 static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
988  struct xhci_scratchpad *scratch = &xhci->scratch;
989  size_t buffer_len;
990  size_t array_len;
991  physaddr_t addr;
992  unsigned int i;
993  int rc;
994 
995  /* Do nothing if no scratchpad buffers are used */
996  if ( ! scratch->count )
997  return 0;
998 
999  /* Allocate scratchpad buffers */
1000  buffer_len = ( scratch->count * xhci->pagesize );
1001  scratch->buffer = dma_umalloc ( xhci->dma, &scratch->buffer_map,
1002  buffer_len, xhci->pagesize );
1003  if ( ! scratch->buffer ) {
1004  DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
1005  xhci->name );
1006  rc = -ENOMEM;
1007  goto err_alloc;
1008  }
1009  memset ( scratch->buffer, 0, buffer_len );
1010 
1011  /* Allocate scratchpad array */
1012  array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
1013  scratch->array = dma_alloc ( xhci->dma, &scratch->array_map,
1014  array_len, xhci_align ( array_len ) );
1015  if ( ! scratch->array ) {
1016  DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
1017  "array\n", xhci->name );
1018  rc = -ENOMEM;
1019  goto err_alloc_array;
1020  }
1021 
1022  /* Populate scratchpad array */
1023  addr = dma ( &scratch->buffer_map, scratch->buffer );
1024  for ( i = 0 ; i < scratch->count ; i++ ) {
1025  scratch->array[i] = cpu_to_le64 ( addr );
1026  addr += xhci->pagesize;
1027  }
1028 
1029  /* Set scratchpad array pointer */
1030  assert ( xhci->dcbaa.context != NULL );
1031  xhci->dcbaa.context[0] = cpu_to_le64 ( dma ( &scratch->array_map,
1032  scratch->array ) );
1033 
1034  DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
1035  xhci->name, virt_to_phys ( scratch->buffer ),
1036  ( virt_to_phys ( scratch->buffer ) + buffer_len ),
1037  virt_to_phys ( scratch->array ),
1038  ( virt_to_phys ( scratch->array ) + array_len ) );
1039  return 0;
1040 
1041  dma_free ( &scratch->array_map, scratch->array, array_len );
1042  err_alloc_array:
1043  dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
1044  err_alloc:
1045  return rc;
1046 }
1047 
1048 /**
1049  * Free scratchpad buffers
1050  *
1051  * @v xhci xHCI device
1052  */
1053 static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
1054  struct xhci_scratchpad *scratch = &xhci->scratch;
1055  size_t array_len;
1056  size_t buffer_len;
1057 
1058  /* Do nothing if no scratchpad buffers are used */
1059  if ( ! scratch->count )
1060  return;
1061 
1062  /* Clear scratchpad array pointer */
1063  assert ( xhci->dcbaa.context != NULL );
1064  xhci->dcbaa.context[0] = 0;
1065 
1066  /* Free scratchpad array */
1067  array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
1068  dma_free ( &scratch->array_map, scratch->array, array_len );
1069 
1070  /* Free scratchpad buffers */
1071  buffer_len = ( scratch->count * xhci->pagesize );
1072  dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
1073 }
1074 
1075 /******************************************************************************
1076  *
1077  * Run / stop / reset
1078  *
1079  ******************************************************************************
1080  */
1081 
1082 /**
1083  * Start xHCI device
1084  *
1085  * @v xhci xHCI device
1086  */
1087 static void xhci_run ( struct xhci_device *xhci ) {
1088  uint32_t config;
1089  uint32_t usbcmd;
1090 
1091  /* Configure number of device slots */
1092  config = readl ( xhci->op + XHCI_OP_CONFIG );
1093  config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
1094  config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
1095  writel ( config, xhci->op + XHCI_OP_CONFIG );
1096 
1097  /* Set run/stop bit */
1098  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1099  usbcmd |= XHCI_USBCMD_RUN;
1100  writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
1101 }
1102 
1103 /**
1104  * Stop xHCI device
1105  *
1106  * @v xhci xHCI device
1107  * @ret rc Return status code
1108  */
1109 static int xhci_stop ( struct xhci_device *xhci ) {
1110  uint32_t usbcmd;
1111  uint32_t usbsts;
1112  unsigned int i;
1113 
1114  /* Clear run/stop bit */
1115  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1116  usbcmd &= ~XHCI_USBCMD_RUN;
1117  writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
1118 
1119  /* Wait for device to stop */
1120  for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
1121 
1122  /* Check if device is stopped */
1123  usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
1124  if ( usbsts & XHCI_USBSTS_HCH )
1125  return 0;
1126 
1127  /* Delay */
1128  mdelay ( 1 );
1129  }
1130 
1131  DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name );
1132  return -ETIMEDOUT;
1133 }
1134 
1135 /**
1136  * Reset xHCI device
1137  *
1138  * @v xhci xHCI device
1139  * @ret rc Return status code
1140  */
1141 static int xhci_reset ( struct xhci_device *xhci ) {
1142  uint32_t usbcmd;
1143  unsigned int i;
1144  int rc;
1145 
1146  /* The xHCI specification states that resetting a running
1147  * device may result in undefined behaviour, so try stopping
1148  * it first.
1149  */
1150  if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
1151  /* Ignore errors and attempt to reset the device anyway */
1152  }
1153 
1154  /* Reset device */
1155  writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
1156 
1157  /* Wait for reset to complete */
1158  for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
1159 
1160  /* Check if reset is complete */
1161  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1162  if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
1163  return 0;
1164 
1165  /* Delay */
1166  mdelay ( 1 );
1167  }
1168 
1169  DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name );
1170  return -ETIMEDOUT;
1171 }
1172 
1173 /**
1174  * Mark xHCI device as permanently failed
1175  *
1176  * @v xhci xHCI device
1177  * @ret rc Return status code
1178  */
1179 static int xhci_fail ( struct xhci_device *xhci ) {
1180  size_t len;
1181  int rc;
1182 
1183  /* Mark command mechanism as permanently failed */
1184  xhci->failed = 1;
1185 
1186  /* Reset device */
1187  if ( ( rc = xhci_reset ( xhci ) ) != 0 )
1188  return rc;
1189 
1190  /* Discard DCBAA entries since DCBAAP has been cleared */
1191  assert ( xhci->dcbaa.context != NULL );
1192  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
1193  memset ( xhci->dcbaa.context, 0, len );
1194 
1195  return 0;
1196 }
1197 
1198 /******************************************************************************
1199  *
1200  * Transfer request blocks
1201  *
1202  ******************************************************************************
1203  */
1204 
1205 /**
1206  * Allocate transfer request block ring
1207  *
1208  * @v xhci xHCI device
1209  * @v ring TRB ring
1210  * @v shift Ring size (log2)
1211  * @v slot Device slot
1212  * @v target Doorbell target
1213  * @v stream Doorbell stream ID
1214  * @ret rc Return status code
1215  */
1216 static int xhci_ring_alloc ( struct xhci_device *xhci,
1217  struct xhci_trb_ring *ring,
1218  unsigned int shift, unsigned int slot,
1219  unsigned int target, unsigned int stream ) {
1220  struct xhci_trb_link *link;
1221  unsigned int count;
1222  int rc;
1223 
1224  /* Sanity check */
1225  assert ( shift > 0 );
1226 
1227  /* Initialise structure */
1228  memset ( ring, 0, sizeof ( *ring ) );
1229  ring->shift = shift;
1230  count = ( 1U << shift );
1231  ring->mask = ( count - 1 );
1232  ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
1233  ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
1234  ring->dbval = XHCI_DBVAL ( target, stream );
1235 
1236  /* Allocate I/O buffers */
1237  ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
1238  if ( ! ring->iobuf ) {
1239  rc = -ENOMEM;
1240  goto err_alloc_iobuf;
1241  }
1242 
1243  /* Allocate TRBs */
1244  ring->trb = dma_alloc ( xhci->dma, &ring->map, ring->len,
1245  xhci_align ( ring->len ) );
1246  if ( ! ring->trb ) {
1247  rc = -ENOMEM;
1248  goto err_alloc_trb;
1249  }
1250  memset ( ring->trb, 0, ring->len );
1251 
1252  /* Initialise Link TRB */
1253  link = &ring->trb[count].link;
1254  link->next = cpu_to_le64 ( dma ( &ring->map, ring->trb ) );
1255  link->flags = XHCI_TRB_TC;
1256  link->type = XHCI_TRB_LINK;
1257  ring->link = link;
1258 
1259  return 0;
1260 
1261  dma_free ( &ring->map, ring->trb, ring->len );
1262  err_alloc_trb:
1263  free ( ring->iobuf );
1264  err_alloc_iobuf:
1265  return rc;
1266 }
1267 
1268 /**
1269  * Reset transfer request block ring
1270  *
1271  * @v ring TRB ring
1272  */
1273 static void xhci_ring_reset ( struct xhci_trb_ring *ring ) {
1274  unsigned int count = ( 1U << ring->shift );
1275 
1276  /* Reset producer and consumer counters */
1277  ring->prod = 0;
1278  ring->cons = 0;
1279 
1280  /* Reset TRBs (except Link TRB) */
1281  memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) );
1282 }
1283 
1284 /**
1285  * Free transfer request block ring
1286  *
1287  * @v ring TRB ring
1288  */
1289 static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
1290  unsigned int count = ( 1U << ring->shift );
1291  unsigned int i;
1292 
1293  /* Sanity checks */
1294  assert ( ring->cons == ring->prod );
1295  for ( i = 0 ; i < count ; i++ )
1296  assert ( ring->iobuf[i] == NULL );
1297 
1298  /* Free TRBs */
1299  dma_free ( &ring->map, ring->trb, ring->len );
1300 
1301  /* Free I/O buffers */
1302  free ( ring->iobuf );
1303 }
1304 
1305 /**
1306  * Enqueue a transfer request block
1307  *
1308  * @v ring TRB ring
1309  * @v iobuf I/O buffer (if any)
1310  * @v trb Transfer request block (with empty Cycle flag)
1311  * @ret rc Return status code
1312  *
1313  * This operation does not implicitly ring the doorbell register.
1314  */
1315 static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
1316  const union xhci_trb *trb ) {
1317  union xhci_trb *dest;
1318  unsigned int prod;
1319  unsigned int mask;
1320  unsigned int index;
1321  unsigned int cycle;
1322 
1323  /* Sanity check */
1324  assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
1325 
1326  /* Fail if ring is full */
1327  if ( ! xhci_ring_remaining ( ring ) )
1328  return -ENOBUFS;
1329 
1330  /* Update producer counter (and link TRB, if applicable) */
1331  prod = ring->prod++;
1332  mask = ring->mask;
1333  cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
1334  index = ( prod & mask );
1335  if ( index == 0 )
1336  ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
1337 
1338  /* Record I/O buffer */
1339  ring->iobuf[index] = iobuf;
1340 
1341  /* Enqueue TRB */
1342  dest = &ring->trb[index];
1343  dest->template.parameter = trb->template.parameter;
1344  dest->template.status = trb->template.status;
1345  wmb();
1346  dest->template.control = ( trb->template.control |
1347  cpu_to_le32 ( cycle ) );
1348 
1349  return 0;
1350 }
1351 
1352 /**
1353  * Dequeue a transfer request block
1354  *
1355  * @v ring TRB ring
1356  * @ret iobuf I/O buffer
1357  */
1358 static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
1359  struct io_buffer *iobuf;
1360  unsigned int cons;
1361  unsigned int mask;
1362  unsigned int index;
1363 
1364  /* Sanity check */
1365  assert ( xhci_ring_fill ( ring ) != 0 );
1366 
1367  /* Update consumer counter */
1368  cons = ring->cons++;
1369  mask = ring->mask;
1370  index = ( cons & mask );
1371 
1372  /* Retrieve I/O buffer */
1373  iobuf = ring->iobuf[index];
1374  ring->iobuf[index] = NULL;
1375 
1376  return iobuf;
1377 }
1378 
1379 /**
1380  * Enqueue multiple transfer request blocks
1381  *
1382  * @v ring TRB ring
1383  * @v iobuf I/O buffer
1384  * @v trbs Transfer request blocks (with empty Cycle flag)
1385  * @v count Number of transfer request blocks
1386  * @ret rc Return status code
1387  *
1388  * This operation does not implicitly ring the doorbell register.
1389  */
1390 static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
1391  struct io_buffer *iobuf,
1392  const union xhci_trb *trbs,
1393  unsigned int count ) {
1394  const union xhci_trb *trb = trbs;
1395  int rc;
1396 
1397  /* Sanity check */
1398  assert ( iobuf != NULL );
1399 
1400  /* Fail if ring does not have sufficient space */
1401  if ( xhci_ring_remaining ( ring ) < count )
1402  return -ENOBUFS;
1403 
1404  /* Enqueue each TRB, recording the I/O buffer with the final TRB */
1405  while ( count-- ) {
1406  rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
1407  assert ( rc == 0 ); /* Should never be able to fail */
1408  }
1409 
1410  return 0;
1411 }
1412 
1413 /**
1414  * Dequeue multiple transfer request blocks
1415  *
1416  * @v ring TRB ring
1417  * @ret iobuf I/O buffer
1418  */
1419 static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
1420  struct io_buffer *iobuf;
1421 
1422  /* Dequeue TRBs until we reach the final TRB for an I/O buffer */
1423  do {
1424  iobuf = xhci_dequeue ( ring );
1425  } while ( iobuf == NULL );
1426 
1427  return iobuf;
1428 }
1429 
1430 /**
1431  * Ring doorbell register
1432  *
1433  * @v ring TRB ring
1434  */
1435 static inline __attribute__ (( always_inline )) void
1436 xhci_doorbell ( struct xhci_trb_ring *ring ) {
1437 
1438  wmb();
1439  writel ( ring->dbval, ring->db );
1440 }
1441 
1442 /******************************************************************************
1443  *
1444  * Command and event rings
1445  *
1446  ******************************************************************************
1447  */
1448 
1449 /**
1450  * Allocate command ring
1451  *
1452  * @v xhci xHCI device
1453  * @ret rc Return status code
1454  */
1455 static int xhci_command_alloc ( struct xhci_device *xhci ) {
1456  physaddr_t crp;
1457  int rc;
1458 
1459  /* Allocate TRB ring */
1460  if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
1461  0, 0, 0 ) ) != 0 )
1462  goto err_ring_alloc;
1463 
1464  /* Program command ring control register */
1465  crp = dma ( &xhci->command.map, xhci->command.trb );
1466  if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
1467  xhci->op + XHCI_OP_CRCR ) ) != 0 )
1468  goto err_writeq;
1469 
1470  DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n", xhci->name,
1471  virt_to_phys ( xhci->command.trb ),
1472  ( virt_to_phys ( xhci->command.trb ) + xhci->command.len ) );
1473  return 0;
1474 
1475  err_writeq:
1476  xhci_ring_free ( &xhci->command );
1477  err_ring_alloc:
1478  return rc;
1479 }
1480 
1481 /**
1482  * Free command ring
1483  *
1484  * @v xhci xHCI device
1485  */
1486 static void xhci_command_free ( struct xhci_device *xhci ) {
1487 
1488  /* Sanity check */
1489  assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
1490 
1491  /* Clear command ring control register */
1492  xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
1493 
1494  /* Free TRB ring */
1495  xhci_ring_free ( &xhci->command );
1496 }
1497 
1498 /**
1499  * Allocate event ring
1500  *
1501  * @v xhci xHCI device
1502  * @ret rc Return status code
1503  */
1504 static int xhci_event_alloc ( struct xhci_device *xhci ) {
1505  struct xhci_event_ring *event = &xhci->event;
1506  unsigned int count;
1507  size_t len;
1508  int rc;
1509 
1510  /* Allocate event ring */
1511  count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
1512  len = ( count * sizeof ( event->trb[0] ) );
1513  event->trb = dma_alloc ( xhci->dma, &event->trb_map, len,
1514  xhci_align ( len ) );
1515  if ( ! event->trb ) {
1516  rc = -ENOMEM;
1517  goto err_alloc_trb;
1518  }
1519  memset ( event->trb, 0, len );
1520 
1521  /* Allocate event ring segment table */
1522  event->segment = dma_alloc ( xhci->dma, &event->segment_map,
1523  sizeof ( event->segment[0] ),
1524  xhci_align ( sizeof (event->segment[0])));
1525  if ( ! event->segment ) {
1526  rc = -ENOMEM;
1527  goto err_alloc_segment;
1528  }
1529  memset ( event->segment, 0, sizeof ( event->segment[0] ) );
1530  event->segment[0].base = cpu_to_le64 ( dma ( &event->trb_map,
1531  event->trb ) );
1532  event->segment[0].count = cpu_to_le32 ( count );
1533 
1534  /* Program event ring registers */
1535  writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
1536  if ( ( rc = xhci_writeq ( xhci, dma ( &event->trb_map, event->trb ),
1537  xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
1538  goto err_writeq_erdp;
1539  if ( ( rc = xhci_writeq ( xhci,
1540  dma ( &event->segment_map, event->segment ),
1541  xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
1542  goto err_writeq_erstba;
1543 
1544  DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
1545  xhci->name, virt_to_phys ( event->trb ),
1546  ( virt_to_phys ( event->trb ) + len ),
1547  virt_to_phys ( event->segment ),
1548  ( virt_to_phys ( event->segment ) +
1549  sizeof ( event->segment[0] ) ) );
1550  return 0;
1551 
1552  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
1553  err_writeq_erstba:
1554  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
1555  err_writeq_erdp:
1556  dma_free ( &event->segment_map, event->segment,
1557  sizeof ( event->segment[0] ) );
1558  err_alloc_segment:
1559  dma_free ( &event->trb_map, event->trb, len );
1560  err_alloc_trb:
1561  return rc;
1562 }
1563 
1564 /**
1565  * Free event ring
1566  *
1567  * @v xhci xHCI device
1568  */
1569 static void xhci_event_free ( struct xhci_device *xhci ) {
1570  struct xhci_event_ring *event = &xhci->event;
1571  unsigned int count;
1572  size_t len;
1573 
1574  /* Clear event ring registers */
1575  writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
1576  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
1577  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
1578 
1579  /* Free event ring segment table */
1580  dma_free ( &event->segment_map, event->segment,
1581  sizeof ( event->segment[0] ) );
1582 
1583  /* Free event ring */
1584  count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
1585  len = ( count * sizeof ( event->trb[0] ) );
1586  dma_free ( &event->trb_map, event->trb, len );
1587 }
1588 
1589 /**
1590  * Handle transfer event
1591  *
1592  * @v xhci xHCI device
1593  * @v trb Transfer event TRB
1594  */
1595 static void xhci_transfer ( struct xhci_device *xhci,
1596  struct xhci_trb_transfer *trb ) {
1597  struct xhci_slot *slot;
1598  struct xhci_endpoint *endpoint;
1599  struct io_buffer *iobuf;
1600  int rc;
1601 
1602  /* Profile transfer events */
1603  profile_start ( &xhci_transfer_profiler );
1604 
1605  /* Identify slot */
1606  if ( ( trb->slot > xhci->slots ) ||
1607  ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) {
1608  DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n",
1609  xhci->name, trb->slot );
1610  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1611  return;
1612  }
1613 
1614  /* Identify endpoint */
1615  if ( ( trb->endpoint >= XHCI_CTX_END ) ||
1616  ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) {
1617  DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid "
1618  "%d:\n", xhci->name, slot->id, trb->endpoint );
1619  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1620  return;
1621  }
1622 
1623  /* Dequeue TRB(s) */
1624  iobuf = xhci_dequeue_multi ( &endpoint->ring );
1625  assert ( iobuf != NULL );
1626 
1627  /* Unmap I/O buffer */
1628  iob_unmap ( iobuf );
1629 
1630  /* Check for errors */
1631  if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
1632  ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
1633 
1634  /* Construct error */
1635  rc = -ECODE ( trb->code );
1636  DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n",
1637  xhci->name, slot->id, endpoint->ctx, trb->code,
1638  strerror ( rc ) );
1639  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1640 
1641  /* Sanity check */
1642  assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
1643  != XHCI_ENDPOINT_RUNNING );
1644 
1645  /* Report failure to USB core */
1646  usb_complete_err ( endpoint->ep, iobuf, rc );
1647  return;
1648  }
1649 
1650  /* Record actual transfer size */
1651  iob_unput ( iobuf, le16_to_cpu ( trb->residual ) );
1652 
1653  /* Sanity check (for successful completions only) */
1654  assert ( xhci_ring_consumed ( &endpoint->ring ) ==
1655  le64_to_cpu ( trb->transfer ) );
1656 
1657  /* Report completion to USB core */
1658  usb_complete ( endpoint->ep, iobuf );
1659  profile_stop ( &xhci_transfer_profiler );
1660 }
1661 
1662 /**
1663  * Handle command completion event
1664  *
1665  * @v xhci xHCI device
1666  * @v trb Command completion event
1667  */
1668 static void xhci_complete ( struct xhci_device *xhci,
1669  struct xhci_trb_complete *trb ) {
1670  int rc;
1671 
1672  /* Ignore "command ring stopped" notifications */
1673  if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) {
1674  DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name );
1675  return;
1676  }
1677 
1678  /* Ignore unexpected completions */
1679  if ( ! xhci->pending ) {
1680  rc = -ECODE ( trb->code );
1681  DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n",
1682  xhci->name, trb->code, strerror ( rc ) );
1683  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1684  return;
1685  }
1686 
1687  /* Dequeue command TRB */
1688  xhci_dequeue ( &xhci->command );
1689 
1690  /* Sanity check */
1691  assert ( xhci_ring_consumed ( &xhci->command ) ==
1692  le64_to_cpu ( trb->command ) );
1693 
1694  /* Record completion */
1695  memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) );
1696  xhci->pending = NULL;
1697 }
1698 
1699 /**
1700  * Handle port status event
1701  *
1702  * @v xhci xHCI device
1703  * @v trb Port status event
1704  */
1705 static void xhci_port_status ( struct xhci_device *xhci,
1706  struct xhci_trb_port_status *trb ) {
1707  struct usb_port *port = usb_port ( xhci->bus->hub, trb->port );
1708  uint32_t portsc;
1709 
1710  /* Sanity check */
1711  assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) );
1712 
1713  /* Record disconnections and clear changes */
1714  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) );
1715  port->disconnected |= ( portsc & XHCI_PORTSC_CSC );
1716  portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
1717  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) );
1718 
1719  /* Report port status change */
1720  usb_port_changed ( port );
1721 }
1722 
1723 /**
1724  * Handle host controller event
1725  *
1726  * @v xhci xHCI device
1727  * @v trb Host controller event
1728  */
1729 static void xhci_host_controller ( struct xhci_device *xhci,
1730  struct xhci_trb_host_controller *trb ) {
1731  int rc;
1732 
1733  /* Construct error */
1734  rc = -ECODE ( trb->code );
1735  DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n",
1736  xhci->name, trb->code, strerror ( rc ) );
1737 }
1738 
1739 /**
1740  * Poll event ring
1741  *
1742  * @v xhci xHCI device
1743  */
1744 static void xhci_event_poll ( struct xhci_device *xhci ) {
1745  struct xhci_event_ring *event = &xhci->event;
1746  union xhci_trb *trb;
1747  unsigned int shift = XHCI_EVENT_TRBS_LOG2;
1748  unsigned int count = ( 1 << shift );
1749  unsigned int mask = ( count - 1 );
1750  unsigned int consumed;
1751  unsigned int type;
1752 
1753  /* Do nothing if device has permanently failed */
1754  if ( xhci->failed )
1755  return;
1756 
1757  /* Poll for events */
1758  profile_start ( &xhci_event_profiler );
1759  for ( consumed = 0 ; ; consumed++ ) {
1760 
1761  /* Stop if we reach an empty TRB */
1762  rmb();
1763  trb = &event->trb[ event->cons & mask ];
1764  if ( ! ( ( trb->common.flags ^
1765  ( event->cons >> shift ) ) & XHCI_TRB_C ) )
1766  break;
1767 
1768  /* Consume this TRB */
1769  event->cons++;
1770 
1771  /* Handle TRB */
1772  type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
1773  switch ( type ) {
1774 
1775  case XHCI_TRB_TRANSFER :
1776  xhci_transfer ( xhci, &trb->transfer );
1777  break;
1778 
1779  case XHCI_TRB_COMPLETE :
1780  xhci_complete ( xhci, &trb->complete );
1781  break;
1782 
1783  case XHCI_TRB_PORT_STATUS:
1784  xhci_port_status ( xhci, &trb->port );
1785  break;
1786 
1787  case XHCI_TRB_HOST_CONTROLLER:
1788  xhci_host_controller ( xhci, &trb->host );
1789  break;
1790 
1791  default:
1792  DBGC ( xhci, "XHCI %s unrecognised event %#x\n:",
1793  xhci->name, ( event->cons - 1 ) );
1794  DBGC_HDA ( xhci, virt_to_phys ( trb ),
1795  trb, sizeof ( *trb ) );
1796  break;
1797  }
1798  }
1799 
1800  /* Update dequeue pointer if applicable */
1801  if ( consumed ) {
1802  xhci_writeq ( xhci, dma ( &event->trb_map, trb ),
1803  xhci->run + XHCI_RUN_ERDP ( 0 ) );
1804  profile_stop ( &xhci_event_profiler );
1805  }
1806 }
1807 
1808 /**
1809  * Abort command
1810  *
1811  * @v xhci xHCI device
1812  */
1813 static void xhci_abort ( struct xhci_device *xhci ) {
1814  physaddr_t crp;
1815  uint32_t crcr;
1816 
1817  /* Abort the command */
1818  DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
1819  xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR );
1820 
1821  /* Allow time for command to abort */
1822  mdelay ( XHCI_COMMAND_ABORT_DELAY_MS );
1823 
1824  /* Check for failure to abort */
1825  crcr = readl ( xhci->op + XHCI_OP_CRCR );
1826  if ( crcr & XHCI_CRCR_CRR ) {
1827 
1828  /* Device has failed to abort a command and is almost
1829  * certainly beyond repair. Reset device, abandoning
1830  * all state, and mark device as failed to avoid
1831  * delays on any future command attempts.
1832  */
1833  DBGC ( xhci, "XHCI %s failed to abort command\n", xhci->name );
1834  xhci_fail ( xhci );
1835  }
1836 
1837  /* Consume (and ignore) any final command status */
1838  xhci_event_poll ( xhci );
1839 
1840  /* Reset the command ring control register */
1841  xhci_ring_reset ( &xhci->command );
1842  crp = dma ( &xhci->command.map, xhci->command.trb );
1843  xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
1844 }
1845 
1846 /**
1847  * Issue command and wait for completion
1848  *
1849  * @v xhci xHCI device
1850  * @v trb Transfer request block (with empty Cycle flag)
1851  * @ret rc Return status code
1852  *
1853  * On a successful completion, the TRB will be overwritten with the
1854  * completion.
1855  */
1856 static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
1857  struct xhci_trb_complete *complete = &trb->complete;
1858  unsigned int i;
1859  int rc;
1860 
1861  /* Immediately fail all commands if command mechanism has failed */
1862  if ( xhci->failed ) {
1863  rc = -EPIPE;
1864  goto err_failed;
1865  }
1866 
1867  /* Sanity check */
1868  if ( xhci->pending ) {
1869  DBGC ( xhci, "XHCI %s command ring busy\n", xhci->name );
1870  rc = -EBUSY;
1871  goto err_pending;
1872  }
1873 
1874  /* Record the pending command */
1875  xhci->pending = trb;
1876 
1877  /* Enqueue the command */
1878  if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
1879  goto err_enqueue;
1880 
1881  /* Ring the command doorbell */
1882  xhci_doorbell ( &xhci->command );
1883 
1884  /* Wait for the command to complete */
1885  for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
1886 
1887  /* Poll event ring */
1888  xhci_event_poll ( xhci );
1889 
1890  /* Check for completion */
1891  if ( ! xhci->pending ) {
1892  if ( complete->code != XHCI_CMPLT_SUCCESS ) {
1893  rc = -ECODE ( complete->code );
1894  DBGC ( xhci, "XHCI %s command failed (code "
1895  "%d): %s\n", xhci->name, complete->code,
1896  strerror ( rc ) );
1897  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1898  return rc;
1899  }
1900  return 0;
1901  }
1902 
1903  /* Delay */
1904  mdelay ( 1 );
1905  }
1906 
1907  /* Timeout */
1908  DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name );
1909  rc = -ETIMEDOUT;
1910 
1911  /* Abort command */
1912  xhci_abort ( xhci );
1913 
1914  err_enqueue:
1915  xhci->pending = NULL;
1916  err_pending:
1917  err_failed:
1918  return rc;
1919 }
1920 
1921 /**
1922  * Issue NOP and wait for completion
1923  *
1924  * @v xhci xHCI device
1925  * @ret rc Return status code
1926  */
1927 static inline int xhci_nop ( struct xhci_device *xhci ) {
1928  union xhci_trb trb;
1929  struct xhci_trb_common *nop = &trb.common;
1930  int rc;
1931 
1932  /* Construct command */
1933  memset ( nop, 0, sizeof ( *nop ) );
1934  nop->flags = XHCI_TRB_IOC;
1935  nop->type = XHCI_TRB_NOP_CMD;
1936 
1937  /* Issue command and wait for completion */
1938  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1939  DBGC ( xhci, "XHCI %s NOP failed: %s\n",
1940  xhci->name, strerror ( rc ) );
1941  return rc;
1942  }
1943 
1944  DBGC2 ( xhci, "XHCI %s NOP completed successfully\n", xhci->name );
1945  return 0;
1946 }
1947 
1948 /**
1949  * Enable slot
1950  *
1951  * @v xhci xHCI device
1952  * @v type Slot type
1953  * @ret slot Device slot ID, or negative error
1954  */
1955 static inline int xhci_enable_slot ( struct xhci_device *xhci,
1956  unsigned int type ) {
1957  union xhci_trb trb;
1958  struct xhci_trb_enable_slot *enable = &trb.enable;
1959  struct xhci_trb_complete *enabled = &trb.complete;
1960  unsigned int slot;
1961  int rc;
1962 
1963  /* Construct command */
1964  memset ( enable, 0, sizeof ( *enable ) );
1965  enable->slot = type;
1966  enable->type = XHCI_TRB_ENABLE_SLOT;
1967 
1968  /* Issue command and wait for completion */
1969  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1970  DBGC ( xhci, "XHCI %s could not enable new slot: %s\n",
1971  xhci->name, strerror ( rc ) );
1972  return rc;
1973  }
1974 
1975  /* Extract slot number */
1976  slot = enabled->slot;
1977 
1978  DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot );
1979  return slot;
1980 }
1981 
1982 /**
1983  * Disable slot
1984  *
1985  * @v xhci xHCI device
1986  * @v slot Device slot
1987  * @ret rc Return status code
1988  */
1989 static inline int xhci_disable_slot ( struct xhci_device *xhci,
1990  unsigned int slot ) {
1991  union xhci_trb trb;
1992  struct xhci_trb_disable_slot *disable = &trb.disable;
1993  int rc;
1994 
1995  /* Construct command */
1996  memset ( disable, 0, sizeof ( *disable ) );
1997  disable->type = XHCI_TRB_DISABLE_SLOT;
1998  disable->slot = slot;
1999 
2000  /* Issue command and wait for completion */
2001  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2002  DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n",
2003  xhci->name, slot, strerror ( rc ) );
2004  return rc;
2005  }
2006 
2007  DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot );
2008  return 0;
2009 }
2010 
2011 /**
2012  * Issue context-based command and wait for completion
2013  *
2014  * @v xhci xHCI device
2015  * @v slot Device slot
2016  * @v endpoint Endpoint
2017  * @v type TRB type
2018  * @v populate Input context populater
2019  * @ret rc Return status code
2020  */
2021 static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
2022  struct xhci_endpoint *endpoint, unsigned int type,
2023  void ( * populate ) ( struct xhci_device *xhci,
2024  struct xhci_slot *slot,
2025  struct xhci_endpoint *endpoint,
2026  void *input ) ) {
2027  union xhci_trb trb;
2028  struct xhci_trb_context *context = &trb.context;
2029  struct dma_mapping map;
2030  size_t len;
2031  void *input;
2032  int rc;
2033 
2034  /* Allocate an input context */
2035  memset ( &map, 0, sizeof ( map ) );
2036  len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
2037  input = dma_alloc ( xhci->dma, &map, len, xhci_align ( len ) );
2038  if ( ! input ) {
2039  rc = -ENOMEM;
2040  goto err_alloc;
2041  }
2042  memset ( input, 0, len );
2043 
2044  /* Populate input context */
2045  populate ( xhci, slot, endpoint, input );
2046 
2047  /* Construct command */
2048  memset ( context, 0, sizeof ( *context ) );
2049  context->type = type;
2050  context->input = cpu_to_le64 ( dma ( &map, input ) );
2051  context->slot = slot->id;
2052 
2053  /* Issue command and wait for completion */
2054  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
2055  goto err_command;
2056 
2057  err_command:
2058  dma_free ( &map, input, len );
2059  err_alloc:
2060  return rc;
2061 }
2062 
2063 /**
2064  * Populate address device input context
2065  *
2066  * @v xhci xHCI device
2067  * @v slot Device slot
2068  * @v endpoint Endpoint
2069  * @v input Input context
2070  */
2071 static void xhci_address_device_input ( struct xhci_device *xhci,
2072  struct xhci_slot *slot,
2073  struct xhci_endpoint *endpoint,
2074  void *input ) {
2075  struct xhci_trb_ring *ring = &endpoint->ring;
2076  struct xhci_control_context *control_ctx;
2077  struct xhci_slot_context *slot_ctx;
2078  struct xhci_endpoint_context *ep_ctx;
2079 
2080  /* Sanity checks */
2081  assert ( endpoint->ctx == XHCI_CTX_EP0 );
2082 
2083  /* Populate control context */
2084  control_ctx = input;
2085  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2086  ( 1 << XHCI_CTX_EP0 ) );
2087 
2088  /* Populate slot context */
2089  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2090  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
2091  slot->route ) );
2092  slot_ctx->port = slot->port;
2093  slot_ctx->tt_id = slot->tt_id;
2094  slot_ctx->tt_port = slot->tt_port;
2095 
2096  /* Populate control endpoint context */
2097  ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
2098  ep_ctx->type = XHCI_EP_TYPE_CONTROL;
2099  ep_ctx->burst = endpoint->ep->burst;
2100  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2101  ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
2102  XHCI_EP_DCS );
2103  ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
2104 }
2105 
2106 /**
2107  * Address device
2108  *
2109  * @v xhci xHCI device
2110  * @v slot Device slot
2111  * @ret rc Return status code
2112  */
2113 static inline int xhci_address_device ( struct xhci_device *xhci,
2114  struct xhci_slot *slot ) {
2115  struct usb_device *usb = slot->usb;
2116  struct xhci_slot_context *slot_ctx;
2117  int rc;
2118 
2119  /* Assign device address */
2120  if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
2121  XHCI_TRB_ADDRESS_DEVICE,
2122  xhci_address_device_input ) ) != 0 ) {
2123  DBGC ( xhci, "XHCI %s slot %d could not assign address: %s\n",
2124  xhci->name, slot->id, strerror ( rc ) );
2125  return rc;
2126  }
2127 
2128  /* Get assigned address */
2129  slot_ctx = ( slot->context +
2130  xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
2131  usb->address = slot_ctx->address;
2132  DBGC2 ( xhci, "XHCI %s slot %d assigned address %d to %s\n",
2133  xhci->name, slot->id, usb->address, usb->name );
2134 
2135  return 0;
2136 }
2137 
2138 /**
2139  * Populate configure endpoint input context
2140  *
2141  * @v xhci xHCI device
2142  * @v slot Device slot
2143  * @v endpoint Endpoint
2144  * @v input Input context
2145  */
2146 static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
2147  struct xhci_slot *slot,
2148  struct xhci_endpoint *endpoint,
2149  void *input ) {
2150  struct xhci_trb_ring *ring = &endpoint->ring;
2151  struct xhci_control_context *control_ctx;
2152  struct xhci_slot_context *slot_ctx;
2153  struct xhci_endpoint_context *ep_ctx;
2154 
2155  /* Populate control context */
2156  control_ctx = input;
2157  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2158  ( 1 << endpoint->ctx ) );
2159 
2160  /* Populate slot context */
2161  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2162  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2163  ( slot->ports ? 1 : 0 ),
2164  slot->psiv, 0 ) );
2165  slot_ctx->ports = slot->ports;
2166 
2167  /* Populate endpoint context */
2168  ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
2169  ep_ctx->interval = endpoint->interval;
2170  ep_ctx->type = endpoint->type;
2171  ep_ctx->burst = endpoint->ep->burst;
2172  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2173  ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
2174  XHCI_EP_DCS );
2175  ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
2176 }
2177 
2178 /**
2179  * Configure endpoint
2180  *
2181  * @v xhci xHCI device
2182  * @v slot Device slot
2183  * @v endpoint Endpoint
2184  * @ret rc Return status code
2185  */
2186 static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
2187  struct xhci_slot *slot,
2188  struct xhci_endpoint *endpoint ) {
2189  int rc;
2190 
2191  /* Configure endpoint */
2192  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2193  XHCI_TRB_CONFIGURE_ENDPOINT,
2194  xhci_configure_endpoint_input ) ) != 0 ) {
2195  DBGC ( xhci, "XHCI %s slot %d ctx %d could not configure: %s\n",
2196  xhci->name, slot->id, endpoint->ctx, strerror ( rc ) );
2197  return rc;
2198  }
2199 
2200  DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
2201  xhci->name, slot->id, endpoint->ctx );
2202  return 0;
2203 }
2204 
2205 /**
2206  * Populate deconfigure endpoint input context
2207  *
2208  * @v xhci xHCI device
2209  * @v slot Device slot
2210  * @v endpoint Endpoint
2211  * @v input Input context
2212  */
2213 static void
2214 xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
2215  struct xhci_slot *slot __unused,
2216  struct xhci_endpoint *endpoint,
2217  void *input ) {
2218  struct xhci_control_context *control_ctx;
2219  struct xhci_slot_context *slot_ctx;
2220 
2221  /* Populate control context */
2222  control_ctx = input;
2223  control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
2224  control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
2225 
2226  /* Populate slot context */
2227  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2228  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2229  0, 0, 0 ) );
2230 }
2231 
2232 /**
2233  * Deconfigure endpoint
2234  *
2235  * @v xhci xHCI device
2236  * @v slot Device slot
2237  * @v endpoint Endpoint
2238  * @ret rc Return status code
2239  */
2240 static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
2241  struct xhci_slot *slot,
2242  struct xhci_endpoint *endpoint ) {
2243  int rc;
2244 
2245  /* Deconfigure endpoint */
2246  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2247  XHCI_TRB_CONFIGURE_ENDPOINT,
2248  xhci_deconfigure_endpoint_input ) ) != 0 ) {
2249  DBGC ( xhci, "XHCI %s slot %d ctx %d could not deconfigure: "
2250  "%s\n", xhci->name, slot->id, endpoint->ctx,
2251  strerror ( rc ) );
2252  return rc;
2253  }
2254 
2255  DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
2256  xhci->name, slot->id, endpoint->ctx );
2257  return 0;
2258 }
2259 
2260 /**
2261  * Populate evaluate context input context
2262  *
2263  * @v xhci xHCI device
2264  * @v slot Device slot
2265  * @v endpoint Endpoint
2266  * @v input Input context
2267  */
2268 static void xhci_evaluate_context_input ( struct xhci_device *xhci,
2269  struct xhci_slot *slot __unused,
2270  struct xhci_endpoint *endpoint,
2271  void *input ) {
2272  struct xhci_control_context *control_ctx;
2273  struct xhci_slot_context *slot_ctx;
2274  struct xhci_endpoint_context *ep_ctx;
2275 
2276  /* Populate control context */
2277  control_ctx = input;
2278  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2279  ( 1 << endpoint->ctx ) );
2280 
2281  /* Populate slot context */
2282  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2283  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2284  0, 0, 0 ) );
2285 
2286  /* Populate endpoint context */
2287  ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
2288  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2289 }
2290 
2291 /**
2292  * Evaluate context
2293  *
2294  * @v xhci xHCI device
2295  * @v slot Device slot
2296  * @v endpoint Endpoint
2297  * @ret rc Return status code
2298  */
2299 static inline int xhci_evaluate_context ( struct xhci_device *xhci,
2300  struct xhci_slot *slot,
2301  struct xhci_endpoint *endpoint ) {
2302  int rc;
2303 
2304  /* Configure endpoint */
2305  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2306  XHCI_TRB_EVALUATE_CONTEXT,
2307  xhci_evaluate_context_input ) ) != 0 ) {
2308  DBGC ( xhci, "XHCI %s slot %d ctx %d could not (re-)evaluate: "
2309  "%s\n", xhci->name, slot->id, endpoint->ctx,
2310  strerror ( rc ) );
2311  return rc;
2312  }
2313 
2314  DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
2315  xhci->name, slot->id, endpoint->ctx );
2316  return 0;
2317 }
2318 
2319 /**
2320  * Reset endpoint
2321  *
2322  * @v xhci xHCI device
2323  * @v slot Device slot
2324  * @v endpoint Endpoint
2325  * @ret rc Return status code
2326  */
2327 static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
2328  struct xhci_slot *slot,
2329  struct xhci_endpoint *endpoint ) {
2330  union xhci_trb trb;
2331  struct xhci_trb_reset_endpoint *reset = &trb.reset;
2332  int rc;
2333 
2334  /* Construct command */
2335  memset ( reset, 0, sizeof ( *reset ) );
2336  reset->slot = slot->id;
2337  reset->endpoint = endpoint->ctx;
2338  reset->type = XHCI_TRB_RESET_ENDPOINT;
2339 
2340  /* Issue command and wait for completion */
2341  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2342  DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint "
2343  "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
2344  endpoint->context->state, strerror ( rc ) );
2345  return rc;
2346  }
2347 
2348  return 0;
2349 }
2350 
2351 /**
2352  * Stop endpoint
2353  *
2354  * @v xhci xHCI device
2355  * @v slot Device slot
2356  * @v endpoint Endpoint
2357  * @ret rc Return status code
2358  */
2359 static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
2360  struct xhci_slot *slot,
2361  struct xhci_endpoint *endpoint ) {
2362  union xhci_trb trb;
2363  struct xhci_trb_stop_endpoint *stop = &trb.stop;
2364  int rc;
2365 
2366  /* Construct command */
2367  memset ( stop, 0, sizeof ( *stop ) );
2368  stop->slot = slot->id;
2369  stop->endpoint = endpoint->ctx;
2370  stop->type = XHCI_TRB_STOP_ENDPOINT;
2371 
2372  /* Issue command and wait for completion */
2373  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2374  DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint "
2375  "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
2376  endpoint->context->state, strerror ( rc ) );
2377  return rc;
2378  }
2379 
2380  return 0;
2381 }
2382 
2383 /**
2384  * Set transfer ring dequeue pointer
2385  *
2386  * @v xhci xHCI device
2387  * @v slot Device slot
2388  * @v endpoint Endpoint
2389  * @ret rc Return status code
2390  */
2391 static inline int
2392 xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
2393  struct xhci_slot *slot,
2394  struct xhci_endpoint *endpoint ) {
2395  union xhci_trb trb;
2396  struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
2397  struct xhci_trb_ring *ring = &endpoint->ring;
2398  unsigned int cons;
2399  unsigned int mask;
2400  unsigned int index;
2401  unsigned int dcs;
2402  physaddr_t addr;
2403  int rc;
2404 
2405  /* Construct command */
2406  memset ( dequeue, 0, sizeof ( *dequeue ) );
2407  cons = ring->cons;
2408  mask = ring->mask;
2409  dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
2410  index = ( cons & mask );
2411  addr = dma ( &ring->map, &ring->trb[index] );
2412  dequeue->dequeue = cpu_to_le64 ( addr | dcs );
2413  dequeue->slot = slot->id;
2414  dequeue->endpoint = endpoint->ctx;
2415  dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
2416 
2417  /* Issue command and wait for completion */
2418  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2419  DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue "
2420  "pointer in state %d: %s\n", xhci->name, slot->id,
2421  endpoint->ctx, endpoint->context->state, strerror ( rc));
2422  return rc;
2423  }
2424 
2425  return 0;
2426 }
2427 
2428 /******************************************************************************
2429  *
2430  * Endpoint operations
2431  *
2432  ******************************************************************************
2433  */
2434 
2435 /**
2436  * Open endpoint
2437  *
2438  * @v ep USB endpoint
2439  * @ret rc Return status code
2440  */
2441 static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
2442  struct usb_device *usb = ep->usb;
2443  struct xhci_slot *slot = usb_get_hostdata ( usb );
2444  struct xhci_device *xhci = slot->xhci;
2445  struct xhci_endpoint *endpoint;
2446  unsigned int ctx;
2447  unsigned int type;
2448  unsigned int interval;
2449  int rc;
2450 
2451  /* Calculate context index */
2452  ctx = XHCI_CTX ( ep->address );
2453  assert ( slot->endpoint[ctx] == NULL );
2454 
2455  /* Calculate endpoint type */
2456  type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
2457  if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
2458  type = XHCI_EP_TYPE_CONTROL;
2459  if ( ep->address & USB_DIR_IN )
2460  type |= XHCI_EP_TYPE_IN;
2461 
2462  /* Calculate interval */
2463  if ( type & XHCI_EP_TYPE_PERIODIC ) {
2464  interval = ( fls ( ep->interval ) - 1 );
2465  } else {
2466  interval = ep->interval;
2467  }
2468 
2469  /* Allocate and initialise structure */
2470  endpoint = zalloc ( sizeof ( *endpoint ) );
2471  if ( ! endpoint ) {
2472  rc = -ENOMEM;
2473  goto err_alloc;
2474  }
2475  usb_endpoint_set_hostdata ( ep, endpoint );
2476  slot->endpoint[ctx] = endpoint;
2477  endpoint->xhci = xhci;
2478  endpoint->slot = slot;
2479  endpoint->ep = ep;
2480  endpoint->ctx = ctx;
2481  endpoint->type = type;
2482  endpoint->interval = interval;
2483  endpoint->context = ( ( ( void * ) slot->context ) +
2484  xhci_device_context_offset ( xhci, ctx ) );
2485 
2486  /* Allocate transfer ring */
2487  if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
2488  XHCI_TRANSFER_TRBS_LOG2,
2489  slot->id, ctx, 0 ) ) != 0 )
2490  goto err_ring_alloc;
2491 
2492  /* Configure endpoint, if applicable */
2493  if ( ( ctx != XHCI_CTX_EP0 ) &&
2494  ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
2495  goto err_configure_endpoint;
2496 
2497  DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n",
2498  xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
2499  ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
2500  return 0;
2501 
2502  xhci_deconfigure_endpoint ( xhci, slot, endpoint );
2503  err_configure_endpoint:
2504  xhci_ring_free ( &endpoint->ring );
2505  err_ring_alloc:
2506  slot->endpoint[ctx] = NULL;
2507  free ( endpoint );
2508  err_alloc:
2509  return rc;
2510 }
2511 
2512 /**
2513  * Close endpoint
2514  *
2515  * @v ep USB endpoint
2516  */
2517 static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
2518  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2519  struct xhci_slot *slot = endpoint->slot;
2520  struct xhci_device *xhci = slot->xhci;
2521  struct io_buffer *iobuf;
2522  unsigned int ctx = endpoint->ctx;
2523 
2524  /* Deconfigure endpoint, if applicable */
2525  if ( ctx != XHCI_CTX_EP0 )
2526  xhci_deconfigure_endpoint ( xhci, slot, endpoint );
2527 
2528  /* Cancel any incomplete transfers */
2529  while ( xhci_ring_fill ( &endpoint->ring ) ) {
2530  iobuf = xhci_dequeue_multi ( &endpoint->ring );
2531  iob_unmap ( iobuf );
2532  usb_complete_err ( ep, iobuf, -ECANCELED );
2533  }
2534 
2535  /* Free endpoint */
2536  xhci_ring_free ( &endpoint->ring );
2537  slot->endpoint[ctx] = NULL;
2538  free ( endpoint );
2539 }
2540 
2541 /**
2542  * Reset endpoint
2543  *
2544  * @v ep USB endpoint
2545  * @ret rc Return status code
2546  */
2547 static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
2548  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2549  struct xhci_slot *slot = endpoint->slot;
2550  struct xhci_device *xhci = slot->xhci;
2551  int rc;
2552 
2553  /* Reset endpoint context */
2554  if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
2555  return rc;
2556 
2557  /* Set transfer ring dequeue pointer */
2558  if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
2559  return rc;
2560 
2561  /* Ring doorbell to resume processing */
2562  xhci_doorbell ( &endpoint->ring );
2563 
2564  DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n",
2565  xhci->name, slot->id, endpoint->ctx );
2566  return 0;
2567 }
2568 
2569 /**
2570  * Update MTU
2571  *
2572  * @v ep USB endpoint
2573  * @ret rc Return status code
2574  */
2575 static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
2576  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2577  struct xhci_slot *slot = endpoint->slot;
2578  struct xhci_device *xhci = slot->xhci;
2579  int rc;
2580 
2581  /* Evalulate context */
2582  if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
2583  return rc;
2584 
2585  return 0;
2586 }
2587 
2588 /**
2589  * Enqueue message transfer
2590  *
2591  * @v ep USB endpoint
2592  * @v iobuf I/O buffer
2593  * @ret rc Return status code
2594  */
2595 static int xhci_endpoint_message ( struct usb_endpoint *ep,
2596  struct io_buffer *iobuf ) {
2597  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2598  struct xhci_device *xhci = endpoint->xhci;
2599  struct usb_setup_packet *packet;
2600  unsigned int input;
2601  size_t len;
2602  union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
2603  1 /* status */ ];
2604  union xhci_trb *trb = trbs;
2605  struct xhci_trb_setup *setup;
2606  struct xhci_trb_data *data;
2607  struct xhci_trb_status *status;
2608  int rc;
2609 
2610  /* Profile message transfers */
2611  profile_start ( &xhci_message_profiler );
2612 
2613  /* Construct setup stage TRB */
2614  memset ( trbs, 0, sizeof ( trbs ) );
2615  assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
2616  packet = iobuf->data;
2617  iob_pull ( iobuf, sizeof ( *packet ) );
2618  setup = &(trb++)->setup;
2619  memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
2620  setup->len = cpu_to_le32 ( sizeof ( *packet ) );
2621  setup->flags = XHCI_TRB_IDT;
2622  setup->type = XHCI_TRB_SETUP;
2623  len = iob_len ( iobuf );
2624  input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
2625  if ( len )
2626  setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
2627 
2628  /* Map I/O buffer */
2629  if ( ( rc = iob_map ( iobuf, xhci->dma, len,
2630  ( input ? DMA_RX : DMA_TX ) ) ) != 0 )
2631  goto err_map;
2632 
2633  /* Construct data stage TRB, if applicable */
2634  if ( len ) {
2635  data = &(trb++)->data;
2636  data->data = cpu_to_le64 ( iob_dma ( iobuf ) );
2637  data->len = cpu_to_le32 ( len );
2638  data->type = XHCI_TRB_DATA;
2639  data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
2640  }
2641 
2642  /* Construct status stage TRB */
2643  status = &(trb++)->status;
2644  status->flags = XHCI_TRB_IOC;
2645  status->type = XHCI_TRB_STATUS;
2646  status->direction =
2647  ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
2648 
2649  /* Enqueue TRBs */
2650  if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
2651  ( trb - trbs ) ) ) != 0 )
2652  goto err_enqueue;
2653 
2654  /* Ring the doorbell */
2655  xhci_doorbell ( &endpoint->ring );
2656 
2657  profile_stop ( &xhci_message_profiler );
2658  return 0;
2659 
2660  err_enqueue:
2661  iob_unmap ( iobuf );
2662  err_map:
2663  return rc;
2664 }
2665 
2666 /**
2667  * Calculate number of TRBs
2668  *
2669  * @v len Length of data
2670  * @v zlp Append a zero-length packet
2671  * @ret count Number of transfer descriptors
2672  */
2673 static unsigned int xhci_endpoint_count ( size_t len, int zlp ) {
2674  unsigned int count;
2675 
2676  /* Split into 64kB TRBs */
2677  count = ( ( len + XHCI_MTU - 1 ) / XHCI_MTU );
2678 
2679  /* Append a zero-length TRB if applicable */
2680  if ( zlp || ( count == 0 ) )
2681  count++;
2682 
2683  return count;
2684 }
2685 
2686 /**
2687  * Enqueue stream transfer
2688  *
2689  * @v ep USB endpoint
2690  * @v iobuf I/O buffer
2691  * @v zlp Append a zero-length packet
2692  * @ret rc Return status code
2693  */
2694 static int xhci_endpoint_stream ( struct usb_endpoint *ep,
2695  struct io_buffer *iobuf, int zlp ) {
2696  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2697  struct xhci_device *xhci = endpoint->xhci;
2698  size_t len = iob_len ( iobuf );
2699  unsigned int count = xhci_endpoint_count ( len, zlp );
2700  union xhci_trb trbs[count];
2701  union xhci_trb *trb = trbs;
2702  struct xhci_trb_normal *normal;
2703  physaddr_t data;
2704  unsigned int i;
2705  size_t trb_len;
2706  int rc;
2707 
2708  /* Profile stream transfers */
2709  profile_start ( &xhci_stream_profiler );
2710 
2711  /* Map I/O buffer */
2712  if ( ( rc = iob_map ( iobuf, xhci->dma, len,
2713  ( ( ep->address & USB_DIR_IN ) ?
2714  DMA_RX : DMA_TX ) ) ) != 0 )
2715  goto err_map;
2716  data = iob_dma ( iobuf );
2717 
2718  /* Construct normal TRBs */
2719  memset ( &trbs, 0, sizeof ( trbs ) );
2720  for ( i = 0 ; i < count ; i ++ ) {
2721 
2722  /* Calculate TRB length */
2723  trb_len = XHCI_MTU;
2724  if ( trb_len > len )
2725  trb_len = len;
2726 
2727  /* Construct normal TRB */
2728  normal = &trb->normal;
2729  normal->data = cpu_to_le64 ( data );
2730  normal->len = cpu_to_le32 ( trb_len );
2731  normal->type = XHCI_TRB_NORMAL;
2732  normal->flags = XHCI_TRB_CH;
2733 
2734  /* Move to next TRB */
2735  data += trb_len;
2736  len -= trb_len;
2737  trb++;
2738  }
2739 
2740  /* Mark zero-length packet (if present) as a separate transfer */
2741  if ( zlp && ( count > 1 ) )
2742  trb[-2].normal.flags = 0;
2743 
2744  /* Generate completion for final TRB */
2745  trb[-1].normal.flags = XHCI_TRB_IOC;
2746 
2747  /* Enqueue TRBs */
2748  if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
2749  count ) ) != 0 )
2750  goto err_enqueue;
2751 
2752  /* Ring the doorbell */
2753  xhci_doorbell ( &endpoint->ring );
2754 
2755  profile_stop ( &xhci_stream_profiler );
2756  return 0;
2757 
2758  err_enqueue:
2759  iob_unmap ( iobuf );
2760  err_map:
2761  return rc;
2762 }
2763 
2764 /******************************************************************************
2765  *
2766  * Device operations
2767  *
2768  ******************************************************************************
2769  */
2770 
2771 /**
2772  * Open device
2773  *
2774  * @v usb USB device
2775  * @ret rc Return status code
2776  */
2777 static int xhci_device_open ( struct usb_device *usb ) {
2778  struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
2779  struct usb_port *root_port = usb_root_hub_port ( usb );
2780  struct usb_port *tt = usb_transaction_translator ( usb );
2781  struct xhci_slot *slot;
2782  struct xhci_slot *tt_slot;
2783  size_t len;
2784  int type;
2785  int id;
2786  int rc;
2787 
2788  /* Determine applicable slot type */
2789  type = xhci_port_slot_type ( xhci, root_port->address );
2790  if ( type < 0 ) {
2791  rc = type;
2792  DBGC ( xhci, "XHCI %s-%d has no slot type\n",
2793  xhci->name, usb->port->address );
2794  goto err_type;
2795  }
2796 
2797  /* Allocate a device slot number */
2798  id = xhci_enable_slot ( xhci, type );
2799  if ( id < 0 ) {
2800  rc = id;
2801  goto err_enable_slot;
2802  }
2803  assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) );
2804  assert ( xhci->slot[id] == NULL );
2805 
2806  /* Allocate and initialise structure */
2807  slot = zalloc ( sizeof ( *slot ) );
2808  if ( ! slot ) {
2809  rc = -ENOMEM;
2810  goto err_alloc;
2811  }
2812  usb_set_hostdata ( usb, slot );
2813  xhci->slot[id] = slot;
2814  slot->xhci = xhci;
2815  slot->usb = usb;
2816  slot->id = id;
2817  if ( tt ) {
2818  tt_slot = usb_get_hostdata ( tt->hub->usb );
2819  slot->tt_id = tt_slot->id;
2820  slot->tt_port = tt->address;
2821  }
2822 
2823  /* Allocate a device context */
2824  len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
2825  slot->context = dma_alloc ( xhci->dma, &slot->map, len,
2826  xhci_align ( len ) );
2827  if ( ! slot->context ) {
2828  rc = -ENOMEM;
2829  goto err_alloc_context;
2830  }
2831  memset ( slot->context, 0, len );
2832 
2833  /* Set device context base address */
2834  assert ( xhci->dcbaa.context[id] == 0 );
2835  xhci->dcbaa.context[id] = cpu_to_le64 ( dma ( &slot->map,
2836  slot->context ) );
2837 
2838  DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
2839  xhci->name, slot->id, virt_to_phys ( slot->context ),
2840  ( virt_to_phys ( slot->context ) + len ), usb->name );
2841  return 0;
2842 
2843  xhci->dcbaa.context[id] = 0;
2844  dma_free ( &slot->map, slot->context, len );
2845  err_alloc_context:
2846  xhci->slot[id] = NULL;
2847  free ( slot );
2848  err_alloc:
2849  xhci_disable_slot ( xhci, id );
2850  err_enable_slot:
2851  err_type:
2852  return rc;
2853 }
2854 
2855 /**
2856  * Close device
2857  *
2858  * @v usb USB device
2859  */
2860 static void xhci_device_close ( struct usb_device *usb ) {
2861  struct xhci_slot *slot = usb_get_hostdata ( usb );
2862  struct xhci_device *xhci = slot->xhci;
2863  size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
2864  unsigned int id = slot->id;
2865  int rc;
2866 
2867  /* Disable slot */
2868  if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) {
2869  /* Slot is still enabled. Leak the slot context,
2870  * since the controller may still write to this
2871  * memory, and leave the DCBAA entry intact.
2872  *
2873  * If the controller later reports that this same slot
2874  * has been re-enabled, then some assertions will be
2875  * triggered.
2876  */
2877  DBGC ( xhci, "XHCI %s slot %d leaking context memory\n",
2878  xhci->name, slot->id );
2879  slot->context = NULL;
2880  }
2881 
2882  /* Free slot */
2883  if ( slot->context ) {
2884  dma_free ( &slot->map, slot->context, len );
2885  xhci->dcbaa.context[id] = 0;
2886  }
2887  xhci->slot[id] = NULL;
2888  free ( slot );
2889 }
2890 
2891 /**
2892  * Assign device address
2893  *
2894  * @v usb USB device
2895  * @ret rc Return status code
2896  */
2897 static int xhci_device_address ( struct usb_device *usb ) {
2898  struct xhci_slot *slot = usb_get_hostdata ( usb );
2899  struct xhci_device *xhci = slot->xhci;
2900  struct usb_port *root_port;
2901  int psiv;
2902  int rc;
2903 
2904  /* Calculate route string */
2905  slot->route = usb_route_string ( usb );
2906 
2907  /* Calculate root hub port number */
2908  root_port = usb_root_hub_port ( usb );
2909  slot->port = root_port->address;
2910 
2911  /* Calculate protocol speed ID */
2912  psiv = xhci_port_psiv ( xhci, slot->port, usb->speed );
2913  if ( psiv < 0 ) {
2914  rc = psiv;
2915  return rc;
2916  }
2917  slot->psiv = psiv;
2918 
2919  /* Address device */
2920  if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
2921  return rc;
2922 
2923  return 0;
2924 }
2925 
2926 /******************************************************************************
2927  *
2928  * Bus operations
2929  *
2930  ******************************************************************************
2931  */
2932 
2933 /**
2934  * Open USB bus
2935  *
2936  * @v bus USB bus
2937  * @ret rc Return status code
2938  */
2939 static int xhci_bus_open ( struct usb_bus *bus ) {
2940  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2941  int rc;
2942 
2943  /* Allocate device slot array */
2944  xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) );
2945  if ( ! xhci->slot ) {
2946  rc = -ENOMEM;
2947  goto err_slot_alloc;
2948  }
2949 
2950  /* Allocate device context base address array */
2951  if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
2952  goto err_dcbaa_alloc;
2953 
2954  /* Allocate scratchpad buffers */
2955  if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
2956  goto err_scratchpad_alloc;
2957 
2958  /* Allocate command ring */
2959  if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
2960  goto err_command_alloc;
2961 
2962  /* Allocate event ring */
2963  if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
2964  goto err_event_alloc;
2965 
2966  /* Start controller */
2967  xhci_run ( xhci );
2968 
2969  return 0;
2970 
2971  xhci_stop ( xhci );
2972  xhci_event_free ( xhci );
2973  err_event_alloc:
2974  xhci_command_free ( xhci );
2975  err_command_alloc:
2976  xhci_scratchpad_free ( xhci );
2977  err_scratchpad_alloc:
2978  xhci_dcbaa_free ( xhci );
2979  err_dcbaa_alloc:
2980  free ( xhci->slot );
2981  err_slot_alloc:
2982  return rc;
2983 }
2984 
2985 /**
2986  * Close USB bus
2987  *
2988  * @v bus USB bus
2989  */
2990 static void xhci_bus_close ( struct usb_bus *bus ) {
2991  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2992  unsigned int i;
2993 
2994  /* Sanity checks */
2995  assert ( xhci->slot != NULL );
2996  for ( i = 0 ; i <= xhci->slots ; i++ )
2997  assert ( xhci->slot[i] == NULL );
2998 
2999  xhci_stop ( xhci );
3000  xhci_event_free ( xhci );
3001  xhci_command_free ( xhci );
3002  xhci_scratchpad_free ( xhci );
3003  xhci_dcbaa_free ( xhci );
3004  free ( xhci->slot );
3005 }
3006 
3007 /**
3008  * Poll USB bus
3009  *
3010  * @v bus USB bus
3011  */
3012 static void xhci_bus_poll ( struct usb_bus *bus ) {
3013  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
3014 
3015  /* Poll event ring */
3016  xhci_event_poll ( xhci );
3017 }
3018 
3019 /******************************************************************************
3020  *
3021  * Hub operations
3022  *
3023  ******************************************************************************
3024  */
3025 
3026 /**
3027  * Open hub
3028  *
3029  * @v hub USB hub
3030  * @ret rc Return status code
3031  */
3032 static int xhci_hub_open ( struct usb_hub *hub ) {
3033  struct xhci_slot *slot;
3034 
3035  /* Do nothing if this is the root hub */
3036  if ( ! hub->usb )
3037  return 0;
3038 
3039  /* Get device slot */
3040  slot = usb_get_hostdata ( hub->usb );
3041 
3042  /* Update device slot hub parameters. We don't inform the
3043  * hardware of this information until the hub's interrupt
3044  * endpoint is opened, since the only mechanism for so doing
3045  * provided by the xHCI specification is a Configure Endpoint
3046  * command, and we can't issue that command until we have a
3047  * non-EP0 endpoint to configure.
3048  */
3049  slot->ports = hub->ports;
3050 
3051  return 0;
3052 }
3053 
3054 /**
3055  * Close hub
3056  *
3057  * @v hub USB hub
3058  */
3059 static void xhci_hub_close ( struct usb_hub *hub __unused ) {
3060 
3061  /* Nothing to do */
3062 }
3063 
3064 /******************************************************************************
3065  *
3066  * Root hub operations
3067  *
3068  ******************************************************************************
3069  */
3070 
3071 /**
3072  * Open root hub
3073  *
3074  * @v hub USB hub
3075  * @ret rc Return status code
3076  */
3077 static int xhci_root_open ( struct usb_hub *hub ) {
3078  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3079  struct usb_port *port;
3080  uint32_t portsc;
3081  unsigned int i;
3082 
3083  /* Enable power to all ports */
3084  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3085  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
3086  portsc &= XHCI_PORTSC_PRESERVE;
3087  portsc |= XHCI_PORTSC_PP;
3088  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
3089  }
3090 
3091  /* xHCI spec requires us to potentially wait 20ms after
3092  * enabling power to a port.
3093  */
3094  mdelay ( XHCI_PORT_POWER_DELAY_MS );
3095 
3096  /* USB3 ports may power up as Disabled */
3097  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3098  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
3099  port = usb_port ( hub, i );
3100  if ( ( port->protocol >= USB_PROTO_3_0 ) &&
3101  ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
3102  XHCI_PORTSC_PLS_DISABLED ) ) {
3103  /* Force link state to RxDetect */
3104  portsc &= XHCI_PORTSC_PRESERVE;
3105  portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
3106  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
3107  }
3108  }
3109 
3110  /* Some xHCI cards seem to require an additional delay after
3111  * setting the link state to RxDetect.
3112  */
3113  mdelay ( XHCI_LINK_STATE_DELAY_MS );
3114 
3115  return 0;
3116 }
3117 
3118 /**
3119  * Close root hub
3120  *
3121  * @v hub USB hub
3122  */
3123 static void xhci_root_close ( struct usb_hub *hub __unused ) {
3124 
3125  /* Nothing to do */
3126 }
3127 
3128 /**
3129  * Enable port
3130  *
3131  * @v hub USB hub
3132  * @v port USB port
3133  * @ret rc Return status code
3134  */
3135 static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
3136  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3137  uint32_t portsc;
3138  unsigned int i;
3139 
3140  /* Reset port */
3141  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3142  portsc &= XHCI_PORTSC_PRESERVE;
3143  portsc |= XHCI_PORTSC_PR;
3144  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3145 
3146  /* Wait for port to become enabled */
3147  for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
3148 
3149  /* Check port status */
3150  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3151  if ( portsc & XHCI_PORTSC_PED )
3152  return 0;
3153 
3154  /* Delay */
3155  mdelay ( 1 );
3156  }
3157 
3158  DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n",
3159  xhci->name, port->address );
3160  return -ETIMEDOUT;
3161 }
3162 
3163 /**
3164  * Disable port
3165  *
3166  * @v hub USB hub
3167  * @v port USB port
3168  * @ret rc Return status code
3169  */
3170 static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
3171  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3172  uint32_t portsc;
3173 
3174  /* Disable port */
3175  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3176  portsc &= XHCI_PORTSC_PRESERVE;
3177  portsc |= XHCI_PORTSC_PED;
3178  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3179 
3180  /* Allow time for link state to stabilise */
3181  mdelay ( XHCI_LINK_STATE_DELAY_MS );
3182 
3183  /* Set link state to RxDetect for USB3 ports */
3184  if ( port->protocol >= USB_PROTO_3_0 ) {
3185  portsc &= XHCI_PORTSC_PRESERVE;
3186  portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS );
3187  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3188  }
3189 
3190  /* Allow time for link state to stabilise */
3191  mdelay ( XHCI_LINK_STATE_DELAY_MS );
3192 
3193  return 0;
3194 }
3195 
3196 /**
3197  * Update root hub port speed
3198  *
3199  * @v hub USB hub
3200  * @v port USB port
3201  * @ret rc Return status code
3202  */
3203 static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
3204  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3205  uint32_t portsc;
3206  unsigned int psiv;
3207  int ccs;
3208  int ped;
3209  int csc;
3210  int speed;
3211  int rc;
3212 
3213  /* Read port status */
3214  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3215  DBGC2 ( xhci, "XHCI %s-%d status is %08x\n",
3216  xhci->name, port->address, portsc );
3217  ccs = ( portsc & XHCI_PORTSC_CCS );
3218  ped = ( portsc & XHCI_PORTSC_PED );
3219  csc = ( portsc & XHCI_PORTSC_CSC );
3220  psiv = XHCI_PORTSC_PSIV ( portsc );
3221 
3222  /* Record disconnections and clear changes */
3223  port->disconnected |= csc;
3224  portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
3225  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3226 
3227  /* Port speed is not valid unless port is connected */
3228  if ( ! ccs ) {
3229  port->speed = USB_SPEED_NONE;
3230  return 0;
3231  }
3232 
3233  /* For USB2 ports, the PSIV field is not valid until the port
3234  * completes reset and becomes enabled.
3235  */
3236  if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
3237  port->speed = USB_SPEED_FULL;
3238  return 0;
3239  }
3240 
3241  /* Get port speed and map to generic USB speed */
3242  speed = xhci_port_speed ( xhci, port->address, psiv );
3243  if ( speed < 0 ) {
3244  rc = speed;
3245  return rc;
3246  }
3247 
3248  port->speed = speed;
3249  return 0;
3250 }
3251 
3252 /**
3253  * Clear transaction translator buffer
3254  *
3255  * @v hub USB hub
3256  * @v port USB port
3257  * @v ep USB endpoint
3258  * @ret rc Return status code
3259  */
3260 static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
3261  struct usb_endpoint *ep ) {
3262  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3263 
3264  /* Should never be called; this is a root hub */
3265  DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name,
3266  port->address, ep->usb->name, usb_endpoint_name ( ep ) );
3267 
3268  return -ENOTSUP;
3269 }
3270 
3271 /******************************************************************************
3272  *
3273  * Hardware-independent interface
3274  *
3275  ******************************************************************************
3276  */
3277 
3278 /** USB host controller operations */
3279 static struct usb_host_operations xhci_operations = {
3280  .endpoint = {
3281  .open = xhci_endpoint_open,
3282  .close = xhci_endpoint_close,
3283  .reset = xhci_endpoint_reset,
3284  .mtu = xhci_endpoint_mtu,
3285  .message = xhci_endpoint_message,
3286  .stream = xhci_endpoint_stream,
3287  },
3288  .device = {
3289  .open = xhci_device_open,
3290  .close = xhci_device_close,
3291  .address = xhci_device_address,
3292  },
3293  .bus = {
3294  .open = xhci_bus_open,
3295  .close = xhci_bus_close,
3296  .poll = xhci_bus_poll,
3297  },
3298  .hub = {
3299  .open = xhci_hub_open,
3300  .close = xhci_hub_close,
3301  },
3302  .root = {
3303  .open = xhci_root_open,
3304  .close = xhci_root_close,
3305  .enable = xhci_root_enable,
3306  .disable = xhci_root_disable,
3307  .speed = xhci_root_speed,
3308  .clear_tt = xhci_root_clear_tt,
3309  },
3310 };
3311 
3312 /**
3313  * Register xHCI controller
3314  *
3315  * @v xhci xHCI device
3316  * @ret rc Return status code
3317  */
3318 int xhci_register ( struct xhci_device *xhci ) {
3319  struct usb_port *port;
3320  unsigned int i;
3321  int rc;
3322 
3323  /* Reset device */
3324  if ( ( rc = xhci_reset ( xhci ) ) != 0 )
3325  goto err_reset;
3326 
3327  /* Allocate USB bus */
3328  xhci->bus = alloc_usb_bus ( xhci->dev, xhci->ports, XHCI_MTU,
3329  &xhci_operations );
3330  if ( ! xhci->bus ) {
3331  rc = -ENOMEM;
3332  goto err_alloc_bus;
3333  }
3334  usb_bus_set_hostdata ( xhci->bus, xhci );
3335  usb_hub_set_drvdata ( xhci->bus->hub, xhci );
3336 
3337  /* Set port protocols */
3338  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3339  port = usb_port ( xhci->bus->hub, i );
3340  port->protocol = xhci_port_protocol ( xhci, i );
3341  }
3342 
3343  /* Register USB bus */
3344  if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
3345  goto err_register;
3346 
3347  return 0;
3348 
3349  unregister_usb_bus ( xhci->bus );
3350  err_register:
3351  free_usb_bus ( xhci->bus );
3352  err_alloc_bus:
3353  xhci_reset ( xhci );
3354  err_reset:
3355  return rc;
3356 }
3357 
3358 /**
3359  * Unregister xHCI controller
3360  *
3361  * @v xhci xHCI device
3362  */
3363 void xhci_unregister ( struct xhci_device *xhci ) {
3364  struct usb_bus *bus = xhci->bus;
3365 
3366  /* Unregister and free USB bus */
3367  unregister_usb_bus ( bus );
3368  free_usb_bus ( bus );
3369 
3370  /* Reset device */
3371  xhci_reset ( xhci );
3372 }
3373 
3374 /******************************************************************************
3375  *
3376  * PCI interface
3377  *
3378  ******************************************************************************
3379  */
3380 
3381 /**
3382  * Fix Intel PCH-specific quirks
3383  *
3384  * @v xhci xHCI device
3385  * @v pci PCI device
3386  */
3387 static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) {
3388  struct xhci_pch *pch = &xhci->pch;
3389  uint32_t xusb2pr;
3390  uint32_t xusb2prm;
3391  uint32_t usb3pssen;
3392  uint32_t usb3prm;
3393 
3394  /* Enable SuperSpeed capability. Do this before rerouting
3395  * USB2 ports, so that USB3 devices connect at SuperSpeed.
3396  */
3397  pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen );
3398  pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm );
3399  if ( usb3prm & ~usb3pssen ) {
3400  DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n",
3401  xhci->name, ( usb3prm & ~usb3pssen ) );
3402  }
3403  pch->usb3pssen = usb3pssen;
3404  usb3pssen |= usb3prm;
3405  pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen );
3406 
3407  /* Route USB2 ports from EHCI to xHCI */
3408  pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr );
3409  pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm );
3410  if ( xusb2prm & ~xusb2pr ) {
3411  DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n",
3412  xhci->name, ( xusb2prm & ~xusb2pr ) );
3413  }
3414  pch->xusb2pr = xusb2pr;
3415  xusb2pr |= xusb2prm;
3416  pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr );
3417 }
3418 
3419 /**
3420  * Undo Intel PCH-specific quirk fixes
3421  *
3422  * @v xhci xHCI device
3423  * @v pci PCI device
3424  */
3425 static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) {
3426  struct xhci_pch *pch = &xhci->pch;
3427 
3428  /* Restore USB2 port routing to original state */
3429  pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr );
3430 
3431  /* Restore SuperSpeed capability to original state */
3432  pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen );
3433 }
3434 
3435 /**
3436  * Probe PCI device
3437  *
3438  * @v pci PCI device
3439  * @ret rc Return status code
3440  */
3441 static int xhci_probe ( struct pci_device *pci ) {
3442  struct xhci_device *xhci;
3443  unsigned long bar_start;
3444  size_t bar_size;
3445  int rc;
3446 
3447  /* Allocate and initialise structure */
3448  xhci = zalloc ( sizeof ( *xhci ) );
3449  if ( ! xhci ) {
3450  rc = -ENOMEM;
3451  goto err_alloc;
3452  }
3453  xhci->dev = &pci->dev;
3454  xhci->dma = &pci->dma;
3455  xhci->quirks = pci->id->driver_data;
3456 
3457  /* Fix up PCI device */
3458  adjust_pci_device ( pci );
3459 
3460  /* Map registers */
3461  bar_start = pci_bar_start ( pci, XHCI_BAR );
3462  bar_size = pci_bar_size ( pci, XHCI_BAR );
3463  xhci->regs = pci_ioremap ( pci, bar_start, bar_size );
3464  if ( ! xhci->regs ) {
3465  rc = -ENODEV;
3466  goto err_ioremap;
3467  }
3468 
3469  /* Initialise xHCI device */
3470  xhci_init ( xhci );
3471 
3472  /* Initialise USB legacy support and claim ownership */
3473  xhci_legacy_init ( xhci );
3474  xhci_legacy_claim ( xhci );
3475 
3476  /* Fix Intel PCH-specific quirks, if applicable */
3477  if ( xhci->quirks & XHCI_PCH )
3478  xhci_pch_fix ( xhci, pci );
3479 
3480  /* Register xHCI device */
3481  if ( ( rc = xhci_register ( xhci ) ) != 0 )
3482  goto err_register;
3483 
3484  pci_set_drvdata ( pci, xhci );
3485  return 0;
3486 
3487  xhci_unregister ( xhci );
3488  err_register:
3489  if ( xhci->quirks & XHCI_PCH )
3490  xhci_pch_undo ( xhci, pci );
3491  xhci_legacy_release ( xhci );
3492  iounmap ( xhci->regs );
3493  err_ioremap:
3494  free ( xhci );
3495  err_alloc:
3496  return rc;
3497 }
3498 
3499 /**
3500  * Remove PCI device
3501  *
3502  * @v pci PCI device
3503  */
3504 static void xhci_remove ( struct pci_device *pci ) {
3505  struct xhci_device *xhci = pci_get_drvdata ( pci );
3506  uint16_t command;
3507 
3508  /* Some systems are observed to disable bus mastering on
3509  * Thunderbolt controllers before we get a chance to shut
3510  * down. Detect this and avoid attempting any DMA operations,
3511  * which are guaranteed to fail and may end up spuriously
3512  * completing after the operating system kernel starts up.
3513  */
3514  pci_read_config_word ( pci, PCI_COMMAND, &command );
3515  if ( ! ( command & PCI_COMMAND_MASTER ) ) {
3516  DBGC ( xhci, "XHCI %s DMA was disabled\n", xhci->name );
3517  xhci_fail ( xhci );
3518  }
3519 
3520  /* Unregister xHCI controller */
3521  xhci_unregister ( xhci );
3522 
3523  /* Undo any PCH-specific fixes */
3524  if ( xhci->quirks & XHCI_PCH )
3525  xhci_pch_undo ( xhci, pci );
3526 
3527  /* Release ownership back to BIOS */
3528  xhci_legacy_release ( xhci );
3529 
3530  /* Unmap registers */
3531  iounmap ( xhci->regs );
3532 
3533  /* Free device */
3534  free ( xhci );
3535 }
3536 
3537 /** XHCI PCI device IDs */
3538 static struct pci_device_id xhci_ids[] = {
3539  PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ),
3540  PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ),
3541  PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
3542 };
3543 
3544 /** XHCI PCI driver */
3545 struct pci_driver xhci_driver __pci_driver = {
3546  .ids = xhci_ids,
3547  .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
3548  .class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
3549  PCI_CLASS_SERIAL_USB_XHCI ),
3550  .probe = xhci_probe,
3551  .remove = xhci_remove,
3552 };
3553 
3554 /**
3555  * Prepare for exit
3556  *
3557  * @v booting System is shutting down for OS boot
3558  */
3559 static void xhci_shutdown ( int booting ) {
3560  /* If we are shutting down to boot an OS, then prevent the
3561  * release of ownership back to BIOS.
3562  */
3563  xhci_legacy_prevent_release = booting;
3564 }
3565 
3566 /** Startup/shutdown function */
3567 struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = {
3568  .name = "xhci",
3569  .shutdown = xhci_shutdown,
3570 };
XHCI_BAD_PSIV
#define XHCI_BAD_PSIV
Invalid protocol speed ID values quirk.
Definition: xhci.h:1041
xhci_endpoint_context
An endpoint context.
Definition: xhci.h:768
XHCI_CMPLT_SHORT
Short packet.
Definition: xhci.h:636
iob_pull
#define iob_pull(iobuf, len)
Definition: iobuf.h:106
xhci_endpoint_context::burst
uint8_t burst
Maximum burst size.
Definition: xhci.h:780
__attribute__
#define __attribute__(x)
Definition: compiler.h:10
xhci_event_ring
An event ring.
Definition: xhci.h:868
XHCI_USBLEGSUP_MAX_WAIT_MS
#define XHCI_USBLEGSUP_MAX_WAIT_MS
Maximum time to wait for BIOS to release ownership.
Definition: xhci.h:983
XHCI_SUPPORTED_PORTS
#define XHCI_SUPPORTED_PORTS
Supported protocol ports.
Definition: xhci.h:112
xhci_trb_disable_slot
A disable slot transfer request block.
Definition: xhci.h:484
xhci_root_speed
static int xhci_root_speed(struct usb_hub *hub, struct usb_port *port)
Update root hub port speed.
Definition: xhci.c:3203
XHCI_SUPPORTED_SLOT_TYPE
#define XHCI_SUPPORTED_SLOT_TYPE(slot)
Supported protocol slot type.
Definition: xhci.h:127
usb_endpoint_set_hostdata
static void usb_endpoint_set_hostdata(struct usb_endpoint *ep, void *priv)
Set USB endpoint host controller private data.
Definition: usb.h:575
xhci_command_free
static void xhci_command_free(struct xhci_device *xhci)
Free command ring.
Definition: xhci.c:1486
XHCI_TRB_TC
#define XHCI_TRB_TC
Transfer request block toggle cycle bit flag.
Definition: xhci.h:331
xhci_endpoint_context::type
uint8_t type
Endpoint type.
Definition: xhci.h:778
PCI_CLASS_ID
#define PCI_CLASS_ID(base, sub, progif)
Construct PCI class ID.
Definition: pci.h:202
rc
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
XHCI_STATUS_IN
#define XHCI_STATUS_IN
Input status direction.
Definition: xhci.h:439
name
const char * name
Definition: ath9k_hw.c:1984
uint16_t
unsigned short uint16_t
Definition: stdint.h:11
DMA_TX
#define DMA_TX
Device will read data from host memory.
Definition: dma.h:134
xhci_trb_status
A status stage transfer request block.
Definition: xhci.h:420
wmb
wmb()
xhci_trb::context
struct xhci_trb_context context
Input context TRB.
Definition: xhci.h:702
xhci_trb_complete::command
uint64_t command
Command TRB pointer.
Definition: xhci.h:613
xhci_trb_common
A transfer request block.
Definition: xhci.h:314
readb
uint8_t readb(volatile uint8_t *io_addr)
Read byte from memory-mapped device.
xhci_trb_ring::dbval
uint32_t dbval
Doorbell register value.
Definition: xhci.h:864
xhci_dequeue
static struct io_buffer * xhci_dequeue(struct xhci_trb_ring *ring)
Dequeue a transfer request block.
Definition: xhci.c:1358
pci_device::dma
struct dma_device dma
DMA device.
Definition: pci.h:214
xhci_trb::common
struct xhci_trb_common common
Common fields.
Definition: xhci.h:686
xhci_root_clear_tt
static int xhci_root_clear_tt(struct usb_hub *hub, struct usb_port *port, struct usb_endpoint *ep)
Clear transaction translator buffer.
Definition: xhci.c:3260
xhci_evaluate_context_input
static void xhci_evaluate_context_input(struct xhci_device *xhci, struct xhci_slot *slot __unused, struct xhci_endpoint *endpoint, void *input)
Populate evaluate context input context.
Definition: xhci.c:2268
__profiler
static struct profiler xhci_message_profiler __profiler
Message transfer profiler.
Definition: xhci.c:47
xhci_device::ports
unsigned int ports
Number of ports.
Definition: xhci.h:1092
XHCI_CONFIG_MAX_SLOTS_EN
#define XHCI_CONFIG_MAX_SLOTS_EN(slots)
Maximum device slots enabled.
Definition: xhci.h:196
xhci_scratchpad::buffer
void * buffer
Scratchpad buffer area.
Definition: xhci.h:1056
xhci_stop
static int xhci_stop(struct xhci_device *xhci)
Stop xHCI device.
Definition: xhci.c:1109
xhci_device::dcbaa
struct xhci_dcbaa dcbaa
Device context base address array.
Definition: xhci.h:1108
xhci_device_open
static int xhci_device_open(struct usb_device *usb)
Open device.
Definition: xhci.c:2777
pci_driver
A PCI driver.
Definition: pci.h:251
EBUSY
#define EBUSY
Device or resource busy.
Definition: errno.h:338
xhci_register
int xhci_register(struct xhci_device *xhci)
Register xHCI controller.
Definition: xhci.c:3318
normal
struct arbelprm_completion_queue_entry normal
Definition: arbel.h:11
XHCI_TRB_ENABLE_SLOT
#define XHCI_TRB_ENABLE_SLOT
An enable slot transfer request block.
Definition: xhci.h:481
reg
static unsigned int unsigned int reg
Definition: myson.h:162
xhci_ring_consumed
static physaddr_t xhci_ring_consumed(struct xhci_trb_ring *ring)
Calculate physical address of most recently consumed TRB.
Definition: xhci.h:935
XHCI_OP_USBSTS
#define XHCI_OP_USBSTS
USB status register.
Definition: xhci.h:163
XHCI_USBLEGSUP_BIOS_OWNED
#define XHCI_USBLEGSUP_BIOS_OWNED
USB legacy support BIOS ownership flag.
Definition: xhci.h:88
XHCI_PORTSC_PSIV
#define XHCI_PORTSC_PSIV(portsc)
Port speed ID value.
Definition: xhci.h:233
usb_hub
A USB hub.
Definition: usb.h:840
xhci_address_device
static int xhci_address_device(struct xhci_device *xhci, struct xhci_slot *slot)
Address device.
Definition: xhci.c:2113
XHCI_CTX_END
#define XHCI_CTX_END
End of contexts.
Definition: xhci.h:953
XHCI_ICI
#define XHCI_ICI(ctx)
Input context index.
Definition: xhci.h:959
xhci_hub_close
static void xhci_hub_close(struct usb_hub *hub __unused)
Close hub.
Definition: xhci.c:3059
XHCI_XECP_NEXT
#define XHCI_XECP_NEXT(xecp)
Next xHCI extended capability pointer.
Definition: xhci.h:79
xhci_extended_capability
static unsigned int xhci_extended_capability(struct xhci_device *xhci, unsigned int id, unsigned int offset)
Find extended capability.
Definition: xhci.c:329
xhci_slot::xhci
struct xhci_device * xhci
xHCI device
Definition: xhci.h:1135
XHCI_TRB_SET_TR_DEQUEUE_POINTER
#define XHCI_TRB_SET_TR_DEQUEUE_POINTER
A set transfer ring dequeue pointer transfer request block.
Definition: xhci.h:582
xhci_trb_ring::cons
unsigned int cons
Consumer counter.
Definition: xhci.h:843
xhci_device_context_offset
static size_t xhci_device_context_offset(struct xhci_device *xhci, unsigned int ctx)
Calculate device context offset.
Definition: xhci.c:418
xhci_device::run
void * run
Runtime registers.
Definition: xhci.h:1083
xhci_device
An xHCI device.
Definition: xhci.h:1066
xhci_trb_set_tr_dequeue_pointer
A set transfer ring dequeue pointer transfer request block.
Definition: xhci.h:566
alloc_usb_bus
struct usb_bus * alloc_usb_bus(struct device *dev, unsigned int ports, size_t mtu, struct usb_host_operations *op)
Allocate USB bus.
Definition: usb.c:2094
errno.h
Error codes.
USB_SPEED_LOW
Low speed (1.5Mbps)
Definition: usb.h:48
XHCI_PORT_RESET_MAX_WAIT_MS
#define XHCI_PORT_RESET_MAX_WAIT_MS
Maximum time to wait for a port reset to complete.
Definition: xhci.h:1015
USB_ENDPOINT_ATTR_TYPE_MASK
#define USB_ENDPOINT_ATTR_TYPE_MASK
Endpoint attribute transfer type mask.
Definition: usb.h:280
xhci_trb_ring::len
size_t len
Length of transfer request blocks.
Definition: xhci.h:855
XHCI_SUPPORTED_SLOT
#define XHCI_SUPPORTED_SLOT
Supported protocol slot.
Definition: xhci.h:124
xhci_device::regs
void * regs
Registers.
Definition: xhci.h:1068
xhci_control_context
An input control context.
Definition: xhci.h:720
XHCI_HCSPARAMS1_PORTS
#define XHCI_HCSPARAMS1_PORTS(params)
Number of ports.
Definition: xhci.h:48
command
A command-line command.
Definition: command.h:9
xhci_align
static size_t xhci_align(size_t len)
Calculate buffer alignment.
Definition: xhci.c:399
pci_device_id::driver_data
unsigned long driver_data
Arbitrary driver data.
Definition: pci.h:182
xhci_trb_enable_slot::slot
uint8_t slot
Slot type.
Definition: xhci.h:475
xhci_device::command
struct xhci_trb_ring command
Command ring.
Definition: xhci.h:1114
xhci_ring_fill
static unsigned int xhci_ring_fill(struct xhci_trb_ring *ring)
Calculate space used in TRB ring.
Definition: xhci.h:897
xhci_trb_complete
A command completion event transfer request block.
Definition: xhci.h:611
XHCI_TRB_DISABLE_SLOT
#define XHCI_TRB_DISABLE_SLOT
A disable slot transfer request block.
Definition: xhci.h:500
xhci_endpoint_context::state
uint8_t state
Endpoint state.
Definition: xhci.h:770
pci_driver::ids
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:253
usb_set_hostdata
static void usb_set_hostdata(struct usb_device *usb, void *priv)
Set USB device host controller private data.
Definition: usb.h:783
type
uint32_t type
Operating system type.
Definition: ena.h:12
XHCI_PCH_XUSB2PR
#define XHCI_PCH_XUSB2PR
Intel PCH USB2 port routing register.
Definition: xhci.h:1029
xhci_trb::transfer
struct xhci_trb_transfer transfer
Transfer event.
Definition: xhci.h:710
EPIPE
#define EPIPE
Broken pipe.
Definition: errno.h:619
xhci_trb_host_controller
A port status change transfer request block.
Definition: xhci.h:663
xhci_endpoint_context::trb_len
uint16_t trb_len
Average TRB length.
Definition: xhci.h:786
xhci_device::pending
union xhci_trb * pending
Current command (if any)
Definition: xhci.h:1118
XHCI_SUPPORTED_PSI_VALUE
#define XHCI_SUPPORTED_PSI_VALUE(psi)
Supported protocol PSI value.
Definition: xhci.h:133
DMA_RX
#define DMA_RX
Device will write data to host memory.
Definition: dma.h:137
XHCI_EP_TYPE
#define XHCI_EP_TYPE(type)
Endpoint type.
Definition: xhci.h:811
xhci_slot::endpoint
struct xhci_endpoint * endpoint[XHCI_CTX_END]
Endpoints, indexed by context ID.
Definition: xhci.h:1157
readl
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
xhci_device::dev
struct device * dev
Underlying hardware device.
Definition: xhci.h:1070
xhci_set_tr_dequeue_pointer
static int xhci_set_tr_dequeue_pointer(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Set transfer ring dequeue pointer.
Definition: xhci.c:2392
XHCI_XECP_ID_SUPPORTED
#define XHCI_XECP_ID_SUPPORTED
Supported protocol extended capability.
Definition: xhci.h:100
DBGC
#define DBGC(...)
Definition: compiler.h:505
xhci_trb_common::type
uint8_t type
Type.
Definition: xhci.h:322
xhci_trb_normal::flags
uint8_t flags
Flags.
Definition: xhci.h:355
xhci_trb_link::flags
uint8_t flags
Flags.
Definition: xhci.h:451
xhci_context
static int xhci_context(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, unsigned int type, void(*populate)(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input))
Issue context-based command and wait for completion.
Definition: xhci.c:2021
pci_bar_size
unsigned long pci_bar_size(struct pci_device *pci, unsigned int reg)
Get the size of a PCI BAR.
Definition: pci.c:163
xhci_trb_ring::map
struct dma_mapping map
DMA mapping.
Definition: xhci.h:857
__pci_driver
struct pci_driver xhci_driver __pci_driver
XHCI PCI driver.
Definition: xhci.c:3545
XHCI_EP_TYPE_CONTROL
#define XHCI_EP_TYPE_CONTROL
Control endpoint type.
Definition: xhci.h:814
xhci_trb_host_controller::code
uint8_t code
Completion code.
Definition: xhci.h:669
xhci_trb::disable
struct xhci_trb_disable_slot disable
Disable slot TRB.
Definition: xhci.h:700
device::name
char name[40]
Name.
Definition: device.h:78
XHCI_HCSPARAMS1_INTRS
#define XHCI_HCSPARAMS1_INTRS(params)
Number of interrupters.
Definition: xhci.h:45
input
Definition: bnxt_hsi.h:68
index
long index
Definition: bigint.h:62
ENOENT
#define ENOENT
No such file or directory.
Definition: errno.h:514
xhci_trb_enable_slot
An enable slot transfer request block.
Definition: xhci.h:465
usb_port::usb
struct usb_device * usb
Currently attached device (if in use)
Definition: usb.h:834
XHCI_SETUP_OUT
#define XHCI_SETUP_OUT
Setup stage output data direction.
Definition: xhci.h:392
xhci_deconfigure_endpoint_input
static void xhci_deconfigure_endpoint_input(struct xhci_device *xhci __unused, struct xhci_slot *slot __unused, struct xhci_endpoint *endpoint, void *input)
Populate deconfigure endpoint input context.
Definition: xhci.c:2214
XHCI_OP_PORTHLPMC
#define XHCI_OP_PORTHLPMC(port)
Port hardware link power management control register.
Definition: xhci.h:292
xhci_device::bus
struct usb_bus * bus
USB bus.
Definition: xhci.h:1126
xhci_trb::enable
struct xhci_trb_enable_slot enable
Enable slot TRB.
Definition: xhci.h:698
dma_set_mask_64bit
static __always_inline void dma_set_mask_64bit(struct dma_device *dma)
Set 64-bit addressable space mask.
Definition: dma.h:466
xhci_trb_disable_slot::slot
uint8_t slot
Slot ID.
Definition: xhci.h:496
xhci_ring_free
static void xhci_ring_free(struct xhci_trb_ring *ring)
Free transfer request block ring.
Definition: xhci.c:1289
xhci_scratchpad::buffer_map
struct dma_mapping buffer_map
Buffer DMA mapping.
Definition: xhci.h:1058
xhci_configure_endpoint
static int xhci_configure_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Configure endpoint.
Definition: xhci.c:2186
cpu_to_le64
#define cpu_to_le64(value)
Definition: byteswap.h:108
xhci_scratchpad::count
unsigned int count
Number of page-sized scratchpad buffers.
Definition: xhci.h:1054
pci_read_config_word
int pci_read_config_word(struct pci_device *pci, unsigned int where, uint16_t *value)
Read 16-bit word from PCI configuration space.
xhci_command_alloc
static int xhci_command_alloc(struct xhci_device *xhci)
Allocate command ring.
Definition: xhci.c:1455
xhci_trb_data
A data stage transfer request block.
Definition: xhci.h:395
xhci_trb_transfer::code
uint8_t code
Completion code.
Definition: xhci.h:596
xhci_pch
Intel PCH quirk.
Definition: xhci.h:1018
xhci_trb_ring::db
void * db
Doorbell register.
Definition: xhci.h:862
XHCI_TRB_RESET_ENDPOINT
#define XHCI_TRB_RESET_ENDPOINT
A reset endpoint transfer request block.
Definition: xhci.h:544
XHCI_SPEED_FULL
Full speed (12Mbps)
Definition: xhci.h:144
PCI_COMMAND
#define PCI_COMMAND
PCI command.
Definition: pci.h:25
XHCI_TRANSFER_TRBS_LOG2
#define XHCI_TRANSFER_TRBS_LOG2
Number of TRBs in a transfer ring.
Definition: xhci.h:977
XHCI_DATA_IN
#define XHCI_DATA_IN
Input data direction.
Definition: xhci.h:414
xhci_trb_common::flags
uint8_t flags
Flags.
Definition: xhci.h:320
XHCI_TRB_PORT_STATUS
#define XHCI_TRB_PORT_STATUS
A port status change transfer request block.
Definition: xhci.h:660
iob_map
static __always_inline int iob_map(struct io_buffer *iobuf, struct dma_device *dma, size_t len, int flags)
Map I/O buffer for DMA.
Definition: iobuf.h:230
XHCI_PORTSC_CSC
#define XHCI_PORTSC_CSC
Connect status change.
Definition: xhci.h:248
xhci_endpoint
An xHCI endpoint.
Definition: xhci.h:1161
xhci_trb_reset_endpoint::type
uint8_t type
Type.
Definition: xhci.h:536
xhci_trb_setup::direction
uint8_t direction
Transfer direction.
Definition: xhci.h:380
ctx
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
profiler
A data structure for storing profiling information.
Definition: profile.h:26
XHCI_CMPLT_SUCCESS
Success.
Definition: xhci.h:634
xhci_endpoint_stream
static int xhci_endpoint_stream(struct usb_endpoint *ep, struct io_buffer *iobuf, int zlp)
Enqueue stream transfer.
Definition: xhci.c:2694
USB_SPEED_SUPER
Super speed (5Gbps)
Definition: usb.h:54
XHCI_RESET_MAX_WAIT_MS
#define XHCI_RESET_MAX_WAIT_MS
Maximum time to wait for reset to complete.
Definition: xhci.h:995
profile_stop
static void profile_stop(struct profiler *profiler)
Stop profiling.
Definition: profile.h:173
XHCI_COMMAND_ABORT_DELAY_MS
#define XHCI_COMMAND_ABORT_DELAY_MS
Time to delay after aborting a command.
Definition: xhci.h:1009
XHCI_OP_PORTLI
#define XHCI_OP_PORTLI(port)
Port link info register.
Definition: xhci.h:289
XHCI_PORTSC_PRESERVE
#define XHCI_PORTSC_PRESERVE
Port status and control bits which should be preserved.
Definition: xhci.h:283
xhci_trb_transfer::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:602
adjust_pci_device
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:240
startup_fn::name
const char * name
Definition: init.h:43
xhci_trb_set_tr_dequeue_pointer::dequeue
uint64_t dequeue
Dequeue pointer.
Definition: xhci.h:568
pci_device::dev
struct device dev
Generic device.
Definition: pci.h:212
XHCI_TRB_STATUS
#define XHCI_TRB_STATUS
A status stage transfer request block.
Definition: xhci.h:436
xhci_ids
static struct pci_device_id xhci_ids[]
XHCI PCI device IDs.
Definition: xhci.c:3538
xhci_device::slots
unsigned int slots
Number of device slots.
Definition: xhci.h:1088
xhci_control_context::add
uint32_t add
Add context flags.
Definition: xhci.h:724
enabled
uint16_t enabled
Single-entry bitmask of the enabled option value.
Definition: ena.h:14
xhci_endpoint_context::dequeue
uint64_t dequeue
Transfer ring dequeue pointer.
Definition: xhci.h:784
xhci_device::scratch
struct xhci_scratchpad scratch
Scratchpad buffer.
Definition: xhci.h:1111
XHCI_USBLEGSUP_OS
#define XHCI_USBLEGSUP_OS
USB legacy support OS owned semaphore.
Definition: xhci.h:91
usb_device::address
unsigned int address
Device address, if assigned.
Definition: usb.h:732
xhci_ring_remaining
static unsigned int xhci_ring_remaining(struct xhci_trb_ring *ring)
Calculate space remaining in TRB ring.
Definition: xhci.h:912
PCI_CLASS_SERIAL_USB_XHCI
#define PCI_CLASS_SERIAL_USB_XHCI
xHCI USB controller
Definition: pci.h:140
xhci_scratchpad::array
uint64_t * array
Scratchpad array.
Definition: xhci.h:1060
xhci_endpoint_mtu
static int xhci_endpoint_mtu(struct usb_endpoint *ep)
Update MTU.
Definition: xhci.c:2575
writeb
void writeb(uint8_t data, volatile uint8_t *io_addr)
Write byte to memory-mapped device.
usb_device::speed
unsigned int speed
Device speed.
Definition: usb.h:728
xhci_slot_context::address
uint8_t address
USB address.
Definition: xhci.h:754
ECANCELED
#define ECANCELED
Operation canceled.
Definition: errno.h:343
XHCI_TRB_ADDRESS_DEVICE
#define XHCI_TRB_ADDRESS_DEVICE
An address device transfer request block.
Definition: xhci.h:519
ENOTSUP
#define ENOTSUP
Operation not supported.
Definition: errno.h:589
xhci_slot_context::ports
uint8_t ports
Number of downstream ports.
Definition: xhci.h:746
STARTUP_LATE
#define STARTUP_LATE
Late startup.
Definition: init.h:65
usb_port_changed
void usb_port_changed(struct usb_port *port)
Report port status change.
Definition: usb.c:1857
PCI_CLASS_SERIAL
#define PCI_CLASS_SERIAL
Definition: Pci22.h:266
ECODE
#define ECODE(code)
Definition: xhci.c:233
malloc.h
Dynamic memory allocation.
xhci_port_psiv
static int xhci_port_psiv(struct xhci_device *xhci, unsigned int port, unsigned int speed)
Find protocol speed ID value.
Definition: xhci.c:854
XHCI_DATA_OUT
#define XHCI_DATA_OUT
Output data direction.
Definition: xhci.h:417
xhci_transfer
static void xhci_transfer(struct xhci_device *xhci, struct xhci_trb_transfer *trb)
Handle transfer event.
Definition: xhci.c:1595
usb_bus_set_hostdata
static void usb_bus_set_hostdata(struct usb_bus *bus, void *priv)
Set USB bus host controller private data.
Definition: usb.h:1050
xhci_endpoint::type
unsigned int type
Endpoint type.
Definition: xhci.h:1171
xhci_trb::host
struct xhci_trb_host_controller host
Host controller event.
Definition: xhci.h:716
XHCI_EP_DCS
#define XHCI_EP_DCS
Endpoint dequeue cycle state.
Definition: xhci.h:823
xhci_trb_context::input
uint64_t input
Input context.
Definition: xhci.h:505
XHCI_EP_TYPE_IN
#define XHCI_EP_TYPE_IN
Input endpoint type.
Definition: xhci.h:817
xhci_speed_name
static const char * xhci_speed_name(uint32_t psi)
Transcribe port speed (for debugging)
Definition: xhci.c:648
xhci_root_enable
static int xhci_root_enable(struct usb_hub *hub, struct usb_port *port)
Enable port.
Definition: xhci.c:3135
XHCI_PORTSC_PP
#define XHCI_PORTSC_PP
Port power.
Definition: xhci.h:227
startup_fn
A startup/shutdown function.
Definition: init.h:42
PCI_COMMAND_MASTER
#define PCI_COMMAND_MASTER
Bus master.
Definition: pci.h:28
xhci_bus_poll
static void xhci_bus_poll(struct usb_bus *bus)
Poll USB bus.
Definition: xhci.c:3012
XHCI_PORTSC_CCS
#define XHCI_PORTSC_CCS
Current connect status.
Definition: xhci.h:206
xhci_ring_reset
static void xhci_ring_reset(struct xhci_trb_ring *ring)
Reset transfer request block ring.
Definition: xhci.c:1273
usb_port
A USB port.
Definition: usb.h:812
dma_free
void dma_free(struct dma_mapping *map, void *addr, size_t len)
Unmap and free DMA-coherent buffer.
pci_set_drvdata
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:365
rmb
#define rmb()
Definition: io.h:544
ENOMEM
#define ENOMEM
Not enough space.
Definition: errno.h:534
usb_endpoint
A USB endpoint.
Definition: usb.h:403
XHCI_COMMAND_MAX_WAIT_MS
#define XHCI_COMMAND_MAX_WAIT_MS
Maximum time to wait for a command to complete.
Definition: xhci.h:1003
usb_hub::usb
struct usb_device * usb
Underlying USB device, if any.
Definition: usb.h:846
xhci_scratchpad::array_map
struct dma_mapping array_map
Array DMA mapping.
Definition: xhci.h:1062
xhci_enable_slot
static int xhci_enable_slot(struct xhci_device *xhci, unsigned int type)
Enable slot.
Definition: xhci.c:1955
xhci_trb_stop_endpoint::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:557
XHCI_OP_CRCR
#define XHCI_OP_CRCR
Command ring control register.
Definition: xhci.h:178
xhci_trb_ring::trb
union xhci_trb * trb
Transfer request blocks.
Definition: xhci.h:853
memcpy
void * memcpy(void *dest, const void *src, size_t len) __nonnull
XHCI_TRB_COMPLETE
#define XHCI_TRB_COMPLETE
A command completion event transfer request block.
Definition: xhci.h:629
xhci_trb_transfer::slot
uint8_t slot
Slot ID.
Definition: xhci.h:604
xhci_init
void xhci_init(struct xhci_device *xhci)
Initialise device.
Definition: xhci.c:263
profiler::name
const char * name
Name.
Definition: profile.h:28
xhci_probe
static int xhci_probe(struct pci_device *pci)
Probe PCI device.
Definition: xhci.c:3441
xhci_device::pch
struct xhci_pch pch
Intel PCH quirk.
Definition: xhci.h:1129
port
u8 port
Port number.
Definition: CIB_PRM.h:31
XHCI_CAP_RTSOFF
#define XHCI_CAP_RTSOFF
Runtime register space offset.
Definition: xhci.h:73
xhci_trb_reset_endpoint::slot
uint8_t slot
Slot ID.
Definition: xhci.h:540
xhci_trb_reset_endpoint
A reset endpoint transfer request block.
Definition: xhci.h:528
xhci_port_slot_type
static int xhci_port_slot_type(struct xhci_device *xhci, unsigned int port)
Find port slot type.
Definition: xhci.c:766
unregister_usb_bus
void unregister_usb_bus(struct usb_bus *bus)
Unregister USB bus.
Definition: usb.c:2170
usb_device::port
struct usb_port * port
USB port.
Definition: usb.h:726
xhci_event_ring::trb
union xhci_trb * trb
Transfer request blocks.
Definition: xhci.h:876
xhci_trb_stop_endpoint::slot
uint8_t slot
Slot ID.
Definition: xhci.h:559
xhci_control_context::drop
uint32_t drop
Drop context flags.
Definition: xhci.h:722
assert
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
XHCI_EP_TYPE_PERIODIC
#define XHCI_EP_TYPE_PERIODIC
Periodic endpoint type.
Definition: xhci.h:820
USB_PROTO_3_0
USB 3.0.
Definition: usb.h:24
xhci_device_close
static void xhci_device_close(struct usb_device *usb)
Close device.
Definition: xhci.c:2860
pci_read_config_dword
int pci_read_config_dword(struct pci_device *pci, unsigned int where, uint32_t *value)
Read 32-bit dword from PCI configuration space.
XHCI_TRB_DATA
#define XHCI_TRB_DATA
A data stage transfer request block.
Definition: xhci.h:411
xhci_trb::stop
struct xhci_trb_stop_endpoint stop
Stop endpoint TRB.
Definition: xhci.h:706
XHCI_MTU
#define XHCI_MTU
Maximum transfer size.
Definition: xhci.h:27
USB_SPEED_NONE
Not connected.
Definition: usb.h:46
xhci_trb::template
struct xhci_trb_template template
Template.
Definition: xhci.h:684
xhci_dump_port
static void xhci_dump_port(struct xhci_device *xhci, unsigned int port)
Dump port registers.
Definition: xhci.c:489
xhci_endpoint::ctx
unsigned int ctx
Context index.
Definition: xhci.h:1169
xhci_trb_setup::len
uint32_t len
Length.
Definition: xhci.h:374
value
pseudo_bit_t value[0x00020]
Definition: arbel.h:13
DBGC_HDA
#define DBGC_HDA(...)
Definition: compiler.h:506
__unused
#define __unused
Declare a variable or data structure as unused.
Definition: compiler.h:573
xhci_slot::usb
struct usb_device * usb
USB device.
Definition: xhci.h:1137
xhci_device_address
static int xhci_device_address(struct usb_device *usb)
Assign device address.
Definition: xhci.c:2897
xhci_pch::xusb2pr
uint32_t xusb2pr
USB2 port routing register original value.
Definition: xhci.h:1020
iob_unmap
static __always_inline void iob_unmap(struct io_buffer *iobuf)
Unmap I/O buffer for DMA.
Definition: iobuf.h:278
XHCI_PORTSC_CHANGE
#define XHCI_PORTSC_CHANGE
Port status change mask.
Definition: xhci.h:269
len
ring len
Length.
Definition: dwmac.h:231
writel
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
xhci_slot_context::port
uint8_t port
Root hub port number.
Definition: xhci.h:744
XHCI_SLOT_INFO
#define XHCI_SLOT_INFO(entries, hub, speed, route)
Construct slot context device info.
Definition: xhci.h:764
XHCI_TRB_TRANSFER
#define XHCI_TRB_TRANSFER
A transfer event transfer request block.
Definition: xhci.h:608
XHCI_USBLEGSUP_CTLSTS
#define XHCI_USBLEGSUP_CTLSTS
USB legacy support control/status.
Definition: xhci.h:97
xhci_trb_setup
A setup stage transfer request block.
Definition: xhci.h:370
xhci_endpoint_context::mtu
uint16_t mtu
Maximum packet size.
Definition: xhci.h:782
xhci_trb_transfer
A transfer event transfer request block.
Definition: xhci.h:588
usb_endpoint::burst
unsigned int burst
Maximum burst size.
Definition: usb.h:413
xhci_bus_close
static void xhci_bus_close(struct usb_bus *bus)
Close USB bus.
Definition: xhci.c:2990
XHCI_PCH
#define XHCI_PCH
Intel PCH quirk flag.
Definition: xhci.h:1026
XHCI_TRB_NOP_CMD
#define XHCI_TRB_NOP_CMD
A no-op command transfer request block.
Definition: xhci.h:585
XHCI_TRB_SETUP
#define XHCI_TRB_SETUP
A setup stage transfer request block.
Definition: xhci.h:386
iob_dma
static __always_inline physaddr_t iob_dma(struct io_buffer *iobuf)
Get I/O buffer DMA address.
Definition: iobuf.h:267
xhci_root_open
static int xhci_root_open(struct usb_hub *hub)
Open root hub.
Definition: xhci.c:3077
link
u32 link
Link to next descriptor.
Definition: ar9003_mac.h:68
count
static unsigned int count
Number of entries.
Definition: dwmac.h:225
pci_bar_start
unsigned long pci_bar_start(struct pci_device *pci, unsigned int reg)
Find the start of a PCI BAR.
Definition: pci.c:96
xhci_trb_setup::type
uint8_t type
Type.
Definition: xhci.h:378
xhci_pch::usb3pssen
uint32_t usb3pssen
USB3 port SuperSpeed enable register original value.
Definition: xhci.h:1022
xhci_fail
static int xhci_fail(struct xhci_device *xhci)
Mark xHCI device as permanently failed.
Definition: xhci.c:1179
USB_DIR_IN
#define USB_DIR_IN
Data transfer is from device to host.
Definition: usb.h:97
profile_start
static void profile_start(struct profiler *profiler)
Start profiling.
Definition: profile.h:160
usb_device::name
char name[32]
Name.
Definition: usb.h:724
xhci_legacy_claim
static void xhci_legacy_claim(struct xhci_device *xhci)
Claim ownership from BIOS.
Definition: xhci.c:570
usb_hub::ports
unsigned int ports
Number of ports.
Definition: usb.h:850
xhci_event_free
static void xhci_event_free(struct xhci_device *xhci)
Free event ring.
Definition: xhci.c:1569
profile.h
Profiling.
USB_SPEED
#define USB_SPEED(mantissa, exponent)
Define a USB speed.
Definition: usb.h:35
xhci_slot_context::info
uint32_t info
Device info.
Definition: xhci.h:740
xhci_endpoint::slot
struct xhci_slot * slot
xHCI slot
Definition: xhci.h:1165
XHCI_SUPPORTED_PORTS_PSIC
#define XHCI_SUPPORTED_PORTS_PSIC(ports)
Supported protocol PSI count.
Definition: xhci.h:121
xhci_endpoint::context
struct xhci_endpoint_context * context
Endpoint context.
Definition: xhci.h:1175
cpu_to_le32
#define cpu_to_le32(value)
Definition: byteswap.h:107
id
uint8_t id
Request identifier.
Definition: ena.h:12
usb_device
A USB device.
Definition: usb.h:722
revision
uint32_t revision
Entry point revision.
Definition: ib_mad.h:20
xhci_trb_setup::flags
uint8_t flags
Flags.
Definition: xhci.h:376
XHCI_PORTSC_LWS
#define XHCI_PORTSC_LWS
Port link state write strobe.
Definition: xhci.h:242
XHCI_DCI
#define XHCI_DCI(ctx)
Device context index.
Definition: xhci.h:956
xhci_deconfigure_endpoint
static int xhci_deconfigure_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Deconfigure endpoint.
Definition: xhci.c:2240
xhci_slot
An xHCI device slot.
Definition: xhci.h:1133
XHCI_OP_DCBAAP
#define XHCI_OP_DCBAAP
Device context base address array pointer.
Definition: xhci.h:190
xhci_trb_context
A context transfer request block.
Definition: xhci.h:503
XHCI_TRB_STOP_ENDPOINT
#define XHCI_TRB_STOP_ENDPOINT
A stop endpoint transfer request block.
Definition: xhci.h:563
XHCI_TRB_HOST_CONTROLLER
#define XHCI_TRB_HOST_CONTROLLER
A port status change transfer request block.
Definition: xhci.h:679
iob_unput
#define iob_unput(iobuf, len)
Definition: iobuf.h:139
XHCI_SUPPORTED_PSI
#define XHCI_SUPPORTED_PSI(index)
Supported protocol PSI.
Definition: xhci.h:130
xhci_bus_open
static int xhci_bus_open(struct usb_bus *bus)
Open USB bus.
Definition: xhci.c:2939
strerror
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
xhci_enqueue_multi
static int xhci_enqueue_multi(struct xhci_trb_ring *ring, struct io_buffer *iobuf, const union xhci_trb *trbs, unsigned int count)
Enqueue multiple transfer request blocks.
Definition: xhci.c:1390
free
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
xhci_shutdown
static void xhci_shutdown(int booting)
Prepare for exit.
Definition: xhci.c:3559
xhci_root_disable
static int xhci_root_disable(struct usb_hub *hub, struct usb_port *port)
Disable port.
Definition: xhci.c:3170
xhci_trb_template::status
uint32_t status
Status.
Definition: xhci.h:308
zalloc
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:661
XHCI_XECP_ID
#define XHCI_XECP_ID(xecp)
xHCI extended capability ID
Definition: xhci.h:76
cons
uint16_t cons
Consumer index.
Definition: ena.h:22
xhci_device::failed
int failed
Command mechanism has permanently failed.
Definition: xhci.h:1120
PCI_CLASS_SERIAL_USB
#define PCI_CLASS_SERIAL_USB
Definition: Pci22.h:272
xhci_trb_link
A link transfer request block.
Definition: xhci.h:445
xhci_trb
A transfer request block.
Definition: xhci.h:682
xhci_dcbaa_free
static void xhci_dcbaa_free(struct xhci_device *xhci)
Free device context base address array.
Definition: xhci.c:958
usb_route_string
unsigned int usb_route_string(struct usb_device *usb)
Get USB route string.
Definition: usb.c:2335
dma_umalloc
void * dma_umalloc(struct dma_device *dma, struct dma_mapping *map, size_t len, size_t align)
Allocate and map DMA-coherent buffer from external (user) memory.
xhci_device::name
const char * name
Name.
Definition: xhci.h:1074
pci.h
PCI bus.
pci_device
A PCI device.
Definition: pci.h:210
xhci_trb_set_tr_dequeue_pointer::slot
uint8_t slot
Slot ID.
Definition: xhci.h:578
iob_len
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:159
xhci_root_close
static void xhci_root_close(struct usb_hub *hub __unused)
Close root hub.
Definition: xhci.c:3123
xhci_slot::id
unsigned int id
Slot ID.
Definition: xhci.h:1139
addr
uint32_t addr
Buffer address.
Definition: dwmac.h:20
XHCI_USBLEGSUP_BIOS
#define XHCI_USBLEGSUP_BIOS
USB legacy support BIOS owned semaphore.
Definition: xhci.h:85
XHCI_USBLEGSUP_OS_OWNED
#define XHCI_USBLEGSUP_OS_OWNED
USB legacy support OS ownership flag.
Definition: xhci.h:94
XHCI_OP_PORTPMSC
#define XHCI_OP_PORTPMSC(port)
Port power management status and control register.
Definition: xhci.h:286
xhci_dcbaa::context
uint64_t * context
Context base addresses.
Definition: xhci.h:1046
ENODEV
#define ENODEV
No such device.
Definition: errno.h:509
XHCI_HCCPARAMS1_XECP
#define XHCI_HCCPARAMS1_XECP(params)
xHCI extended capabilities pointer
Definition: xhci.h:67
xhci_scratchpad_alloc
static int xhci_scratchpad_alloc(struct xhci_device *xhci)
Allocate scratchpad buffers.
Definition: xhci.c:987
XHCI_BAR
#define XHCI_BAR
xHCI PCI BAR
Definition: xhci.h:30
XHCI_HCCPARAMS1_ADDR64
#define XHCI_HCCPARAMS1_ADDR64(params)
64-bit addressing capability
Definition: xhci.h:61
XHCI_CAP_CAPLENGTH
#define XHCI_CAP_CAPLENGTH
Capability register length.
Definition: xhci.h:33
xhci_dump
static void xhci_dump(struct xhci_device *xhci)
Dump host controller registers.
Definition: xhci.c:448
uint8_t
unsigned char uint8_t
Definition: stdint.h:10
nop
uint8_t nop
Definition: tcp.h:13
slot
uint8_t slot
Slot.
Definition: edd.h:16
xhci_endpoint_open
static int xhci_endpoint_open(struct usb_endpoint *ep)
Open endpoint.
Definition: xhci.c:2441
xhci_remove
static void xhci_remove(struct pci_device *pci)
Remove PCI device.
Definition: xhci.c:3504
xhci_device::intrs
unsigned int intrs
Number of interrupters.
Definition: xhci.h:1090
xhci_writeq
static int xhci_writeq(struct xhci_device *xhci, physaddr_t value, void *reg)
Write potentially 64-bit register.
Definition: xhci.c:365
usb_get_hostdata
static void * usb_get_hostdata(struct usb_device *usb)
Get USB device host controller private data.
Definition: usb.h:794
xhci_trb_ring::prod
unsigned int prod
Producer counter.
Definition: xhci.h:841
xhci_trb_disable_slot::type
uint8_t type
Type.
Definition: xhci.h:492
XHCI_EP0_TRB_LEN
#define XHCI_EP0_TRB_LEN
Control endpoint average TRB length.
Definition: xhci.h:826
xhci_trb::complete
struct xhci_trb_complete complete
Command completion event.
Definition: xhci.h:712
xhci_trb::dequeue
struct xhci_trb_set_tr_dequeue_pointer dequeue
Set transfer ring dequeue pointer TRB.
Definition: xhci.h:708
XHCI_SUPPORTED_PORTS_COUNT
#define XHCI_SUPPORTED_PORTS_COUNT(ports)
Supported protocol port count.
Definition: xhci.h:118
pci_device_id
A PCI device ID list entry.
Definition: pci.h:174
command
static int command
Definition: epic100.c:68
dma_ufree
void dma_ufree(struct dma_mapping *map, void *addr, size_t len)
Unmap and free DMA-coherent buffer from external (user) memory.
xhci_endpoint_message
static int xhci_endpoint_message(struct usb_endpoint *ep, struct io_buffer *iobuf)
Enqueue message transfer.
Definition: xhci.c:2595
le16_to_cpu
#define le16_to_cpu(value)
Definition: byteswap.h:112
XHCI_HCSPARAMS1_SLOTS
#define XHCI_HCSPARAMS1_SLOTS(params)
Number of device slots.
Definition: xhci.h:42
uint32_t
unsigned int uint32_t
Definition: stdint.h:12
dma_alloc
void * dma_alloc(struct dma_device *dma, struct dma_mapping *map, size_t len, size_t align)
Allocate and map DMA-coherent buffer.
next
uint32_t next
Next descriptor address.
Definition: dwmac.h:22
usb_setup_packet
A USB setup data packet.
Definition: usb.h:82
usb_endpoint_name
const char * usb_endpoint_name(struct usb_endpoint *ep)
Get USB endpoint name (for debugging)
Definition: usb.c:220
xhci_scratchpad
Scratchpad buffer.
Definition: xhci.h:1052
xhci_device::cap
void * cap
Capability registers.
Definition: xhci.h:1079
usb_root_hub_port
struct usb_port * usb_root_hub_port(struct usb_device *usb)
Get USB root hub port.
Definition: usb.c:2355
xhci_device::xecp
unsigned int xecp
xHCI extended capabilities offset
Definition: xhci.h:1099
xhci_abort
static void xhci_abort(struct xhci_device *xhci)
Abort command.
Definition: xhci.c:1813
XHCI_OP_PAGESIZE
#define XHCI_OP_PAGESIZE
Page size register.
Definition: xhci.h:169
XHCI_RUN_ERSTBA
#define XHCI_RUN_ERSTBA(intr)
Event ring segment table base address register.
Definition: xhci.h:298
XHCI_SUPPORTED_REVISION
#define XHCI_SUPPORTED_REVISION
Supported protocol revision.
Definition: xhci.h:103
xhci_trb_normal
A normal transfer request block.
Definition: xhci.h:349
XHCI_TRB_IDT
#define XHCI_TRB_IDT
Transfer request block immediate data flag.
Definition: xhci.h:340
XHCI_XECP_ID_LEGACY
#define XHCI_XECP_ID_LEGACY
USB legacy support extended capability.
Definition: xhci.h:82
xhci_device::addr64
int addr64
64-bit addressing capability
Definition: xhci.h:1095
usb_endpoint::mtu
size_t mtu
Maximum transfer size.
Definition: usb.h:411
XHCI_USBCMD_HCRST
#define XHCI_USBCMD_HCRST
Host controller reset.
Definition: xhci.h:160
status
uint8_t status
Status.
Definition: ena.h:16
xhci_trb_port_status
A port status change transfer request block.
Definition: xhci.h:642
usb_setup_packet::request
uint16_t request
Request.
Definition: usb.h:84
xhci_trb_port_status::port
uint8_t port
Port ID.
Definition: xhci.h:646
xhci_trb::normal
struct xhci_trb_normal normal
Normal TRB.
Definition: xhci.h:688
__startup_fn
struct startup_fn xhci_startup __startup_fn(STARTUP_LATE)
Startup/shutdown function.
xhci_configure_endpoint_input
static void xhci_configure_endpoint_input(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input)
Populate configure endpoint input context.
Definition: xhci.c:2146
xhci_reset
static int xhci_reset(struct xhci_device *xhci)
Reset xHCI device.
Definition: xhci.c:1141
usb_transaction_translator
struct usb_port * usb_transaction_translator(struct usb_device *usb)
Get USB transaction translator.
Definition: usb.c:2371
physaddr_t
unsigned long physaddr_t
Definition: stdint.h:20
xhci_run
static void xhci_run(struct xhci_device *xhci)
Start xHCI device.
Definition: xhci.c:1087
pci_get_drvdata
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:375
XHCI_ENDPOINT_RUNNING
Endpoint is running.
Definition: xhci.h:798
XHCI_DBVAL
#define XHCI_DBVAL(target, stream)
Calculate doorbell register value.
Definition: xhci.h:888
xhci_dequeue_multi
static struct io_buffer * xhci_dequeue_multi(struct xhci_trb_ring *ring)
Dequeue multiple transfer request blocks.
Definition: xhci.c:1419
XHCI_PAGESIZE
#define XHCI_PAGESIZE(pagesize)
Page size.
Definition: xhci.h:172
mdelay
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
xhci_trb_stop_endpoint::type
uint8_t type
Type.
Definition: xhci.h:555
XHCI_PORTSC_PLS_RXDETECT
#define XHCI_PORTSC_PLS_RXDETECT
RxDetect port link state.
Definition: xhci.h:221
xhci_trb::reset
struct xhci_trb_reset_endpoint reset
Reset endpoint TRB.
Definition: xhci.h:704
XHCI_TRB_CONFIGURE_ENDPOINT
#define XHCI_TRB_CONFIGURE_ENDPOINT
A configure endpoint transfer request block.
Definition: xhci.h:522
XHCI_SUPPORTED_PORTS_OFFSET
#define XHCI_SUPPORTED_PORTS_OFFSET(ports)
Supported protocol port offset.
Definition: xhci.h:115
XHCI_CMPLT_CMD_STOPPED
Command ring stopped.
Definition: xhci.h:638
xhci_supported_protocol
static unsigned int xhci_supported_protocol(struct xhci_device *xhci, unsigned int port)
Find supported protocol extended capability for a port.
Definition: xhci.c:671
XHCI_OP_CONFIG
#define XHCI_OP_CONFIG
Configure register.
Definition: xhci.h:193
xhci.h
USB eXtensible Host Controller Interface (xHCI) driver.
FILE_LICENCE
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
XHCI_PCH_USB3PSSEN
#define XHCI_PCH_USB3PSSEN
Intel PCH SuperSpeed enable register.
Definition: xhci.h:1035
XHCI_OP_PORTSC
#define XHCI_OP_PORTSC(port)
Port status and control register.
Definition: xhci.h:203
usb_bus_get_hostdata
static void * usb_bus_get_hostdata(struct usb_bus *bus)
Get USB bus host controller private data.
Definition: usb.h:1061
xhci_disable_slot
static int xhci_disable_slot(struct xhci_device *xhci, unsigned int slot)
Disable slot.
Definition: xhci.c:1989
xhci_slot_context::tt_id
uint8_t tt_id
TT hub slot ID.
Definition: xhci.h:748
xhci_pch_fix
static void xhci_pch_fix(struct xhci_device *xhci, struct pci_device *pci)
Fix Intel PCH-specific quirks.
Definition: xhci.c:3387
xhci_endpoint::ep
struct usb_endpoint * ep
USB endpoint.
Definition: xhci.h:1167
xhci_trb_template::control
uint32_t control
Control.
Definition: xhci.h:310
map
static __always_inline int struct dma_mapping * map
Definition: dma.h:183
XHCI_CTX
#define XHCI_CTX(address)
Calculate context index from USB endpoint address.
Definition: xhci.h:945
XHCI_STOP_MAX_WAIT_MS
#define XHCI_STOP_MAX_WAIT_MS
Maximum time to wait for host controller to stop.
Definition: xhci.h:989
XHCI_TRB_NORMAL
#define XHCI_TRB_NORMAL
A normal transfer request block.
Definition: xhci.h:363
ENOBUFS
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
DBGC2
#define DBGC2(...)
Definition: compiler.h:522
xhci_device::pagesize
size_t pagesize
Page size.
Definition: xhci.h:1102
usb.h
Universal Serial Bus (USB)
XHCI_PORT_POWER_DELAY_MS
#define XHCI_PORT_POWER_DELAY_MS
Time to delay after enabling power to a port.
Definition: xhci.h:230
register_usb_bus
int register_usb_bus(struct usb_bus *bus)
Register USB bus.
Definition: usb.c:2130
xhci_event_poll
static void xhci_event_poll(struct xhci_device *xhci)
Poll event ring.
Definition: xhci.c:1744
XHCI_CRCR_RCS
#define XHCI_CRCR_RCS
Command ring cycle state.
Definition: xhci.h:181
xhci_trb_stop_endpoint
A stop endpoint transfer request block.
Definition: xhci.h:547
xhci_trb_ring::mask
unsigned int mask
Ring counter mask.
Definition: xhci.h:847
xhci_scratchpad_free
static void xhci_scratchpad_free(struct xhci_device *xhci)
Free scratchpad buffers.
Definition: xhci.c:1053
XHCI_TRB_TYPE_MASK
#define XHCI_TRB_TYPE_MASK
Transfer request block type mask.
Definition: xhci.h:346
usb_complete
static void usb_complete(struct usb_endpoint *ep, struct io_buffer *iobuf)
Complete transfer (without error)
Definition: usb.h:1086
xhci_endpoint_context::interval
uint8_t interval
Polling interval.
Definition: xhci.h:774
xhci_slot_context::tt_port
uint8_t tt_port
TT port number.
Definition: xhci.h:750
usb_port::address
unsigned int address
Port address.
Definition: usb.h:816
io_buffer::data
void * data
Start of data.
Definition: iobuf.h:52
xhci_host_controller
static void xhci_host_controller(struct xhci_device *xhci, struct xhci_trb_host_controller *trb)
Handle host controller event.
Definition: xhci.c:1729
usb_port::hub
struct usb_hub * hub
USB hub.
Definition: usb.h:814
xhci_dcbaa::map
struct dma_mapping map
DMA mapping.
Definition: xhci.h:1048
xhci_trb_ring::iobuf
struct io_buffer ** iobuf
I/O buffers.
Definition: xhci.h:850
XHCI_TRB_EVALUATE_CONTEXT
#define XHCI_TRB_EVALUATE_CONTEXT
An evaluate context transfer request block.
Definition: xhci.h:525
xhci_complete
static void xhci_complete(struct xhci_device *xhci, struct xhci_trb_complete *trb)
Handle command completion event.
Definition: xhci.c:1668
XHCI_CAP_DBOFF
#define XHCI_CAP_DBOFF
Doorbell offset.
Definition: xhci.h:70
USB_SPEED_HIGH
High speed (480Mbps)
Definition: usb.h:52
usb_device::ep
struct usb_endpoint * ep[32]
Endpoint list.
Definition: usb.h:744
xhci_port_speed
static int xhci_port_speed(struct xhci_device *xhci, unsigned int port, unsigned int psiv)
Find port speed.
Definition: xhci.c:790
xhci_endpoint::ring
struct xhci_trb_ring ring
Transfer ring.
Definition: xhci.h:1177
usb_endpoint::interval
unsigned int interval
Interval (in microframes)
Definition: usb.h:415
pci_device::id
struct pci_device_id * id
Driver device ID.
Definition: pci.h:247
xhci_stop_endpoint
static int xhci_stop_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Stop endpoint.
Definition: xhci.c:2359
xhci_input_context_offset
static size_t xhci_input_context_offset(struct xhci_device *xhci, unsigned int ctx)
Calculate input context offset.
Definition: xhci.c:430
XHCI_HCCPARAMS1_CSZ_SHIFT
#define XHCI_HCCPARAMS1_CSZ_SHIFT(params)
Context size shift.
Definition: xhci.h:64
XHCI_CRCR_CA
#define XHCI_CRCR_CA
Command abort.
Definition: xhci.h:184
usb_host_operations::endpoint
struct usb_endpoint_host_operations endpoint
Endpoint operations.
Definition: usb.h:1032
pci_write_config_dword
int pci_write_config_dword(struct pci_device *pci, unsigned int where, uint32_t value)
Write 32-bit dword to PCI configuration space.
XHCI_PCH_USB3PRM
#define XHCI_PCH_USB3PRM
Intel PCH USB3 port routing mask register.
Definition: xhci.h:1038
xhci_trb_set_tr_dequeue_pointer::type
uint8_t type
Type.
Definition: xhci.h:574
cpu_to_le16
#define cpu_to_le16(value)
Definition: byteswap.h:106
xhci_pch_undo
static void xhci_pch_undo(struct xhci_device *xhci, struct pci_device *pci)
Undo Intel PCH-specific quirk fixes.
Definition: xhci.c:3425
XHCI_USBCMD_RUN
#define XHCI_USBCMD_RUN
Run/stop.
Definition: xhci.h:157
dest
if(len >=6 *4) __asm__ __volatile__("movsl" if(len >=5 *4) __asm__ __volatile__("movsl" if(len >=4 *4) __asm__ __volatile__("movsl" if(len >=3 *4) __asm__ __volatile__("movsl" if(len >=2 *4) __asm__ __volatile__("movsl" if(len >=1 *4) __asm__ __volatile__("movsl" if((len % 4) >=2) __asm__ __volatile__("movsw" if((len % 2) >=1) __asm__ __volatile__("movsb" return dest
Definition: string.h:150
iounmap
void iounmap(volatile const void *io_addr)
Unmap I/O address.
data
uint8_t data[48]
Additional event data.
Definition: ena.h:22
XHCI_TRB_CH
#define XHCI_TRB_CH
Transfer request block chain flag.
Definition: xhci.h:334
XHCI_CAP_HCSPARAMS1
#define XHCI_CAP_HCSPARAMS1
Structural parameters 1.
Definition: xhci.h:39
usb_host_operations
USB host controller operations.
Definition: usb.h:1030
USB_ENDPOINT_ATTR_CONTROL
#define USB_ENDPOINT_ATTR_CONTROL
Control endpoint transfer type.
Definition: usb.h:286
xhci_trb::port
struct xhci_trb_port_status port
Port status changed event.
Definition: xhci.h:714
xhci_device::dma
struct dma_device * dma
DMA device.
Definition: xhci.h:1072
xhci_endpoint_reset
static int xhci_endpoint_reset(struct usb_endpoint *ep)
Reset endpoint.
Definition: xhci.c:2547
xhci_enqueue
static int xhci_enqueue(struct xhci_trb_ring *ring, struct io_buffer *iobuf, const union xhci_trb *trb)
Enqueue a transfer request block.
Definition: xhci.c:1315
xhci_evaluate_context
static int xhci_evaluate_context(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Evaluate context.
Definition: xhci.c:2299
xhci_doorbell
static void xhci_doorbell(struct xhci_trb_ring *ring)
Ring doorbell register.
Definition: xhci.c:1436
xhci_port_protocol
static unsigned int xhci_port_protocol(struct xhci_device *xhci, unsigned int port)
Find port protocol.
Definition: xhci.c:705
xhci_trb_transfer::residual
uint16_t residual
Residual transfer length.
Definition: xhci.h:592
XHCI_SPEED_SUPER
Super speed.
Definition: xhci.h:150
XHCI_PORTSC_PR
#define XHCI_PORTSC_PR
Port reset.
Definition: xhci.h:212
xhci_device::legacy
unsigned int legacy
USB legacy support capability (if present and enabled)
Definition: xhci.h:1105
raw
__be32 raw[7]
Definition: CIB_PRM.h:28
usb_bus::hub
struct usb_hub * hub
Root hub.
Definition: usb.h:994
XHCI_CRCR_CRR
#define XHCI_CRCR_CRR
Command ring running.
Definition: xhci.h:187
XHCI_CONFIG_MAX_SLOTS_EN_MASK
#define XHCI_CONFIG_MAX_SLOTS_EN_MASK
Maximum device slots enabled mask.
Definition: xhci.h:199
XHCI_PORTSC_PED
#define XHCI_PORTSC_PED
Port enabled.
Definition: xhci.h:209
xhci_trb_ring::link
struct xhci_trb_link * link
Link TRB (if applicable)
Definition: xhci.h:859
XHCI_CAP_HCCPARAMS1
#define XHCI_CAP_HCCPARAMS1
Capability parameters.
Definition: xhci.h:58
snprintf
int snprintf(char *buf, size_t size, const char *fmt,...)
Write a formatted string to a buffer.
Definition: vsprintf.c:382
XHCI_SPEED_LOW
Low speed (1.5Mbps)
Definition: xhci.h:146
xhci_trb_ring::shift
unsigned int shift
Ring size (log2)
Definition: xhci.h:845
xhci_trb_ring
A transfer request block command/transfer ring.
Definition: xhci.h:839
usb_hub_set_drvdata
static void usb_hub_set_drvdata(struct usb_hub *hub, void *priv)
Set USB hub driver private data.
Definition: usb.h:936
usb_endpoint::usb
struct usb_device * usb
USB device.
Definition: usb.h:405
xhci_address_device_input
static void xhci_address_device_input(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input)
Populate address device input context.
Definition: xhci.c:2071
xhci_trb_transfer::transfer
uint64_t transfer
Transfer TRB pointer.
Definition: xhci.h:590
usb_port
static struct usb_port * usb_port(struct usb_hub *hub, unsigned int address)
Get USB port.
Definition: usb.h:959
xhci_device::csz_shift
unsigned int csz_shift
Context size shift.
Definition: xhci.h:1097
XHCI_EVENT_TRBS_LOG2
#define XHCI_EVENT_TRBS_LOG2
Number of TRBs in the event ring.
Definition: xhci.h:971
xhci_trb_setup::packet
struct usb_setup_packet packet
Setup packet.
Definition: xhci.h:372
XHCI_SUPPORTED_PSI_MANTISSA
#define XHCI_SUPPORTED_PSI_MANTISSA(psi)
Supported protocol PSI mantissa.
Definition: xhci.h:136
XHCI_MIN_ALIGN
#define XHCI_MIN_ALIGN
Minimum alignment required for data structures.
Definition: xhci.h:24
protocol
uint16_t protocol
Protocol ID.
Definition: stp.h:18
xhci_device::db
void * db
Doorbell registers.
Definition: xhci.h:1085
XHCI_TRB_LINK
#define XHCI_TRB_LINK
A link transfer request block.
Definition: xhci.h:459
xhci_endpoint::interval
unsigned int interval
Endpoint interval.
Definition: xhci.h:1173
XHCI_SETUP_IN
#define XHCI_SETUP_IN
Setup stage input data direction.
Definition: xhci.h:389
XHCI_PORTSC_PLS_DISABLED
#define XHCI_PORTSC_PLS_DISABLED
Disabled port link state.
Definition: xhci.h:218
xhci_reset_endpoint
static int xhci_reset_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Reset endpoint.
Definition: xhci.c:2327
DBG_EXTRA
#define DBG_EXTRA
Definition: compiler.h:319
usb_endpoint_host_operations::open
int(* open)(struct usb_endpoint *ep)
Open endpoint.
Definition: usb.h:449
XHCI_SUPPORTED_REVISION_VER
#define XHCI_SUPPORTED_REVISION_VER(revision)
Supported protocol minor revision.
Definition: xhci.h:106
XHCI_OP_USBCMD
#define XHCI_OP_USBCMD
USB command register.
Definition: xhci.h:154
offset
uint16_t offset
Offset to command line.
Definition: bzimage.h:8
xhci_trb_reset_endpoint::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:538
XHCI_RUN_ERSTSZ
#define XHCI_RUN_ERSTSZ(intr)
Event ring segment table size register.
Definition: xhci.h:295
XHCI_HCSPARAMS2_SCRATCHPADS
#define XHCI_HCSPARAMS2_SCRATCHPADS(params)
Number of page-sized scratchpad buffers.
Definition: xhci.h:54
XHCI_STATUS_OUT
#define XHCI_STATUS_OUT
Output status direction.
Definition: xhci.h:442
xhci_trb_complete::code
uint8_t code
Completion code.
Definition: xhci.h:617
supported
uint16_t supported
Bitmask of supported option values.
Definition: ena.h:12
pci_ioremap
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
XHCI_PCH_XUSB2PRM
#define XHCI_PCH_XUSB2PRM
Intel PCH USB2 port routing mask register.
Definition: xhci.h:1032
XHCI_USBSTS_HCH
#define XHCI_USBSTS_HCH
Host controller halted.
Definition: xhci.h:166
usb_hub::bus
struct usb_bus * bus
USB bus.
Definition: usb.h:844
fls
#define fls(x)
Find last (i.e.
Definition: strings.h:166
le64_to_cpu
#define le64_to_cpu(value)
Definition: byteswap.h:114
xhci_trb::link
struct xhci_trb_link link
Link TRB.
Definition: xhci.h:696
xhci_legacy_init
static void xhci_legacy_init(struct xhci_device *xhci)
Initialise USB legacy support.
Definition: xhci.c:539
DBG_LOG
#define DBG_LOG
Definition: compiler.h:317
xhci_device::slot
struct xhci_slot ** slot
Device slots, indexed by slot ID.
Definition: xhci.h:1123
XHCI_TRB_C
#define XHCI_TRB_C
Transfer request block cycle bit flag.
Definition: xhci.h:328
XHCI_CAP_HCSPARAMS2
#define XHCI_CAP_HCSPARAMS2
Structural parameters 2.
Definition: xhci.h:51
XHCI_SPEED_HIGH
High speed (480Mbps)
Definition: xhci.h:148
dma_mapping
A DMA mapping.
Definition: dma.h:32
xhci_event_alloc
static int xhci_event_alloc(struct xhci_device *xhci)
Allocate event ring.
Definition: xhci.c:1504
xhci_command
static int xhci_command(struct xhci_device *xhci, union xhci_trb *trb)
Issue command and wait for completion.
Definition: xhci.c:1856
xhci_device::quirks
unsigned int quirks
Quirks.
Definition: xhci.h:1076
NULL
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
xhci_trb_template::parameter
uint64_t parameter
Parameter.
Definition: xhci.h:306
XHCI_SUPPORTED_PSI_EXPONENT
#define XHCI_SUPPORTED_PSI_EXPONENT(psi)
Supported protocol PSI exponent.
Definition: xhci.h:139
xhci_endpoint::xhci
struct xhci_device * xhci
xHCI device
Definition: xhci.h:1163
ETIMEDOUT
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:669
string.h
String functions.
PCI_ROM
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:307
xhci_device::event
struct xhci_event_ring event
Event ring.
Definition: xhci.h:1116
xhci_endpoint_count
static unsigned int xhci_endpoint_count(size_t len, int zlp)
Calculate number of TRBs.
Definition: xhci.c:2673
usb_bus
A USB bus.
Definition: usb.h:965
XHCI_LINK_STATE_DELAY_MS
#define XHCI_LINK_STATE_DELAY_MS
Time to delay after writing the port link state.
Definition: xhci.h:245
usb_endpoint::attributes
unsigned int attributes
Attributes.
Definition: usb.h:409
xhci_legacy_prevent_release
static int xhci_legacy_prevent_release
Prevent the release of ownership back to BIOS.
Definition: xhci.c:532
XHCI_OP_DNCTRL
#define XHCI_OP_DNCTRL
Device notifcation control register.
Definition: xhci.h:175
bus
uint8_t bus
Bus.
Definition: edd.h:14
USB_SPEED_FULL
Full speed (12Mbps)
Definition: usb.h:50
writeq
void writeq(uint64_t data, volatile uint64_t *io_addr)
Write 64-bit qword to memory-mapped device.
dma
physaddr_t dma(struct dma_mapping *map, void *addr)
Get DMA address from virtual address.
xhci_device::op
void * op
Operational registers.
Definition: xhci.h:1081
usb_endpoint::address
unsigned int address
Endpoint address.
Definition: usb.h:407
XHCI_RUN_ERDP
#define XHCI_RUN_ERDP(intr)
Event ring dequeue pointer register.
Definition: xhci.h:301
xhci_trb_enable_slot::type
uint8_t type
Type.
Definition: xhci.h:473
free_usb_bus
void free_usb_bus(struct usb_bus *bus)
Free USB bus.
Definition: usb.c:2194
xhci_nop
static int xhci_nop(struct xhci_device *xhci)
Issue NOP and wait for completion.
Definition: xhci.c:1927
xhci_unregister
void xhci_unregister(struct xhci_device *xhci)
Unregister xHCI controller.
Definition: xhci.c:3363
XHCI_CTX_EP0
#define XHCI_CTX_EP0
Endpoint zero context index.
Definition: xhci.h:950
xhci_slot_context
A slot context.
Definition: xhci.h:738
XHCI_PORTSC_PLS_MASK
#define XHCI_PORTSC_PLS_MASK
Port link state mask.
Definition: xhci.h:224
xhci_trb_set_tr_dequeue_pointer::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:576
XHCI_TRB_IOC
#define XHCI_TRB_IOC
Transfer request block interrupt on completion flag.
Definition: xhci.h:337
xhci_hub_open
static int xhci_hub_open(struct usb_hub *hub)
Open hub.
Definition: xhci.c:3032
xhci_trb_context::type
uint8_t type
Type.
Definition: xhci.h:511
xhci_port_status
static void xhci_port_status(struct xhci_device *xhci, struct xhci_trb_port_status *trb)
Handle port status event.
Definition: xhci.c:1705
xhci_ring_alloc
static int xhci_ring_alloc(struct xhci_device *xhci, struct xhci_trb_ring *ring, unsigned int shift, unsigned int slot, unsigned int target, unsigned int stream)
Allocate transfer request block ring.
Definition: xhci.c:1216
usb_complete_err
void usb_complete_err(struct usb_endpoint *ep, struct io_buffer *iobuf, int rc)
Complete transfer (possibly with error)
Definition: usb.c:586
XHCI_ENDPOINT_STATE_MASK
#define XHCI_ENDPOINT_STATE_MASK
Endpoint state mask.
Definition: xhci.h:808
usb_hub_get_drvdata
static void * usb_hub_get_drvdata(struct usb_hub *hub)
Get USB hub driver private data.
Definition: usb.h:947
XHCI_CMD_TRBS_LOG2
#define XHCI_CMD_TRBS_LOG2
Number of TRBs (excluding Link TRB) in the command ring.
Definition: xhci.h:965
xhci_operations
static struct usb_host_operations xhci_operations
USB host controller operations.
Definition: xhci.c:3279
usb_endpoint_get_hostdata
static void * usb_endpoint_get_hostdata(struct usb_endpoint *ep)
Get USB endpoint host controller private data.
Definition: usb.h:586
strings.h
String functions.
xhci_legacy_release
static void xhci_legacy_release(struct xhci_device *xhci)
Release ownership back to BIOS.
Definition: xhci.c:617
xhci_trb_context::slot
uint8_t slot
Slot ID.
Definition: xhci.h:515
xhci_dcbaa_alloc
static int xhci_dcbaa_alloc(struct xhci_device *xhci)
Allocate device context base address array.
Definition: xhci.c:916
XHCI_SUPPORTED_NAME
#define XHCI_SUPPORTED_NAME
Supported protocol name.
Definition: xhci.h:109
memset
void * memset(void *dest, int character, size_t len) __nonnull
XHCI_CTX_SLOT
#define XHCI_CTX_SLOT
Slot context index.
Definition: xhci.h:942
xhci_endpoint_close
static void xhci_endpoint_close(struct usb_endpoint *ep)
Close endpoint.
Definition: xhci.c:2517
io_buffer
A persistent I/O buffer.
Definition: iobuf.h:37
Generated by   doxygen 1.8.15