iPXE
xhci.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 FILE_SECBOOT ( PERMITTED );
26 
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <unistd.h>
30 #include <string.h>
31 #include <strings.h>
32 #include <errno.h>
33 #include <byteswap.h>
34 #include <ipxe/malloc.h>
35 #include <ipxe/pci.h>
36 #include <ipxe/usb.h>
37 #include <ipxe/init.h>
38 #include <ipxe/profile.h>
39 #include <ipxe/xhci.h>
40 
41 /** @file
42  *
43  * USB eXtensible Host Controller Interface (xHCI) driver
44  *
45  */
46 
47 /** Message transfer profiler */
48 static struct profiler xhci_message_profiler __profiler =
49  { .name = "xhci.message" };
50 
51 /** Stream transfer profiler */
52 static struct profiler xhci_stream_profiler __profiler =
53  { .name = "xhci.stream" };
54 
55 /** Event ring profiler */
56 static struct profiler xhci_event_profiler __profiler =
57  { .name = "xhci.event" };
58 
59 /** Transfer event profiler */
60 static struct profiler xhci_transfer_profiler __profiler =
61  { .name = "xhci.transfer" };
62 
63 /* Disambiguate the various error causes */
64 #define EIO_DATA \
65  __einfo_error ( EINFO_EIO_DATA )
66 #define EINFO_EIO_DATA \
67  __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \
68  "Data buffer error" )
69 #define EIO_BABBLE \
70  __einfo_error ( EINFO_EIO_BABBLE )
71 #define EINFO_EIO_BABBLE \
72  __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \
73  "Babble detected" )
74 #define EIO_USB \
75  __einfo_error ( EINFO_EIO_USB )
76 #define EINFO_EIO_USB \
77  __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \
78  "USB transaction error" )
79 #define EIO_TRB \
80  __einfo_error ( EINFO_EIO_TRB )
81 #define EINFO_EIO_TRB \
82  __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \
83  "TRB error" )
84 #define EIO_STALL \
85  __einfo_error ( EINFO_EIO_STALL )
86 #define EINFO_EIO_STALL \
87  __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \
88  "Stall error" )
89 #define EIO_RESOURCE \
90  __einfo_error ( EINFO_EIO_RESOURCE )
91 #define EINFO_EIO_RESOURCE \
92  __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \
93  "Resource error" )
94 #define EIO_BANDWIDTH \
95  __einfo_error ( EINFO_EIO_BANDWIDTH )
96 #define EINFO_EIO_BANDWIDTH \
97  __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \
98  "Bandwidth error" )
99 #define EIO_NO_SLOTS \
100  __einfo_error ( EINFO_EIO_NO_SLOTS )
101 #define EINFO_EIO_NO_SLOTS \
102  __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \
103  "No slots available" )
104 #define EIO_STREAM_TYPE \
105  __einfo_error ( EINFO_EIO_STREAM_TYPE )
106 #define EINFO_EIO_STREAM_TYPE \
107  __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \
108  "Invalid stream type" )
109 #define EIO_SLOT \
110  __einfo_error ( EINFO_EIO_SLOT )
111 #define EINFO_EIO_SLOT \
112  __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \
113  "Slot not enabled" )
114 #define EIO_ENDPOINT \
115  __einfo_error ( EINFO_EIO_ENDPOINT )
116 #define EINFO_EIO_ENDPOINT \
117  __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \
118  "Endpoint not enabled" )
119 #define EIO_SHORT \
120  __einfo_error ( EINFO_EIO_SHORT )
121 #define EINFO_EIO_SHORT \
122  __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \
123  "Short packet" )
124 #define EIO_UNDERRUN \
125  __einfo_error ( EINFO_EIO_UNDERRUN )
126 #define EINFO_EIO_UNDERRUN \
127  __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \
128  "Ring underrun" )
129 #define EIO_OVERRUN \
130  __einfo_error ( EINFO_EIO_OVERRUN )
131 #define EINFO_EIO_OVERRUN \
132  __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \
133  "Ring overrun" )
134 #define EIO_VF_RING_FULL \
135  __einfo_error ( EINFO_EIO_VF_RING_FULL )
136 #define EINFO_EIO_VF_RING_FULL \
137  __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \
138  "Virtual function event ring full" )
139 #define EIO_PARAMETER \
140  __einfo_error ( EINFO_EIO_PARAMETER )
141 #define EINFO_EIO_PARAMETER \
142  __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \
143  "Parameter error" )
144 #define EIO_BANDWIDTH_OVERRUN \
145  __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
146 #define EINFO_EIO_BANDWIDTH_OVERRUN \
147  __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \
148  "Bandwidth overrun" )
149 #define EIO_CONTEXT \
150  __einfo_error ( EINFO_EIO_CONTEXT )
151 #define EINFO_EIO_CONTEXT \
152  __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \
153  "Context state error" )
154 #define EIO_NO_PING \
155  __einfo_error ( EINFO_EIO_NO_PING )
156 #define EINFO_EIO_NO_PING \
157  __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \
158  "No ping response" )
159 #define EIO_RING_FULL \
160  __einfo_error ( EINFO_EIO_RING_FULL )
161 #define EINFO_EIO_RING_FULL \
162  __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \
163  "Event ring full" )
164 #define EIO_INCOMPATIBLE \
165  __einfo_error ( EINFO_EIO_INCOMPATIBLE )
166 #define EINFO_EIO_INCOMPATIBLE \
167  __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \
168  "Incompatible device" )
169 #define EIO_MISSED \
170  __einfo_error ( EINFO_EIO_MISSED )
171 #define EINFO_EIO_MISSED \
172  __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \
173  "Missed service error" )
174 #define EIO_CMD_STOPPED \
175  __einfo_error ( EINFO_EIO_CMD_STOPPED )
176 #define EINFO_EIO_CMD_STOPPED \
177  __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \
178  "Command ring stopped" )
179 #define EIO_CMD_ABORTED \
180  __einfo_error ( EINFO_EIO_CMD_ABORTED )
181 #define EINFO_EIO_CMD_ABORTED \
182  __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \
183  "Command aborted" )
184 #define EIO_STOP \
185  __einfo_error ( EINFO_EIO_STOP )
186 #define EINFO_EIO_STOP \
187  __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \
188  "Stopped" )
189 #define EIO_STOP_LEN \
190  __einfo_error ( EINFO_EIO_STOP_LEN )
191 #define EINFO_EIO_STOP_LEN \
192  __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \
193  "Stopped - length invalid" )
194 #define EIO_STOP_SHORT \
195  __einfo_error ( EINFO_EIO_STOP_SHORT )
196 #define EINFO_EIO_STOP_SHORT \
197  __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \
198  "Stopped - short packet" )
199 #define EIO_LATENCY \
200  __einfo_error ( EINFO_EIO_LATENCY )
201 #define EINFO_EIO_LATENCY \
202  __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \
203  "Maximum exit latency too large" )
204 #define EIO_ISOCH \
205  __einfo_error ( EINFO_EIO_ISOCH )
206 #define EINFO_EIO_ISOCH \
207  __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \
208  "Isochronous buffer overrun" )
209 #define EPROTO_LOST \
210  __einfo_error ( EINFO_EPROTO_LOST )
211 #define EINFO_EPROTO_LOST \
212  __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \
213  "Event lost" )
214 #define EPROTO_UNDEFINED \
215  __einfo_error ( EINFO_EPROTO_UNDEFINED )
216 #define EINFO_EPROTO_UNDEFINED \
217  __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \
218  "Undefined error" )
219 #define EPROTO_STREAM_ID \
220  __einfo_error ( EINFO_EPROTO_STREAM_ID )
221 #define EINFO_EPROTO_STREAM_ID \
222  __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \
223  "Invalid stream ID" )
224 #define EPROTO_SECONDARY \
225  __einfo_error ( EINFO_EPROTO_SECONDARY )
226 #define EINFO_EPROTO_SECONDARY \
227  __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \
228  "Secondary bandwidth error" )
229 #define EPROTO_SPLIT \
230  __einfo_error ( EINFO_EPROTO_SPLIT )
231 #define EINFO_EPROTO_SPLIT \
232  __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \
233  "Split transaction error" )
234 #define ECODE(code) \
235  ( ( (code) < 32 ) ? \
236  EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \
237  EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \
238  EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \
239  EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \
240  EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \
241  EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \
242  EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \
243  EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \
244  EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \
245  EIO_ISOCH ) : \
246  ( (code) < 64 ) ? \
247  EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \
248  EPROTO_UNDEFINED, EPROTO_STREAM_ID, \
249  EPROTO_SECONDARY, EPROTO_SPLIT ) : \
250  EFAULT )
251 
252 /******************************************************************************
253  *
254  * Register access
255  *
256  ******************************************************************************
257  */
258 
259 /**
260  * Initialise device
261  *
262  * @v xhci xHCI device
263  */
264 void xhci_init ( struct xhci_device *xhci ) {
265  uint32_t hcsparams1;
266  uint32_t hcsparams2;
267  uint32_t hccparams1;
268  uint32_t pagesize;
269  size_t caplength;
270  size_t rtsoff;
271  size_t dboff;
272 
273  /* Set device name */
274  xhci->name = xhci->dev->name;
275 
276  /* Locate capability, operational, runtime, and doorbell registers */
277  xhci->cap = xhci->regs;
278  caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
279  rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
280  dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
281  xhci->op = ( xhci->cap + caplength );
282  xhci->run = ( xhci->cap + rtsoff );
283  xhci->db = ( xhci->cap + dboff );
284  DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n",
285  xhci->name, virt_to_phys ( xhci->cap ),
286  virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ),
287  virt_to_phys ( xhci->db ) );
288 
289  /* Read structural parameters 1 */
290  hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
291  xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
292  xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
293  xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
294  DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n",
295  xhci->name, xhci->slots, xhci->intrs, xhci->ports );
296 
297  /* Read structural parameters 2 */
298  hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
299  xhci->scratch.count = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
300  DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
301  xhci->name, xhci->scratch.count );
302 
303  /* Read capability parameters 1 */
304  hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
305  xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
306  xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
307  xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
308 
309  /* Read page size */
310  pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
311  xhci->pagesize = XHCI_PAGESIZE ( pagesize );
312  assert ( xhci->pagesize != 0 );
313  assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
314  DBGC2 ( xhci, "XHCI %s page size %zd bytes\n",
315  xhci->name, xhci->pagesize );
316 
317  /* Configure DMA device */
318  if ( xhci->dma && xhci->addr64 )
319  dma_set_mask_64bit ( xhci->dma );
320 }
321 
322 /**
323  * Find extended capability
324  *
325  * @v xhci xHCI device
326  * @v id Capability ID
327  * @v offset Offset to previous extended capability instance, or zero
328  * @ret offset Offset to extended capability, or zero if not found
329  */
330 static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
331  unsigned int id,
332  unsigned int offset ) {
333  uint32_t xecp;
334  unsigned int next;
335 
336  /* Locate the extended capability */
337  while ( 1 ) {
338 
339  /* Locate first or next capability as applicable */
340  if ( offset ) {
341  xecp = readl ( xhci->cap + offset );
342  next = XHCI_XECP_NEXT ( xecp );
343  } else {
344  next = xhci->xecp;
345  }
346  if ( ! next )
347  return 0;
348  offset += next;
349 
350  /* Check if this is the requested capability */
351  xecp = readl ( xhci->cap + offset );
352  if ( XHCI_XECP_ID ( xecp ) == id )
353  return offset;
354  }
355 }
356 
357 /**
358  * Write potentially 64-bit register
359  *
360  * @v xhci xHCI device
361  * @v value Value
362  * @v reg Register address
363  * @ret rc Return status code
364  */
365 static inline __attribute__ (( always_inline )) int
366 xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
367 
368  /* If this is a 32-bit build, then this can never fail
369  * (allowing the compiler to optimise out the error path).
370  */
371  if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
372  writel ( value, reg );
373  writel ( 0, ( reg + sizeof ( uint32_t ) ) );
374  return 0;
375  }
376 
377  /* If the device does not support 64-bit addresses and this
378  * address is outside the 32-bit address space, then fail.
379  */
380  if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
381  DBGC ( xhci, "XHCI %s cannot access address %lx\n",
382  xhci->name, value );
383  return -ENOTSUP;
384  }
385 
386  /* If this is a 64-bit build, then writeq() is available */
387  writeq ( value, reg );
388  return 0;
389 }
390 
391 /**
392  * Calculate buffer alignment
393  *
394  * @v len Length
395  * @ret align Buffer alignment
396  *
397  * Determine alignment required for a buffer which must be aligned to
398  * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
399  */
400 static inline size_t xhci_align ( size_t len ) {
401  size_t align;
402 
403  /* Align to own length (rounded up to a power of two) */
404  align = ( 1 << fls ( len - 1 ) );
405 
406  /* Round up to XHCI_MIN_ALIGN if needed */
407  if ( align < XHCI_MIN_ALIGN )
408  align = XHCI_MIN_ALIGN;
409 
410  return align;
411 }
412 
413 /**
414  * Calculate device context offset
415  *
416  * @v xhci xHCI device
417  * @v ctx Context index
418  */
419 static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
420  unsigned int ctx ) {
421 
422  return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
423 }
424 
425 /**
426  * Calculate input context offset
427  *
428  * @v xhci xHCI device
429  * @v ctx Context index
430  */
431 static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
432  unsigned int ctx ) {
433 
434  return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
435 }
436 
437 /******************************************************************************
438  *
439  * Diagnostics
440  *
441  ******************************************************************************
442  */
443 
444 /**
445  * Dump host controller registers
446  *
447  * @v xhci xHCI device
448  */
449 static inline void xhci_dump ( struct xhci_device *xhci ) {
450  uint32_t usbcmd;
451  uint32_t usbsts;
452  uint32_t pagesize;
453  uint32_t dnctrl;
454  uint32_t config;
455 
456  /* Do nothing unless debugging is enabled */
457  if ( ! DBG_LOG )
458  return;
459 
460  /* Dump USBCMD */
461  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
462  DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd,
463  ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
464  ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
465 
466  /* Dump USBSTS */
467  usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
468  DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts,
469  ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
470 
471  /* Dump PAGESIZE */
472  pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
473  DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize );
474 
475  /* Dump DNCTRL */
476  dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
477  DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl );
478 
479  /* Dump CONFIG */
480  config = readl ( xhci->op + XHCI_OP_CONFIG );
481  DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config );
482 }
483 
484 /**
485  * Dump port registers
486  *
487  * @v xhci xHCI device
488  * @v port Port number
489  */
490 static inline void xhci_dump_port ( struct xhci_device *xhci,
491  unsigned int port ) {
492  uint32_t portsc;
493  uint32_t portpmsc;
494  uint32_t portli;
495  uint32_t porthlpmc;
496 
497  /* Do nothing unless debugging is enabled */
498  if ( ! DBG_LOG )
499  return;
500 
501  /* Dump PORTSC */
502  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
503  DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n",
504  xhci->name, port, portsc,
505  ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
506  ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
507  ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
508  ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
509  XHCI_PORTSC_PSIV ( portsc ) );
510 
511  /* Dump PORTPMSC */
512  portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
513  DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc );
514 
515  /* Dump PORTLI */
516  portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
517  DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli );
518 
519  /* Dump PORTHLPMC */
520  porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
521  DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n",
522  xhci->name, port, porthlpmc );
523 }
524 
525 /******************************************************************************
526  *
527  * USB legacy support
528  *
529  ******************************************************************************
530  */
531 
532 /** Prevent the release of ownership back to BIOS */
533 static int xhci_legacy_prevent_release;
534 
535 /**
536  * Initialise USB legacy support
537  *
538  * @v xhci xHCI device
539  */
540 static void xhci_legacy_init ( struct xhci_device *xhci ) {
541  unsigned int legacy;
542  uint8_t bios;
543 
544  /* Locate USB legacy support capability (if present) */
545  legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
546  if ( ! legacy ) {
547  /* Not an error; capability may not be present */
548  DBGC ( xhci, "XHCI %s has no USB legacy support capability\n",
549  xhci->name );
550  return;
551  }
552 
553  /* Check if legacy USB support is enabled */
554  bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
555  if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
556  /* Not an error; already owned by OS */
557  DBGC ( xhci, "XHCI %s USB legacy support already disabled\n",
558  xhci->name );
559  return;
560  }
561 
562  /* Record presence of USB legacy support capability */
563  xhci->legacy = legacy;
564 }
565 
566 /**
567  * Claim ownership from BIOS
568  *
569  * @v xhci xHCI device
570  */
571 static void xhci_legacy_claim ( struct xhci_device *xhci ) {
572  uint32_t ctlsts;
573  uint8_t bios;
574  unsigned int i;
575 
576  /* Do nothing unless legacy support capability is present */
577  if ( ! xhci->legacy )
578  return;
579 
580  /* Claim ownership */
581  writeb ( XHCI_USBLEGSUP_OS_OWNED,
582  xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
583 
584  /* Wait for BIOS to release ownership */
585  for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
586 
587  /* Check if BIOS has released ownership */
588  bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
589  if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
590  DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n",
591  xhci->name );
592  ctlsts = readl ( xhci->cap + xhci->legacy +
593  XHCI_USBLEGSUP_CTLSTS );
594  if ( ctlsts ) {
595  DBGC ( xhci, "XHCI %s warning: BIOS retained "
596  "SMIs: %08x\n", xhci->name, ctlsts );
597  }
598  return;
599  }
600 
601  /* Delay */
602  mdelay ( 1 );
603  }
604 
605  /* BIOS did not release ownership. Claim it forcibly by
606  * disabling all SMIs.
607  */
608  DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly "
609  "disabling SMIs\n", xhci->name );
610  writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS );
611 }
612 
613 /**
614  * Release ownership back to BIOS
615  *
616  * @v xhci xHCI device
617  */
618 static void xhci_legacy_release ( struct xhci_device *xhci ) {
619 
620  /* Do nothing unless legacy support capability is present */
621  if ( ! xhci->legacy )
622  return;
623 
624  /* Do nothing if releasing ownership is prevented */
625  if ( xhci_legacy_prevent_release ) {
626  DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n",
627  xhci->name );
628  return;
629  }
630 
631  /* Release ownership */
632  writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
633  DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name );
634 }
635 
636 /******************************************************************************
637  *
638  * Supported protocols
639  *
640  ******************************************************************************
641  */
642 
643 /**
644  * Transcribe port speed (for debugging)
645  *
646  * @v psi Protocol speed ID
647  * @ret speed Transcribed speed
648  */
649 static inline const char * xhci_speed_name ( uint32_t psi ) {
650  static const char *exponents[4] = { "", "k", "M", "G" };
651  static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
652  unsigned int mantissa;
653  unsigned int exponent;
654 
655  /* Extract mantissa and exponent */
656  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
657  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
658 
659  /* Transcribe speed */
660  snprintf ( buf, sizeof ( buf ), "%d%sbps",
661  mantissa, exponents[exponent] );
662  return buf;
663 }
664 
665 /**
666  * Find supported protocol extended capability for a port
667  *
668  * @v xhci xHCI device
669  * @v port Port number
670  * @ret supported Offset to extended capability, or zero if not found
671  */
672 static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
673  unsigned int port ) {
674  unsigned int supported = 0;
675  unsigned int offset;
676  unsigned int count;
677  uint32_t ports;
678 
679  /* Iterate over all supported protocol structures */
680  while ( ( supported = xhci_extended_capability ( xhci,
681  XHCI_XECP_ID_SUPPORTED,
682  supported ) ) ) {
683 
684  /* Determine port range */
685  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
686  offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
687  count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
688 
689  /* Check if port lies within this range */
690  if ( ( port - offset ) < count )
691  return supported;
692  }
693 
694  DBGC ( xhci, "XHCI %s-%d has no supported protocol\n",
695  xhci->name, port );
696  return 0;
697 }
698 
699 /**
700  * Find port protocol
701  *
702  * @v xhci xHCI device
703  * @v port Port number
704  * @ret protocol USB protocol, or zero if not found
705  */
706 static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
707  unsigned int port ) {
708  unsigned int supported = xhci_supported_protocol ( xhci, port );
709  union {
710  uint32_t raw;
711  char text[5];
712  } name;
713  unsigned int protocol;
714  unsigned int type;
715  unsigned int psic;
716  unsigned int psiv;
717  unsigned int i;
718  uint32_t revision;
719  uint32_t ports;
720  uint32_t slot;
721  uint32_t psi;
722 
723  /* Fail if there is no supported protocol */
724  if ( ! supported )
725  return 0;
726 
727  /* Determine protocol version */
728  revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
729  protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
730 
731  /* Describe port protocol */
732  if ( DBG_EXTRA ) {
733  name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
734  XHCI_SUPPORTED_NAME ) );
735  name.text[4] = '\0';
736  slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
737  type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
738  DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d",
739  xhci->name, port, name.text, protocol, type );
740  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
741  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
742  if ( psic ) {
743  DBGC2 ( xhci, " speeds" );
744  for ( i = 0 ; i < psic ; i++ ) {
745  psi = readl ( xhci->cap + supported +
746  XHCI_SUPPORTED_PSI ( i ) );
747  psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
748  DBGC2 ( xhci, " %d:%s", psiv,
749  xhci_speed_name ( psi ) );
750  }
751  }
752  if ( xhci->quirks & XHCI_BAD_PSIV )
753  DBGC2 ( xhci, " (ignored)" );
754  DBGC2 ( xhci, "\n" );
755  }
756 
757  return protocol;
758 }
759 
760 /**
761  * Find port slot type
762  *
763  * @v xhci xHCI device
764  * @v port Port number
765  * @ret type Slot type, or negative error
766  */
767 static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
768  unsigned int supported = xhci_supported_protocol ( xhci, port );
769  unsigned int type;
770  uint32_t slot;
771 
772  /* Fail if there is no supported protocol */
773  if ( ! supported )
774  return -ENOTSUP;
775 
776  /* Get slot type */
777  slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
778  type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
779 
780  return type;
781 }
782 
783 /**
784  * Find port speed
785  *
786  * @v xhci xHCI device
787  * @v port Port number
788  * @v psiv Protocol speed ID value
789  * @ret speed Port speed, or negative error
790  */
791 static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
792  unsigned int psiv ) {
793  unsigned int supported = xhci_supported_protocol ( xhci, port );
794  unsigned int psic;
795  unsigned int mantissa;
796  unsigned int exponent;
797  unsigned int speed;
798  unsigned int i;
799  uint32_t ports;
800  uint32_t psi;
801 
802  /* Fail if there is no supported protocol */
803  if ( ! supported )
804  return -ENOTSUP;
805 
806  /* Get protocol speed ID count */
807  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
808  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
809 
810  /* Use protocol speed ID table unless device is known to be faulty */
811  if ( ! ( xhci->quirks & XHCI_BAD_PSIV ) ) {
812 
813  /* Iterate over PSI dwords looking for a match */
814  for ( i = 0 ; i < psic ; i++ ) {
815  psi = readl ( xhci->cap + supported +
816  XHCI_SUPPORTED_PSI ( i ) );
817  if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
818  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
819  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
820  speed = USB_SPEED ( mantissa, exponent );
821  return speed;
822  }
823  }
824 
825  /* Record device as faulty if no match is found */
826  if ( psic != 0 ) {
827  DBGC ( xhci, "XHCI %s-%d spurious PSI value %d: "
828  "assuming PSI table is invalid\n",
829  xhci->name, port, psiv );
830  xhci->quirks |= XHCI_BAD_PSIV;
831  }
832  }
833 
834  /* Use the default mappings */
835  switch ( psiv ) {
836  case XHCI_SPEED_LOW : return USB_SPEED_LOW;
837  case XHCI_SPEED_FULL : return USB_SPEED_FULL;
838  case XHCI_SPEED_HIGH : return USB_SPEED_HIGH;
839  case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
840  default:
841  DBGC ( xhci, "XHCI %s-%d unrecognised PSI value %d\n",
842  xhci->name, port, psiv );
843  return -ENOTSUP;
844  }
845 }
846 
847 /**
848  * Find protocol speed ID value
849  *
850  * @v xhci xHCI device
851  * @v port Port number
852  * @v speed USB speed
853  * @ret psiv Protocol speed ID value, or negative error
854  */
855 static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
856  unsigned int speed ) {
857  unsigned int supported = xhci_supported_protocol ( xhci, port );
858  unsigned int psic;
859  unsigned int mantissa;
860  unsigned int exponent;
861  unsigned int psiv;
862  unsigned int i;
863  uint32_t ports;
864  uint32_t psi;
865 
866  /* Fail if there is no supported protocol */
867  if ( ! supported )
868  return -ENOTSUP;
869 
870  /* Get protocol speed ID count */
871  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
872  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
873 
874  /* Use the default mappings if applicable */
875  if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
876  switch ( speed ) {
877  case USB_SPEED_LOW : return XHCI_SPEED_LOW;
878  case USB_SPEED_FULL : return XHCI_SPEED_FULL;
879  case USB_SPEED_HIGH : return XHCI_SPEED_HIGH;
880  case USB_SPEED_SUPER : return XHCI_SPEED_SUPER;
881  default:
882  DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n",
883  xhci->name, port, speed );
884  return -ENOTSUP;
885  }
886  }
887 
888  /* Iterate over PSI dwords looking for a match */
889  for ( i = 0 ; i < psic ; i++ ) {
890  psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
891  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
892  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
893  if ( speed == USB_SPEED ( mantissa, exponent ) ) {
894  psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
895  return psiv;
896  }
897  }
898 
899  DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n",
900  xhci->name, port, speed );
901  return -ENOENT;
902 }
903 
904 /******************************************************************************
905  *
906  * Device context base address array
907  *
908  ******************************************************************************
909  */
910 
911 /**
912  * Allocate device context base address array
913  *
914  * @v xhci xHCI device
915  * @ret rc Return status code
916  */
917 static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
918  size_t len;
919  physaddr_t dcbaap;
920  int rc;
921 
922  /* Allocate and initialise structure. Must be at least
923  * 64-byte aligned and must not cross a page boundary, so
924  * align on its own size (rounded up to a power of two and
925  * with a minimum of 64 bytes).
926  */
927  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
928  xhci->dcbaa.context = dma_alloc ( xhci->dma, &xhci->dcbaa.map, len,
929  xhci_align ( len ) );
930  if ( ! xhci->dcbaa.context ) {
931  DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
932  rc = -ENOMEM;
933  goto err_alloc;
934  }
935  memset ( xhci->dcbaa.context, 0, len );
936 
937  /* Program DCBAA pointer */
938  dcbaap = dma ( &xhci->dcbaa.map, xhci->dcbaa.context );
939  if ( ( rc = xhci_writeq ( xhci, dcbaap,
940  xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
941  goto err_writeq;
942 
943  DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n", xhci->name,
944  virt_to_phys ( xhci->dcbaa.context ),
945  ( virt_to_phys ( xhci->dcbaa.context ) + len ) );
946  return 0;
947 
948  err_writeq:
949  dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
950  err_alloc:
951  return rc;
952 }
953 
954 /**
955  * Free device context base address array
956  *
957  * @v xhci xHCI device
958  */
959 static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
960  size_t len;
961  unsigned int i;
962 
963  /* Sanity check */
964  for ( i = 0 ; i <= xhci->slots ; i++ )
965  assert ( xhci->dcbaa.context[i] == 0 );
966 
967  /* Clear DCBAA pointer */
968  xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
969 
970  /* Free DCBAA */
971  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
972  dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
973 }
974 
975 /******************************************************************************
976  *
977  * Scratchpad buffers
978  *
979  ******************************************************************************
980  */
981 
982 /**
983  * Allocate scratchpad buffers
984  *
985  * @v xhci xHCI device
986  * @ret rc Return status code
987  */
988 static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
989  struct xhci_scratchpad *scratch = &xhci->scratch;
990  size_t buffer_len;
991  size_t array_len;
992  physaddr_t addr;
993  unsigned int i;
994  int rc;
995 
996  /* Do nothing if no scratchpad buffers are used */
997  if ( ! scratch->count )
998  return 0;
999 
1000  /* Allocate scratchpad buffers */
1001  buffer_len = ( scratch->count * xhci->pagesize );
1002  scratch->buffer = dma_umalloc ( xhci->dma, &scratch->buffer_map,
1003  buffer_len, xhci->pagesize );
1004  if ( ! scratch->buffer ) {
1005  DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
1006  xhci->name );
1007  rc = -ENOMEM;
1008  goto err_alloc;
1009  }
1010  memset ( scratch->buffer, 0, buffer_len );
1011 
1012  /* Allocate scratchpad array */
1013  array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
1014  scratch->array = dma_alloc ( xhci->dma, &scratch->array_map,
1015  array_len, xhci_align ( array_len ) );
1016  if ( ! scratch->array ) {
1017  DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
1018  "array\n", xhci->name );
1019  rc = -ENOMEM;
1020  goto err_alloc_array;
1021  }
1022 
1023  /* Populate scratchpad array */
1024  addr = dma ( &scratch->buffer_map, scratch->buffer );
1025  for ( i = 0 ; i < scratch->count ; i++ ) {
1026  scratch->array[i] = cpu_to_le64 ( addr );
1027  addr += xhci->pagesize;
1028  }
1029 
1030  /* Set scratchpad array pointer */
1031  assert ( xhci->dcbaa.context != NULL );
1032  xhci->dcbaa.context[0] = cpu_to_le64 ( dma ( &scratch->array_map,
1033  scratch->array ) );
1034 
1035  DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
1036  xhci->name, virt_to_phys ( scratch->buffer ),
1037  ( virt_to_phys ( scratch->buffer ) + buffer_len ),
1038  virt_to_phys ( scratch->array ),
1039  ( virt_to_phys ( scratch->array ) + array_len ) );
1040  return 0;
1041 
1042  dma_free ( &scratch->array_map, scratch->array, array_len );
1043  err_alloc_array:
1044  dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
1045  err_alloc:
1046  return rc;
1047 }
1048 
1049 /**
1050  * Free scratchpad buffers
1051  *
1052  * @v xhci xHCI device
1053  */
1054 static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
1055  struct xhci_scratchpad *scratch = &xhci->scratch;
1056  size_t array_len;
1057  size_t buffer_len;
1058 
1059  /* Do nothing if no scratchpad buffers are used */
1060  if ( ! scratch->count )
1061  return;
1062 
1063  /* Clear scratchpad array pointer */
1064  assert ( xhci->dcbaa.context != NULL );
1065  xhci->dcbaa.context[0] = 0;
1066 
1067  /* Free scratchpad array */
1068  array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
1069  dma_free ( &scratch->array_map, scratch->array, array_len );
1070 
1071  /* Free scratchpad buffers */
1072  buffer_len = ( scratch->count * xhci->pagesize );
1073  dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
1074 }
1075 
1076 /******************************************************************************
1077  *
1078  * Run / stop / reset
1079  *
1080  ******************************************************************************
1081  */
1082 
1083 /**
1084  * Start xHCI device
1085  *
1086  * @v xhci xHCI device
1087  */
1088 static void xhci_run ( struct xhci_device *xhci ) {
1089  uint32_t config;
1090  uint32_t usbcmd;
1091 
1092  /* Configure number of device slots */
1093  config = readl ( xhci->op + XHCI_OP_CONFIG );
1094  config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
1095  config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
1096  writel ( config, xhci->op + XHCI_OP_CONFIG );
1097 
1098  /* Set run/stop bit */
1099  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1100  usbcmd |= XHCI_USBCMD_RUN;
1101  writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
1102 }
1103 
1104 /**
1105  * Stop xHCI device
1106  *
1107  * @v xhci xHCI device
1108  * @ret rc Return status code
1109  */
1110 static int xhci_stop ( struct xhci_device *xhci ) {
1111  uint32_t usbcmd;
1112  uint32_t usbsts;
1113  unsigned int i;
1114 
1115  /* Clear run/stop bit */
1116  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1117  usbcmd &= ~XHCI_USBCMD_RUN;
1118  writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
1119 
1120  /* Wait for device to stop */
1121  for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
1122 
1123  /* Check if device is stopped */
1124  usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
1125  if ( usbsts & XHCI_USBSTS_HCH )
1126  return 0;
1127 
1128  /* Delay */
1129  mdelay ( 1 );
1130  }
1131 
1132  DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name );
1133  return -ETIMEDOUT;
1134 }
1135 
1136 /**
1137  * Reset xHCI device
1138  *
1139  * @v xhci xHCI device
1140  * @ret rc Return status code
1141  */
1142 static int xhci_reset ( struct xhci_device *xhci ) {
1143  uint32_t usbcmd;
1144  unsigned int i;
1145  int rc;
1146 
1147  /* The xHCI specification states that resetting a running
1148  * device may result in undefined behaviour, so try stopping
1149  * it first.
1150  */
1151  if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
1152  /* Ignore errors and attempt to reset the device anyway */
1153  }
1154 
1155  /* Reset device */
1156  writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
1157 
1158  /* Wait for reset to complete */
1159  for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
1160 
1161  /* Check if reset is complete */
1162  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1163  if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
1164  return 0;
1165 
1166  /* Delay */
1167  mdelay ( 1 );
1168  }
1169 
1170  DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name );
1171  return -ETIMEDOUT;
1172 }
1173 
1174 /**
1175  * Mark xHCI device as permanently failed
1176  *
1177  * @v xhci xHCI device
1178  * @ret rc Return status code
1179  */
1180 static int xhci_fail ( struct xhci_device *xhci ) {
1181  size_t len;
1182  int rc;
1183 
1184  /* Mark command mechanism as permanently failed */
1185  xhci->failed = 1;
1186 
1187  /* Reset device */
1188  if ( ( rc = xhci_reset ( xhci ) ) != 0 )
1189  return rc;
1190 
1191  /* Discard DCBAA entries since DCBAAP has been cleared */
1192  assert ( xhci->dcbaa.context != NULL );
1193  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
1194  memset ( xhci->dcbaa.context, 0, len );
1195 
1196  return 0;
1197 }
1198 
1199 /******************************************************************************
1200  *
1201  * Transfer request blocks
1202  *
1203  ******************************************************************************
1204  */
1205 
1206 /**
1207  * Allocate transfer request block ring
1208  *
1209  * @v xhci xHCI device
1210  * @v ring TRB ring
1211  * @v shift Ring size (log2)
1212  * @v slot Device slot
1213  * @v target Doorbell target
1214  * @v stream Doorbell stream ID
1215  * @ret rc Return status code
1216  */
1217 static int xhci_ring_alloc ( struct xhci_device *xhci,
1218  struct xhci_trb_ring *ring,
1219  unsigned int shift, unsigned int slot,
1220  unsigned int target, unsigned int stream ) {
1221  struct xhci_trb_link *link;
1222  unsigned int count;
1223  int rc;
1224 
1225  /* Sanity check */
1226  assert ( shift > 0 );
1227 
1228  /* Initialise structure */
1229  memset ( ring, 0, sizeof ( *ring ) );
1230  ring->shift = shift;
1231  count = ( 1U << shift );
1232  ring->mask = ( count - 1 );
1233  ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
1234  ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
1235  ring->dbval = XHCI_DBVAL ( target, stream );
1236 
1237  /* Allocate I/O buffers */
1238  ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
1239  if ( ! ring->iobuf ) {
1240  rc = -ENOMEM;
1241  goto err_alloc_iobuf;
1242  }
1243 
1244  /* Allocate TRBs */
1245  ring->trb = dma_alloc ( xhci->dma, &ring->map, ring->len,
1246  xhci_align ( ring->len ) );
1247  if ( ! ring->trb ) {
1248  rc = -ENOMEM;
1249  goto err_alloc_trb;
1250  }
1251  memset ( ring->trb, 0, ring->len );
1252 
1253  /* Initialise Link TRB */
1254  link = &ring->trb[count].link;
1255  link->next = cpu_to_le64 ( dma ( &ring->map, ring->trb ) );
1256  link->flags = XHCI_TRB_TC;
1257  link->type = XHCI_TRB_LINK;
1258  ring->link = link;
1259 
1260  return 0;
1261 
1262  dma_free ( &ring->map, ring->trb, ring->len );
1263  err_alloc_trb:
1264  free ( ring->iobuf );
1265  err_alloc_iobuf:
1266  return rc;
1267 }
1268 
1269 /**
1270  * Reset transfer request block ring
1271  *
1272  * @v ring TRB ring
1273  */
1274 static void xhci_ring_reset ( struct xhci_trb_ring *ring ) {
1275  unsigned int count = ( 1U << ring->shift );
1276 
1277  /* Reset producer and consumer counters */
1278  ring->prod = 0;
1279  ring->cons = 0;
1280 
1281  /* Reset TRBs (except Link TRB) */
1282  memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) );
1283 }
1284 
1285 /**
1286  * Free transfer request block ring
1287  *
1288  * @v ring TRB ring
1289  */
1290 static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
1291  unsigned int count = ( 1U << ring->shift );
1292  unsigned int i;
1293 
1294  /* Sanity checks */
1295  assert ( ring->cons == ring->prod );
1296  for ( i = 0 ; i < count ; i++ )
1297  assert ( ring->iobuf[i] == NULL );
1298 
1299  /* Free TRBs */
1300  dma_free ( &ring->map, ring->trb, ring->len );
1301 
1302  /* Free I/O buffers */
1303  free ( ring->iobuf );
1304 }
1305 
1306 /**
1307  * Enqueue a transfer request block
1308  *
1309  * @v ring TRB ring
1310  * @v iobuf I/O buffer (if any)
1311  * @v trb Transfer request block (with empty Cycle flag)
1312  * @ret rc Return status code
1313  *
1314  * This operation does not implicitly ring the doorbell register.
1315  */
1316 static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
1317  const union xhci_trb *trb ) {
1318  union xhci_trb *dest;
1319  unsigned int prod;
1320  unsigned int mask;
1321  unsigned int index;
1322  unsigned int cycle;
1323 
1324  /* Sanity check */
1325  assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
1326 
1327  /* Fail if ring is full */
1328  if ( ! xhci_ring_remaining ( ring ) )
1329  return -ENOBUFS;
1330 
1331  /* Update producer counter (and link TRB, if applicable) */
1332  prod = ring->prod++;
1333  mask = ring->mask;
1334  cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
1335  index = ( prod & mask );
1336  if ( index == 0 )
1337  ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
1338 
1339  /* Record I/O buffer */
1340  ring->iobuf[index] = iobuf;
1341 
1342  /* Enqueue TRB */
1343  dest = &ring->trb[index];
1344  dest->template.parameter = trb->template.parameter;
1345  dest->template.status = trb->template.status;
1346  wmb();
1347  dest->template.control = ( trb->template.control |
1348  cpu_to_le32 ( cycle ) );
1349 
1350  return 0;
1351 }
1352 
1353 /**
1354  * Dequeue a transfer request block
1355  *
1356  * @v ring TRB ring
1357  * @ret iobuf I/O buffer
1358  */
1359 static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
1360  struct io_buffer *iobuf;
1361  unsigned int cons;
1362  unsigned int mask;
1363  unsigned int index;
1364 
1365  /* Sanity check */
1366  assert ( xhci_ring_fill ( ring ) != 0 );
1367 
1368  /* Update consumer counter */
1369  cons = ring->cons++;
1370  mask = ring->mask;
1371  index = ( cons & mask );
1372 
1373  /* Retrieve I/O buffer */
1374  iobuf = ring->iobuf[index];
1375  ring->iobuf[index] = NULL;
1376 
1377  return iobuf;
1378 }
1379 
1380 /**
1381  * Enqueue multiple transfer request blocks
1382  *
1383  * @v ring TRB ring
1384  * @v iobuf I/O buffer
1385  * @v trbs Transfer request blocks (with empty Cycle flag)
1386  * @v count Number of transfer request blocks
1387  * @ret rc Return status code
1388  *
1389  * This operation does not implicitly ring the doorbell register.
1390  */
1391 static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
1392  struct io_buffer *iobuf,
1393  const union xhci_trb *trbs,
1394  unsigned int count ) {
1395  const union xhci_trb *trb = trbs;
1396  int rc;
1397 
1398  /* Sanity check */
1399  assert ( iobuf != NULL );
1400 
1401  /* Fail if ring does not have sufficient space */
1402  if ( xhci_ring_remaining ( ring ) < count )
1403  return -ENOBUFS;
1404 
1405  /* Enqueue each TRB, recording the I/O buffer with the final TRB */
1406  while ( count-- ) {
1407  rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
1408  assert ( rc == 0 ); /* Should never be able to fail */
1409  }
1410 
1411  return 0;
1412 }
1413 
1414 /**
1415  * Dequeue multiple transfer request blocks
1416  *
1417  * @v ring TRB ring
1418  * @ret iobuf I/O buffer
1419  */
1420 static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
1421  struct io_buffer *iobuf;
1422 
1423  /* Dequeue TRBs until we reach the final TRB for an I/O buffer */
1424  do {
1425  iobuf = xhci_dequeue ( ring );
1426  } while ( iobuf == NULL );
1427 
1428  return iobuf;
1429 }
1430 
1431 /**
1432  * Ring doorbell register
1433  *
1434  * @v ring TRB ring
1435  */
1436 static inline __attribute__ (( always_inline )) void
1437 xhci_doorbell ( struct xhci_trb_ring *ring ) {
1438 
1439  wmb();
1440  writel ( ring->dbval, ring->db );
1441 }
1442 
1443 /******************************************************************************
1444  *
1445  * Command and event rings
1446  *
1447  ******************************************************************************
1448  */
1449 
1450 /**
1451  * Allocate command ring
1452  *
1453  * @v xhci xHCI device
1454  * @ret rc Return status code
1455  */
1456 static int xhci_command_alloc ( struct xhci_device *xhci ) {
1457  physaddr_t crp;
1458  int rc;
1459 
1460  /* Allocate TRB ring */
1461  if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
1462  0, 0, 0 ) ) != 0 )
1463  goto err_ring_alloc;
1464 
1465  /* Program command ring control register */
1466  crp = dma ( &xhci->command.map, xhci->command.trb );
1467  if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
1468  xhci->op + XHCI_OP_CRCR ) ) != 0 )
1469  goto err_writeq;
1470 
1471  DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n", xhci->name,
1472  virt_to_phys ( xhci->command.trb ),
1473  ( virt_to_phys ( xhci->command.trb ) + xhci->command.len ) );
1474  return 0;
1475 
1476  err_writeq:
1477  xhci_ring_free ( &xhci->command );
1478  err_ring_alloc:
1479  return rc;
1480 }
1481 
1482 /**
1483  * Free command ring
1484  *
1485  * @v xhci xHCI device
1486  */
1487 static void xhci_command_free ( struct xhci_device *xhci ) {
1488 
1489  /* Sanity check */
1490  assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
1491 
1492  /* Clear command ring control register */
1493  xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
1494 
1495  /* Free TRB ring */
1496  xhci_ring_free ( &xhci->command );
1497 }
1498 
1499 /**
1500  * Allocate event ring
1501  *
1502  * @v xhci xHCI device
1503  * @ret rc Return status code
1504  */
1505 static int xhci_event_alloc ( struct xhci_device *xhci ) {
1506  struct xhci_event_ring *event = &xhci->event;
1507  unsigned int count;
1508  size_t len;
1509  int rc;
1510 
1511  /* Allocate event ring */
1512  count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
1513  len = ( count * sizeof ( event->trb[0] ) );
1514  event->trb = dma_alloc ( xhci->dma, &event->trb_map, len,
1515  xhci_align ( len ) );
1516  if ( ! event->trb ) {
1517  rc = -ENOMEM;
1518  goto err_alloc_trb;
1519  }
1520  memset ( event->trb, 0, len );
1521 
1522  /* Allocate event ring segment table */
1523  event->segment = dma_alloc ( xhci->dma, &event->segment_map,
1524  sizeof ( event->segment[0] ),
1525  xhci_align ( sizeof (event->segment[0])));
1526  if ( ! event->segment ) {
1527  rc = -ENOMEM;
1528  goto err_alloc_segment;
1529  }
1530  memset ( event->segment, 0, sizeof ( event->segment[0] ) );
1531  event->segment[0].base = cpu_to_le64 ( dma ( &event->trb_map,
1532  event->trb ) );
1533  event->segment[0].count = cpu_to_le32 ( count );
1534 
1535  /* Program event ring registers */
1536  writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
1537  if ( ( rc = xhci_writeq ( xhci, dma ( &event->trb_map, event->trb ),
1538  xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
1539  goto err_writeq_erdp;
1540  if ( ( rc = xhci_writeq ( xhci,
1541  dma ( &event->segment_map, event->segment ),
1542  xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
1543  goto err_writeq_erstba;
1544 
1545  DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
1546  xhci->name, virt_to_phys ( event->trb ),
1547  ( virt_to_phys ( event->trb ) + len ),
1548  virt_to_phys ( event->segment ),
1549  ( virt_to_phys ( event->segment ) +
1550  sizeof ( event->segment[0] ) ) );
1551  return 0;
1552 
1553  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
1554  err_writeq_erstba:
1555  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
1556  err_writeq_erdp:
1557  dma_free ( &event->segment_map, event->segment,
1558  sizeof ( event->segment[0] ) );
1559  err_alloc_segment:
1560  dma_free ( &event->trb_map, event->trb, len );
1561  err_alloc_trb:
1562  return rc;
1563 }
1564 
1565 /**
1566  * Free event ring
1567  *
1568  * @v xhci xHCI device
1569  */
1570 static void xhci_event_free ( struct xhci_device *xhci ) {
1571  struct xhci_event_ring *event = &xhci->event;
1572  unsigned int count;
1573  size_t len;
1574 
1575  /* Clear event ring registers */
1576  writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
1577  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
1578  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
1579 
1580  /* Free event ring segment table */
1581  dma_free ( &event->segment_map, event->segment,
1582  sizeof ( event->segment[0] ) );
1583 
1584  /* Free event ring */
1585  count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
1586  len = ( count * sizeof ( event->trb[0] ) );
1587  dma_free ( &event->trb_map, event->trb, len );
1588 }
1589 
1590 /**
1591  * Handle transfer event
1592  *
1593  * @v xhci xHCI device
1594  * @v trb Transfer event TRB
1595  */
1596 static void xhci_transfer ( struct xhci_device *xhci,
1597  struct xhci_trb_transfer *trb ) {
1598  struct xhci_slot *slot;
1599  struct xhci_endpoint *endpoint;
1600  struct io_buffer *iobuf;
1601  int rc;
1602 
1603  /* Profile transfer events */
1604  profile_start ( &xhci_transfer_profiler );
1605 
1606  /* Identify slot */
1607  if ( ( trb->slot > xhci->slots ) ||
1608  ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) {
1609  DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n",
1610  xhci->name, trb->slot );
1611  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1612  return;
1613  }
1614 
1615  /* Identify endpoint */
1616  if ( ( trb->endpoint >= XHCI_CTX_END ) ||
1617  ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) {
1618  DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid "
1619  "%d:\n", xhci->name, slot->id, trb->endpoint );
1620  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1621  return;
1622  }
1623 
1624  /* Dequeue TRB(s) */
1625  iobuf = xhci_dequeue_multi ( &endpoint->ring );
1626  assert ( iobuf != NULL );
1627 
1628  /* Unmap I/O buffer */
1629  iob_unmap ( iobuf );
1630 
1631  /* Check for errors */
1632  if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
1633  ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
1634 
1635  /* Construct error */
1636  rc = -ECODE ( trb->code );
1637  DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n",
1638  xhci->name, slot->id, endpoint->ctx, trb->code,
1639  strerror ( rc ) );
1640  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1641 
1642  /* Sanity check */
1643  assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
1644  != XHCI_ENDPOINT_RUNNING );
1645 
1646  /* Report failure to USB core */
1647  usb_complete_err ( endpoint->ep, iobuf, rc );
1648  return;
1649  }
1650 
1651  /* Record actual transfer size */
1652  iob_unput ( iobuf, le16_to_cpu ( trb->residual ) );
1653 
1654  /* Sanity check (for successful completions only) */
1655  assert ( xhci_ring_consumed ( &endpoint->ring ) ==
1656  le64_to_cpu ( trb->transfer ) );
1657 
1658  /* Report completion to USB core */
1659  usb_complete ( endpoint->ep, iobuf );
1660  profile_stop ( &xhci_transfer_profiler );
1661 }
1662 
1663 /**
1664  * Handle command completion event
1665  *
1666  * @v xhci xHCI device
1667  * @v trb Command completion event
1668  */
1669 static void xhci_complete ( struct xhci_device *xhci,
1670  struct xhci_trb_complete *trb ) {
1671  int rc;
1672 
1673  /* Ignore "command ring stopped" notifications */
1674  if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) {
1675  DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name );
1676  return;
1677  }
1678 
1679  /* Ignore unexpected completions */
1680  if ( ! xhci->pending ) {
1681  rc = -ECODE ( trb->code );
1682  DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n",
1683  xhci->name, trb->code, strerror ( rc ) );
1684  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1685  return;
1686  }
1687 
1688  /* Dequeue command TRB */
1689  xhci_dequeue ( &xhci->command );
1690 
1691  /* Sanity check */
1692  assert ( xhci_ring_consumed ( &xhci->command ) ==
1693  le64_to_cpu ( trb->command ) );
1694 
1695  /* Record completion */
1696  memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) );
1697  xhci->pending = NULL;
1698 }
1699 
1700 /**
1701  * Handle port status event
1702  *
1703  * @v xhci xHCI device
1704  * @v trb Port status event
1705  */
1706 static void xhci_port_status ( struct xhci_device *xhci,
1707  struct xhci_trb_port_status *trb ) {
1708  struct usb_port *port = usb_port ( xhci->bus->hub, trb->port );
1709  uint32_t portsc;
1710 
1711  /* Sanity check */
1712  assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) );
1713 
1714  /* Record disconnections and clear changes */
1715  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) );
1716  port->disconnected |= ( portsc & XHCI_PORTSC_CSC );
1717  portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
1718  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) );
1719 
1720  /* Report port status change */
1721  usb_port_changed ( port );
1722 }
1723 
1724 /**
1725  * Handle host controller event
1726  *
1727  * @v xhci xHCI device
1728  * @v trb Host controller event
1729  */
1730 static void xhci_host_controller ( struct xhci_device *xhci,
1731  struct xhci_trb_host_controller *trb ) {
1732  int rc;
1733 
1734  /* Construct error */
1735  rc = -ECODE ( trb->code );
1736  DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n",
1737  xhci->name, trb->code, strerror ( rc ) );
1738 }
1739 
1740 /**
1741  * Poll event ring
1742  *
1743  * @v xhci xHCI device
1744  */
1745 static void xhci_event_poll ( struct xhci_device *xhci ) {
1746  struct xhci_event_ring *event = &xhci->event;
1747  union xhci_trb *trb;
1748  unsigned int shift = XHCI_EVENT_TRBS_LOG2;
1749  unsigned int count = ( 1 << shift );
1750  unsigned int mask = ( count - 1 );
1751  unsigned int consumed;
1752  unsigned int type;
1753 
1754  /* Do nothing if device has permanently failed */
1755  if ( xhci->failed )
1756  return;
1757 
1758  /* Poll for events */
1759  profile_start ( &xhci_event_profiler );
1760  for ( consumed = 0 ; ; consumed++ ) {
1761 
1762  /* Stop if we reach an empty TRB */
1763  rmb();
1764  trb = &event->trb[ event->cons & mask ];
1765  if ( ! ( ( trb->common.flags ^
1766  ( event->cons >> shift ) ) & XHCI_TRB_C ) )
1767  break;
1768 
1769  /* Consume this TRB */
1770  event->cons++;
1771 
1772  /* Handle TRB */
1773  type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
1774  switch ( type ) {
1775 
1776  case XHCI_TRB_TRANSFER :
1777  xhci_transfer ( xhci, &trb->transfer );
1778  break;
1779 
1780  case XHCI_TRB_COMPLETE :
1781  xhci_complete ( xhci, &trb->complete );
1782  break;
1783 
1784  case XHCI_TRB_PORT_STATUS:
1785  xhci_port_status ( xhci, &trb->port );
1786  break;
1787 
1788  case XHCI_TRB_HOST_CONTROLLER:
1789  xhci_host_controller ( xhci, &trb->host );
1790  break;
1791 
1792  default:
1793  DBGC ( xhci, "XHCI %s unrecognised event %#x\n:",
1794  xhci->name, ( event->cons - 1 ) );
1795  DBGC_HDA ( xhci, virt_to_phys ( trb ),
1796  trb, sizeof ( *trb ) );
1797  break;
1798  }
1799  }
1800 
1801  /* Update dequeue pointer if applicable */
1802  if ( consumed ) {
1803  xhci_writeq ( xhci, dma ( &event->trb_map, trb ),
1804  xhci->run + XHCI_RUN_ERDP ( 0 ) );
1805  profile_stop ( &xhci_event_profiler );
1806  }
1807 }
1808 
1809 /**
1810  * Abort command
1811  *
1812  * @v xhci xHCI device
1813  */
1814 static void xhci_abort ( struct xhci_device *xhci ) {
1815  physaddr_t crp;
1816  uint32_t crcr;
1817 
1818  /* Abort the command */
1819  DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
1820  xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR );
1821 
1822  /* Allow time for command to abort */
1823  mdelay ( XHCI_COMMAND_ABORT_DELAY_MS );
1824 
1825  /* Check for failure to abort */
1826  crcr = readl ( xhci->op + XHCI_OP_CRCR );
1827  if ( crcr & XHCI_CRCR_CRR ) {
1828 
1829  /* Device has failed to abort a command and is almost
1830  * certainly beyond repair. Reset device, abandoning
1831  * all state, and mark device as failed to avoid
1832  * delays on any future command attempts.
1833  */
1834  DBGC ( xhci, "XHCI %s failed to abort command\n", xhci->name );
1835  xhci_fail ( xhci );
1836  }
1837 
1838  /* Consume (and ignore) any final command status */
1839  xhci_event_poll ( xhci );
1840 
1841  /* Reset the command ring control register */
1842  xhci_ring_reset ( &xhci->command );
1843  crp = dma ( &xhci->command.map, xhci->command.trb );
1844  xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
1845 }
1846 
1847 /**
1848  * Issue command and wait for completion
1849  *
1850  * @v xhci xHCI device
1851  * @v trb Transfer request block (with empty Cycle flag)
1852  * @ret rc Return status code
1853  *
1854  * On a successful completion, the TRB will be overwritten with the
1855  * completion.
1856  */
1857 static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
1858  struct xhci_trb_complete *complete = &trb->complete;
1859  unsigned int i;
1860  int rc;
1861 
1862  /* Immediately fail all commands if command mechanism has failed */
1863  if ( xhci->failed ) {
1864  rc = -EPIPE;
1865  goto err_failed;
1866  }
1867 
1868  /* Sanity check */
1869  if ( xhci->pending ) {
1870  DBGC ( xhci, "XHCI %s command ring busy\n", xhci->name );
1871  rc = -EBUSY;
1872  goto err_pending;
1873  }
1874 
1875  /* Record the pending command */
1876  xhci->pending = trb;
1877 
1878  /* Enqueue the command */
1879  if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
1880  goto err_enqueue;
1881 
1882  /* Ring the command doorbell */
1883  xhci_doorbell ( &xhci->command );
1884 
1885  /* Wait for the command to complete */
1886  for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
1887 
1888  /* Poll event ring */
1889  xhci_event_poll ( xhci );
1890 
1891  /* Check for completion */
1892  if ( ! xhci->pending ) {
1893  if ( complete->code != XHCI_CMPLT_SUCCESS ) {
1894  rc = -ECODE ( complete->code );
1895  DBGC ( xhci, "XHCI %s command failed (code "
1896  "%d): %s\n", xhci->name, complete->code,
1897  strerror ( rc ) );
1898  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1899  return rc;
1900  }
1901  return 0;
1902  }
1903 
1904  /* Delay */
1905  mdelay ( 1 );
1906  }
1907 
1908  /* Timeout */
1909  DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name );
1910  rc = -ETIMEDOUT;
1911 
1912  /* Abort command */
1913  xhci_abort ( xhci );
1914 
1915  err_enqueue:
1916  xhci->pending = NULL;
1917  err_pending:
1918  err_failed:
1919  return rc;
1920 }
1921 
1922 /**
1923  * Issue NOP and wait for completion
1924  *
1925  * @v xhci xHCI device
1926  * @ret rc Return status code
1927  */
1928 static inline int xhci_nop ( struct xhci_device *xhci ) {
1929  union xhci_trb trb;
1930  struct xhci_trb_common *nop = &trb.common;
1931  int rc;
1932 
1933  /* Construct command */
1934  memset ( nop, 0, sizeof ( *nop ) );
1935  nop->flags = XHCI_TRB_IOC;
1936  nop->type = XHCI_TRB_NOP_CMD;
1937 
1938  /* Issue command and wait for completion */
1939  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1940  DBGC ( xhci, "XHCI %s NOP failed: %s\n",
1941  xhci->name, strerror ( rc ) );
1942  return rc;
1943  }
1944 
1945  DBGC2 ( xhci, "XHCI %s NOP completed successfully\n", xhci->name );
1946  return 0;
1947 }
1948 
1949 /**
1950  * Enable slot
1951  *
1952  * @v xhci xHCI device
1953  * @v type Slot type
1954  * @ret slot Device slot ID, or negative error
1955  */
1956 static inline int xhci_enable_slot ( struct xhci_device *xhci,
1957  unsigned int type ) {
1958  union xhci_trb trb;
1959  struct xhci_trb_enable_slot *enable = &trb.enable;
1960  struct xhci_trb_complete *enabled = &trb.complete;
1961  unsigned int slot;
1962  int rc;
1963 
1964  /* Construct command */
1965  memset ( enable, 0, sizeof ( *enable ) );
1966  enable->slot = type;
1967  enable->type = XHCI_TRB_ENABLE_SLOT;
1968 
1969  /* Issue command and wait for completion */
1970  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1971  DBGC ( xhci, "XHCI %s could not enable new slot: %s\n",
1972  xhci->name, strerror ( rc ) );
1973  return rc;
1974  }
1975 
1976  /* Extract slot number */
1977  slot = enabled->slot;
1978 
1979  DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot );
1980  return slot;
1981 }
1982 
1983 /**
1984  * Disable slot
1985  *
1986  * @v xhci xHCI device
1987  * @v slot Device slot
1988  * @ret rc Return status code
1989  */
1990 static inline int xhci_disable_slot ( struct xhci_device *xhci,
1991  unsigned int slot ) {
1992  union xhci_trb trb;
1993  struct xhci_trb_disable_slot *disable = &trb.disable;
1994  int rc;
1995 
1996  /* Construct command */
1997  memset ( disable, 0, sizeof ( *disable ) );
1998  disable->type = XHCI_TRB_DISABLE_SLOT;
1999  disable->slot = slot;
2000 
2001  /* Issue command and wait for completion */
2002  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2003  DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n",
2004  xhci->name, slot, strerror ( rc ) );
2005  return rc;
2006  }
2007 
2008  DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot );
2009  return 0;
2010 }
2011 
2012 /**
2013  * Issue context-based command and wait for completion
2014  *
2015  * @v xhci xHCI device
2016  * @v slot Device slot
2017  * @v endpoint Endpoint
2018  * @v type TRB type
2019  * @v populate Input context populater
2020  * @ret rc Return status code
2021  */
2022 static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
2023  struct xhci_endpoint *endpoint, unsigned int type,
2024  void ( * populate ) ( struct xhci_device *xhci,
2025  struct xhci_slot *slot,
2026  struct xhci_endpoint *endpoint,
2027  void *input ) ) {
2028  union xhci_trb trb;
2029  struct xhci_trb_context *context = &trb.context;
2030  struct dma_mapping map;
2031  size_t len;
2032  void *input;
2033  int rc;
2034 
2035  /* Allocate an input context */
2036  memset ( &map, 0, sizeof ( map ) );
2037  len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
2038  input = dma_alloc ( xhci->dma, &map, len, xhci_align ( len ) );
2039  if ( ! input ) {
2040  rc = -ENOMEM;
2041  goto err_alloc;
2042  }
2043  memset ( input, 0, len );
2044 
2045  /* Populate input context */
2046  populate ( xhci, slot, endpoint, input );
2047 
2048  /* Construct command */
2049  memset ( context, 0, sizeof ( *context ) );
2050  context->type = type;
2051  context->input = cpu_to_le64 ( dma ( &map, input ) );
2052  context->slot = slot->id;
2053 
2054  /* Issue command and wait for completion */
2055  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
2056  goto err_command;
2057 
2058  err_command:
2059  dma_free ( &map, input, len );
2060  err_alloc:
2061  return rc;
2062 }
2063 
2064 /**
2065  * Populate address device input context
2066  *
2067  * @v xhci xHCI device
2068  * @v slot Device slot
2069  * @v endpoint Endpoint
2070  * @v input Input context
2071  */
2072 static void xhci_address_device_input ( struct xhci_device *xhci,
2073  struct xhci_slot *slot,
2074  struct xhci_endpoint *endpoint,
2075  void *input ) {
2076  struct xhci_trb_ring *ring = &endpoint->ring;
2077  struct xhci_control_context *control_ctx;
2078  struct xhci_slot_context *slot_ctx;
2079  struct xhci_endpoint_context *ep_ctx;
2080 
2081  /* Sanity checks */
2082  assert ( endpoint->ctx == XHCI_CTX_EP0 );
2083 
2084  /* Populate control context */
2085  control_ctx = input;
2086  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2087  ( 1 << XHCI_CTX_EP0 ) );
2088 
2089  /* Populate slot context */
2090  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2091  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
2092  slot->route ) );
2093  slot_ctx->port = slot->port;
2094  slot_ctx->tt_id = slot->tt_id;
2095  slot_ctx->tt_port = slot->tt_port;
2096 
2097  /* Populate control endpoint context */
2098  ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
2099  ep_ctx->type = XHCI_EP_TYPE_CONTROL;
2100  ep_ctx->burst = endpoint->ep->burst;
2101  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2102  ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
2103  XHCI_EP_DCS );
2104  ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
2105 }
2106 
2107 /**
2108  * Address device
2109  *
2110  * @v xhci xHCI device
2111  * @v slot Device slot
2112  * @ret rc Return status code
2113  */
2114 static inline int xhci_address_device ( struct xhci_device *xhci,
2115  struct xhci_slot *slot ) {
2116  struct usb_device *usb = slot->usb;
2117  struct xhci_slot_context *slot_ctx;
2118  int rc;
2119 
2120  /* Assign device address */
2121  if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
2122  XHCI_TRB_ADDRESS_DEVICE,
2123  xhci_address_device_input ) ) != 0 ) {
2124  DBGC ( xhci, "XHCI %s slot %d could not assign address: %s\n",
2125  xhci->name, slot->id, strerror ( rc ) );
2126  return rc;
2127  }
2128 
2129  /* Get assigned address */
2130  slot_ctx = ( slot->context +
2131  xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
2132  usb->address = slot_ctx->address;
2133  DBGC2 ( xhci, "XHCI %s slot %d assigned address %d to %s\n",
2134  xhci->name, slot->id, usb->address, usb->name );
2135 
2136  return 0;
2137 }
2138 
2139 /**
2140  * Populate configure endpoint input context
2141  *
2142  * @v xhci xHCI device
2143  * @v slot Device slot
2144  * @v endpoint Endpoint
2145  * @v input Input context
2146  */
2147 static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
2148  struct xhci_slot *slot,
2149  struct xhci_endpoint *endpoint,
2150  void *input ) {
2151  struct xhci_trb_ring *ring = &endpoint->ring;
2152  struct xhci_control_context *control_ctx;
2153  struct xhci_slot_context *slot_ctx;
2154  struct xhci_endpoint_context *ep_ctx;
2155 
2156  /* Populate control context */
2157  control_ctx = input;
2158  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2159  ( 1 << endpoint->ctx ) );
2160 
2161  /* Populate slot context */
2162  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2163  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2164  ( slot->ports ? 1 : 0 ),
2165  slot->psiv, 0 ) );
2166  slot_ctx->ports = slot->ports;
2167 
2168  /* Populate endpoint context */
2169  ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
2170  ep_ctx->interval = endpoint->interval;
2171  ep_ctx->type = endpoint->type;
2172  ep_ctx->burst = endpoint->ep->burst;
2173  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2174  ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
2175  XHCI_EP_DCS );
2176  ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
2177 }
2178 
2179 /**
2180  * Configure endpoint
2181  *
2182  * @v xhci xHCI device
2183  * @v slot Device slot
2184  * @v endpoint Endpoint
2185  * @ret rc Return status code
2186  */
2187 static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
2188  struct xhci_slot *slot,
2189  struct xhci_endpoint *endpoint ) {
2190  int rc;
2191 
2192  /* Configure endpoint */
2193  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2194  XHCI_TRB_CONFIGURE_ENDPOINT,
2195  xhci_configure_endpoint_input ) ) != 0 ) {
2196  DBGC ( xhci, "XHCI %s slot %d ctx %d could not configure: %s\n",
2197  xhci->name, slot->id, endpoint->ctx, strerror ( rc ) );
2198  return rc;
2199  }
2200 
2201  DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
2202  xhci->name, slot->id, endpoint->ctx );
2203  return 0;
2204 }
2205 
2206 /**
2207  * Populate deconfigure endpoint input context
2208  *
2209  * @v xhci xHCI device
2210  * @v slot Device slot
2211  * @v endpoint Endpoint
2212  * @v input Input context
2213  */
2214 static void
2215 xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
2216  struct xhci_slot *slot __unused,
2217  struct xhci_endpoint *endpoint,
2218  void *input ) {
2219  struct xhci_control_context *control_ctx;
2220  struct xhci_slot_context *slot_ctx;
2221 
2222  /* Populate control context */
2223  control_ctx = input;
2224  control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
2225  control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
2226 
2227  /* Populate slot context */
2228  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2229  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2230  0, 0, 0 ) );
2231 }
2232 
2233 /**
2234  * Deconfigure endpoint
2235  *
2236  * @v xhci xHCI device
2237  * @v slot Device slot
2238  * @v endpoint Endpoint
2239  * @ret rc Return status code
2240  */
2241 static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
2242  struct xhci_slot *slot,
2243  struct xhci_endpoint *endpoint ) {
2244  int rc;
2245 
2246  /* Deconfigure endpoint */
2247  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2248  XHCI_TRB_CONFIGURE_ENDPOINT,
2249  xhci_deconfigure_endpoint_input ) ) != 0 ) {
2250  DBGC ( xhci, "XHCI %s slot %d ctx %d could not deconfigure: "
2251  "%s\n", xhci->name, slot->id, endpoint->ctx,
2252  strerror ( rc ) );
2253  return rc;
2254  }
2255 
2256  DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
2257  xhci->name, slot->id, endpoint->ctx );
2258  return 0;
2259 }
2260 
2261 /**
2262  * Populate evaluate context input context
2263  *
2264  * @v xhci xHCI device
2265  * @v slot Device slot
2266  * @v endpoint Endpoint
2267  * @v input Input context
2268  */
2269 static void xhci_evaluate_context_input ( struct xhci_device *xhci,
2270  struct xhci_slot *slot __unused,
2271  struct xhci_endpoint *endpoint,
2272  void *input ) {
2273  struct xhci_control_context *control_ctx;
2274  struct xhci_slot_context *slot_ctx;
2275  struct xhci_endpoint_context *ep_ctx;
2276 
2277  /* Populate control context */
2278  control_ctx = input;
2279  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2280  ( 1 << endpoint->ctx ) );
2281 
2282  /* Populate slot context */
2283  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2284  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2285  0, 0, 0 ) );
2286 
2287  /* Populate endpoint context */
2288  ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
2289  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2290 }
2291 
2292 /**
2293  * Evaluate context
2294  *
2295  * @v xhci xHCI device
2296  * @v slot Device slot
2297  * @v endpoint Endpoint
2298  * @ret rc Return status code
2299  */
2300 static inline int xhci_evaluate_context ( struct xhci_device *xhci,
2301  struct xhci_slot *slot,
2302  struct xhci_endpoint *endpoint ) {
2303  int rc;
2304 
2305  /* Configure endpoint */
2306  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2307  XHCI_TRB_EVALUATE_CONTEXT,
2308  xhci_evaluate_context_input ) ) != 0 ) {
2309  DBGC ( xhci, "XHCI %s slot %d ctx %d could not (re-)evaluate: "
2310  "%s\n", xhci->name, slot->id, endpoint->ctx,
2311  strerror ( rc ) );
2312  return rc;
2313  }
2314 
2315  DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
2316  xhci->name, slot->id, endpoint->ctx );
2317  return 0;
2318 }
2319 
2320 /**
2321  * Reset endpoint
2322  *
2323  * @v xhci xHCI device
2324  * @v slot Device slot
2325  * @v endpoint Endpoint
2326  * @ret rc Return status code
2327  */
2328 static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
2329  struct xhci_slot *slot,
2330  struct xhci_endpoint *endpoint ) {
2331  union xhci_trb trb;
2332  struct xhci_trb_reset_endpoint *reset = &trb.reset;
2333  int rc;
2334 
2335  /* Construct command */
2336  memset ( reset, 0, sizeof ( *reset ) );
2337  reset->slot = slot->id;
2338  reset->endpoint = endpoint->ctx;
2339  reset->type = XHCI_TRB_RESET_ENDPOINT;
2340 
2341  /* Issue command and wait for completion */
2342  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2343  DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint "
2344  "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
2345  endpoint->context->state, strerror ( rc ) );
2346  return rc;
2347  }
2348 
2349  return 0;
2350 }
2351 
2352 /**
2353  * Stop endpoint
2354  *
2355  * @v xhci xHCI device
2356  * @v slot Device slot
2357  * @v endpoint Endpoint
2358  * @ret rc Return status code
2359  */
2360 static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
2361  struct xhci_slot *slot,
2362  struct xhci_endpoint *endpoint ) {
2363  union xhci_trb trb;
2364  struct xhci_trb_stop_endpoint *stop = &trb.stop;
2365  int rc;
2366 
2367  /* Construct command */
2368  memset ( stop, 0, sizeof ( *stop ) );
2369  stop->slot = slot->id;
2370  stop->endpoint = endpoint->ctx;
2371  stop->type = XHCI_TRB_STOP_ENDPOINT;
2372 
2373  /* Issue command and wait for completion */
2374  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2375  DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint "
2376  "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
2377  endpoint->context->state, strerror ( rc ) );
2378  return rc;
2379  }
2380 
2381  return 0;
2382 }
2383 
2384 /**
2385  * Set transfer ring dequeue pointer
2386  *
2387  * @v xhci xHCI device
2388  * @v slot Device slot
2389  * @v endpoint Endpoint
2390  * @ret rc Return status code
2391  */
2392 static inline int
2393 xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
2394  struct xhci_slot *slot,
2395  struct xhci_endpoint *endpoint ) {
2396  union xhci_trb trb;
2397  struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
2398  struct xhci_trb_ring *ring = &endpoint->ring;
2399  unsigned int cons;
2400  unsigned int mask;
2401  unsigned int index;
2402  unsigned int dcs;
2403  physaddr_t addr;
2404  int rc;
2405 
2406  /* Construct command */
2407  memset ( dequeue, 0, sizeof ( *dequeue ) );
2408  cons = ring->cons;
2409  mask = ring->mask;
2410  dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
2411  index = ( cons & mask );
2412  addr = dma ( &ring->map, &ring->trb[index] );
2413  dequeue->dequeue = cpu_to_le64 ( addr | dcs );
2414  dequeue->slot = slot->id;
2415  dequeue->endpoint = endpoint->ctx;
2416  dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
2417 
2418  /* Issue command and wait for completion */
2419  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2420  DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue "
2421  "pointer in state %d: %s\n", xhci->name, slot->id,
2422  endpoint->ctx, endpoint->context->state, strerror ( rc));
2423  return rc;
2424  }
2425 
2426  return 0;
2427 }
2428 
2429 /******************************************************************************
2430  *
2431  * Endpoint operations
2432  *
2433  ******************************************************************************
2434  */
2435 
2436 /**
2437  * Open endpoint
2438  *
2439  * @v ep USB endpoint
2440  * @ret rc Return status code
2441  */
2442 static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
2443  struct usb_device *usb = ep->usb;
2444  struct xhci_slot *slot = usb_get_hostdata ( usb );
2445  struct xhci_device *xhci = slot->xhci;
2446  struct xhci_endpoint *endpoint;
2447  unsigned int ctx;
2448  unsigned int type;
2449  unsigned int interval;
2450  int rc;
2451 
2452  /* Calculate context index */
2453  ctx = XHCI_CTX ( ep->address );
2454  assert ( slot->endpoint[ctx] == NULL );
2455 
2456  /* Calculate endpoint type */
2457  type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
2458  if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
2459  type = XHCI_EP_TYPE_CONTROL;
2460  if ( ep->address & USB_DIR_IN )
2461  type |= XHCI_EP_TYPE_IN;
2462 
2463  /* Calculate interval */
2464  if ( type & XHCI_EP_TYPE_PERIODIC ) {
2465  interval = ( fls ( ep->interval ) - 1 );
2466  } else {
2467  interval = ep->interval;
2468  }
2469 
2470  /* Allocate and initialise structure */
2471  endpoint = zalloc ( sizeof ( *endpoint ) );
2472  if ( ! endpoint ) {
2473  rc = -ENOMEM;
2474  goto err_alloc;
2475  }
2476  usb_endpoint_set_hostdata ( ep, endpoint );
2477  slot->endpoint[ctx] = endpoint;
2478  endpoint->xhci = xhci;
2479  endpoint->slot = slot;
2480  endpoint->ep = ep;
2481  endpoint->ctx = ctx;
2482  endpoint->type = type;
2483  endpoint->interval = interval;
2484  endpoint->context = ( ( ( void * ) slot->context ) +
2485  xhci_device_context_offset ( xhci, ctx ) );
2486 
2487  /* Allocate transfer ring */
2488  if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
2489  XHCI_TRANSFER_TRBS_LOG2,
2490  slot->id, ctx, 0 ) ) != 0 )
2491  goto err_ring_alloc;
2492 
2493  /* Configure endpoint, if applicable */
2494  if ( ( ctx != XHCI_CTX_EP0 ) &&
2495  ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
2496  goto err_configure_endpoint;
2497 
2498  DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n",
2499  xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
2500  ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
2501  return 0;
2502 
2503  xhci_deconfigure_endpoint ( xhci, slot, endpoint );
2504  err_configure_endpoint:
2505  xhci_ring_free ( &endpoint->ring );
2506  err_ring_alloc:
2507  slot->endpoint[ctx] = NULL;
2508  free ( endpoint );
2509  err_alloc:
2510  return rc;
2511 }
2512 
2513 /**
2514  * Close endpoint
2515  *
2516  * @v ep USB endpoint
2517  */
2518 static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
2519  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2520  struct xhci_slot *slot = endpoint->slot;
2521  struct xhci_device *xhci = slot->xhci;
2522  struct io_buffer *iobuf;
2523  unsigned int ctx = endpoint->ctx;
2524 
2525  /* Deconfigure endpoint, if applicable */
2526  if ( ctx != XHCI_CTX_EP0 )
2527  xhci_deconfigure_endpoint ( xhci, slot, endpoint );
2528 
2529  /* Cancel any incomplete transfers */
2530  while ( xhci_ring_fill ( &endpoint->ring ) ) {
2531  iobuf = xhci_dequeue_multi ( &endpoint->ring );
2532  iob_unmap ( iobuf );
2533  usb_complete_err ( ep, iobuf, -ECANCELED );
2534  }
2535 
2536  /* Free endpoint */
2537  xhci_ring_free ( &endpoint->ring );
2538  slot->endpoint[ctx] = NULL;
2539  free ( endpoint );
2540 }
2541 
2542 /**
2543  * Reset endpoint
2544  *
2545  * @v ep USB endpoint
2546  * @ret rc Return status code
2547  */
2548 static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
2549  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2550  struct xhci_slot *slot = endpoint->slot;
2551  struct xhci_device *xhci = slot->xhci;
2552  int rc;
2553 
2554  /* Reset endpoint context */
2555  if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
2556  return rc;
2557 
2558  /* Set transfer ring dequeue pointer */
2559  if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
2560  return rc;
2561 
2562  /* Ring doorbell to resume processing */
2563  xhci_doorbell ( &endpoint->ring );
2564 
2565  DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n",
2566  xhci->name, slot->id, endpoint->ctx );
2567  return 0;
2568 }
2569 
2570 /**
2571  * Update MTU
2572  *
2573  * @v ep USB endpoint
2574  * @ret rc Return status code
2575  */
2576 static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
2577  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2578  struct xhci_slot *slot = endpoint->slot;
2579  struct xhci_device *xhci = slot->xhci;
2580  int rc;
2581 
2582  /* Evalulate context */
2583  if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
2584  return rc;
2585 
2586  return 0;
2587 }
2588 
2589 /**
2590  * Enqueue message transfer
2591  *
2592  * @v ep USB endpoint
2593  * @v iobuf I/O buffer
2594  * @ret rc Return status code
2595  */
2596 static int xhci_endpoint_message ( struct usb_endpoint *ep,
2597  struct io_buffer *iobuf ) {
2598  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2599  struct xhci_device *xhci = endpoint->xhci;
2600  struct usb_setup_packet *packet;
2601  unsigned int input;
2602  size_t len;
2603  union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
2604  1 /* status */ ];
2605  union xhci_trb *trb = trbs;
2606  struct xhci_trb_setup *setup;
2607  struct xhci_trb_data *data;
2608  struct xhci_trb_status *status;
2609  int rc;
2610 
2611  /* Profile message transfers */
2612  profile_start ( &xhci_message_profiler );
2613 
2614  /* Construct setup stage TRB */
2615  memset ( trbs, 0, sizeof ( trbs ) );
2616  assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
2617  packet = iobuf->data;
2618  iob_pull ( iobuf, sizeof ( *packet ) );
2619  setup = &(trb++)->setup;
2620  memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
2621  setup->len = cpu_to_le32 ( sizeof ( *packet ) );
2622  setup->flags = XHCI_TRB_IDT;
2623  setup->type = XHCI_TRB_SETUP;
2624  len = iob_len ( iobuf );
2625  input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
2626  if ( len )
2627  setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
2628 
2629  /* Map I/O buffer */
2630  if ( ( rc = iob_map ( iobuf, xhci->dma, len,
2631  ( input ? DMA_RX : DMA_TX ) ) ) != 0 )
2632  goto err_map;
2633 
2634  /* Construct data stage TRB, if applicable */
2635  if ( len ) {
2636  data = &(trb++)->data;
2637  data->data = cpu_to_le64 ( iob_dma ( iobuf ) );
2638  data->len = cpu_to_le32 ( len );
2639  data->type = XHCI_TRB_DATA;
2640  data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
2641  }
2642 
2643  /* Construct status stage TRB */
2644  status = &(trb++)->status;
2645  status->flags = XHCI_TRB_IOC;
2646  status->type = XHCI_TRB_STATUS;
2647  status->direction =
2648  ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
2649 
2650  /* Enqueue TRBs */
2651  if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
2652  ( trb - trbs ) ) ) != 0 )
2653  goto err_enqueue;
2654 
2655  /* Ring the doorbell */
2656  xhci_doorbell ( &endpoint->ring );
2657 
2658  profile_stop ( &xhci_message_profiler );
2659  return 0;
2660 
2661  err_enqueue:
2662  iob_unmap ( iobuf );
2663  err_map:
2664  return rc;
2665 }
2666 
2667 /**
2668  * Calculate number of TRBs
2669  *
2670  * @v len Length of data
2671  * @v zlp Append a zero-length packet
2672  * @ret count Number of transfer descriptors
2673  */
2674 static unsigned int xhci_endpoint_count ( size_t len, int zlp ) {
2675  unsigned int count;
2676 
2677  /* Split into 64kB TRBs */
2678  count = ( ( len + XHCI_MTU - 1 ) / XHCI_MTU );
2679 
2680  /* Append a zero-length TRB if applicable */
2681  if ( zlp || ( count == 0 ) )
2682  count++;
2683 
2684  return count;
2685 }
2686 
2687 /**
2688  * Enqueue stream transfer
2689  *
2690  * @v ep USB endpoint
2691  * @v iobuf I/O buffer
2692  * @v zlp Append a zero-length packet
2693  * @ret rc Return status code
2694  */
2695 static int xhci_endpoint_stream ( struct usb_endpoint *ep,
2696  struct io_buffer *iobuf, int zlp ) {
2697  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2698  struct xhci_device *xhci = endpoint->xhci;
2699  size_t len = iob_len ( iobuf );
2700  unsigned int count = xhci_endpoint_count ( len, zlp );
2701  union xhci_trb trbs[count];
2702  union xhci_trb *trb = trbs;
2703  struct xhci_trb_normal *normal;
2704  physaddr_t data;
2705  unsigned int i;
2706  size_t trb_len;
2707  int rc;
2708 
2709  /* Profile stream transfers */
2710  profile_start ( &xhci_stream_profiler );
2711 
2712  /* Map I/O buffer */
2713  if ( ( rc = iob_map ( iobuf, xhci->dma, len,
2714  ( ( ep->address & USB_DIR_IN ) ?
2715  DMA_RX : DMA_TX ) ) ) != 0 )
2716  goto err_map;
2717  data = iob_dma ( iobuf );
2718 
2719  /* Construct normal TRBs */
2720  memset ( &trbs, 0, sizeof ( trbs ) );
2721  for ( i = 0 ; i < count ; i ++ ) {
2722 
2723  /* Calculate TRB length */
2724  trb_len = XHCI_MTU;
2725  if ( trb_len > len )
2726  trb_len = len;
2727 
2728  /* Construct normal TRB */
2729  normal = &trb->normal;
2730  normal->data = cpu_to_le64 ( data );
2731  normal->len = cpu_to_le32 ( trb_len );
2732  normal->type = XHCI_TRB_NORMAL;
2733  normal->flags = XHCI_TRB_CH;
2734 
2735  /* Move to next TRB */
2736  data += trb_len;
2737  len -= trb_len;
2738  trb++;
2739  }
2740 
2741  /* Mark zero-length packet (if present) as a separate transfer */
2742  if ( zlp && ( count > 1 ) )
2743  trb[-2].normal.flags = 0;
2744 
2745  /* Generate completion for final TRB */
2746  trb[-1].normal.flags = XHCI_TRB_IOC;
2747 
2748  /* Enqueue TRBs */
2749  if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
2750  count ) ) != 0 )
2751  goto err_enqueue;
2752 
2753  /* Ring the doorbell */
2754  xhci_doorbell ( &endpoint->ring );
2755 
2756  profile_stop ( &xhci_stream_profiler );
2757  return 0;
2758 
2759  err_enqueue:
2760  iob_unmap ( iobuf );
2761  err_map:
2762  return rc;
2763 }
2764 
2765 /******************************************************************************
2766  *
2767  * Device operations
2768  *
2769  ******************************************************************************
2770  */
2771 
2772 /**
2773  * Open device
2774  *
2775  * @v usb USB device
2776  * @ret rc Return status code
2777  */
2778 static int xhci_device_open ( struct usb_device *usb ) {
2779  struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
2780  struct usb_port *root_port = usb_root_hub_port ( usb );
2781  struct usb_port *tt = usb_transaction_translator ( usb );
2782  struct xhci_slot *slot;
2783  struct xhci_slot *tt_slot;
2784  size_t len;
2785  int type;
2786  int id;
2787  int rc;
2788 
2789  /* Determine applicable slot type */
2790  type = xhci_port_slot_type ( xhci, root_port->address );
2791  if ( type < 0 ) {
2792  rc = type;
2793  DBGC ( xhci, "XHCI %s-%d has no slot type\n",
2794  xhci->name, usb->port->address );
2795  goto err_type;
2796  }
2797 
2798  /* Allocate a device slot number */
2799  id = xhci_enable_slot ( xhci, type );
2800  if ( id < 0 ) {
2801  rc = id;
2802  goto err_enable_slot;
2803  }
2804  assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) );
2805  assert ( xhci->slot[id] == NULL );
2806 
2807  /* Allocate and initialise structure */
2808  slot = zalloc ( sizeof ( *slot ) );
2809  if ( ! slot ) {
2810  rc = -ENOMEM;
2811  goto err_alloc;
2812  }
2813  usb_set_hostdata ( usb, slot );
2814  xhci->slot[id] = slot;
2815  slot->xhci = xhci;
2816  slot->usb = usb;
2817  slot->id = id;
2818  if ( tt ) {
2819  tt_slot = usb_get_hostdata ( tt->hub->usb );
2820  slot->tt_id = tt_slot->id;
2821  slot->tt_port = tt->address;
2822  }
2823 
2824  /* Allocate a device context */
2825  len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
2826  slot->context = dma_alloc ( xhci->dma, &slot->map, len,
2827  xhci_align ( len ) );
2828  if ( ! slot->context ) {
2829  rc = -ENOMEM;
2830  goto err_alloc_context;
2831  }
2832  memset ( slot->context, 0, len );
2833 
2834  /* Set device context base address */
2835  assert ( xhci->dcbaa.context[id] == 0 );
2836  xhci->dcbaa.context[id] = cpu_to_le64 ( dma ( &slot->map,
2837  slot->context ) );
2838 
2839  DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
2840  xhci->name, slot->id, virt_to_phys ( slot->context ),
2841  ( virt_to_phys ( slot->context ) + len ), usb->name );
2842  return 0;
2843 
2844  xhci->dcbaa.context[id] = 0;
2845  dma_free ( &slot->map, slot->context, len );
2846  err_alloc_context:
2847  xhci->slot[id] = NULL;
2848  free ( slot );
2849  err_alloc:
2850  xhci_disable_slot ( xhci, id );
2851  err_enable_slot:
2852  err_type:
2853  return rc;
2854 }
2855 
2856 /**
2857  * Close device
2858  *
2859  * @v usb USB device
2860  */
2861 static void xhci_device_close ( struct usb_device *usb ) {
2862  struct xhci_slot *slot = usb_get_hostdata ( usb );
2863  struct xhci_device *xhci = slot->xhci;
2864  size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
2865  unsigned int id = slot->id;
2866  int rc;
2867 
2868  /* Disable slot */
2869  if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) {
2870  /* Slot is still enabled. Leak the slot context,
2871  * since the controller may still write to this
2872  * memory, and leave the DCBAA entry intact.
2873  *
2874  * If the controller later reports that this same slot
2875  * has been re-enabled, then some assertions will be
2876  * triggered.
2877  */
2878  DBGC ( xhci, "XHCI %s slot %d leaking context memory\n",
2879  xhci->name, slot->id );
2880  slot->context = NULL;
2881  }
2882 
2883  /* Free slot */
2884  if ( slot->context ) {
2885  dma_free ( &slot->map, slot->context, len );
2886  xhci->dcbaa.context[id] = 0;
2887  }
2888  xhci->slot[id] = NULL;
2889  free ( slot );
2890 }
2891 
2892 /**
2893  * Assign device address
2894  *
2895  * @v usb USB device
2896  * @ret rc Return status code
2897  */
2898 static int xhci_device_address ( struct usb_device *usb ) {
2899  struct xhci_slot *slot = usb_get_hostdata ( usb );
2900  struct xhci_device *xhci = slot->xhci;
2901  struct usb_port *root_port;
2902  int psiv;
2903  int rc;
2904 
2905  /* Calculate route string */
2906  slot->route = usb_route_string ( usb );
2907 
2908  /* Calculate root hub port number */
2909  root_port = usb_root_hub_port ( usb );
2910  slot->port = root_port->address;
2911 
2912  /* Calculate protocol speed ID */
2913  psiv = xhci_port_psiv ( xhci, slot->port, usb->speed );
2914  if ( psiv < 0 ) {
2915  rc = psiv;
2916  return rc;
2917  }
2918  slot->psiv = psiv;
2919 
2920  /* Address device */
2921  if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
2922  return rc;
2923 
2924  return 0;
2925 }
2926 
2927 /******************************************************************************
2928  *
2929  * Bus operations
2930  *
2931  ******************************************************************************
2932  */
2933 
2934 /**
2935  * Open USB bus
2936  *
2937  * @v bus USB bus
2938  * @ret rc Return status code
2939  */
2940 static int xhci_bus_open ( struct usb_bus *bus ) {
2941  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2942  int rc;
2943 
2944  /* Allocate device slot array */
2945  xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) );
2946  if ( ! xhci->slot ) {
2947  rc = -ENOMEM;
2948  goto err_slot_alloc;
2949  }
2950 
2951  /* Allocate device context base address array */
2952  if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
2953  goto err_dcbaa_alloc;
2954 
2955  /* Allocate scratchpad buffers */
2956  if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
2957  goto err_scratchpad_alloc;
2958 
2959  /* Allocate command ring */
2960  if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
2961  goto err_command_alloc;
2962 
2963  /* Allocate event ring */
2964  if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
2965  goto err_event_alloc;
2966 
2967  /* Start controller */
2968  xhci_run ( xhci );
2969 
2970  return 0;
2971 
2972  xhci_stop ( xhci );
2973  xhci_event_free ( xhci );
2974  err_event_alloc:
2975  xhci_command_free ( xhci );
2976  err_command_alloc:
2977  xhci_scratchpad_free ( xhci );
2978  err_scratchpad_alloc:
2979  xhci_dcbaa_free ( xhci );
2980  err_dcbaa_alloc:
2981  free ( xhci->slot );
2982  err_slot_alloc:
2983  return rc;
2984 }
2985 
2986 /**
2987  * Close USB bus
2988  *
2989  * @v bus USB bus
2990  */
2991 static void xhci_bus_close ( struct usb_bus *bus ) {
2992  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2993  unsigned int i;
2994 
2995  /* Sanity checks */
2996  assert ( xhci->slot != NULL );
2997  for ( i = 0 ; i <= xhci->slots ; i++ )
2998  assert ( xhci->slot[i] == NULL );
2999 
3000  xhci_stop ( xhci );
3001  xhci_event_free ( xhci );
3002  xhci_command_free ( xhci );
3003  xhci_scratchpad_free ( xhci );
3004  xhci_dcbaa_free ( xhci );
3005  free ( xhci->slot );
3006 }
3007 
3008 /**
3009  * Poll USB bus
3010  *
3011  * @v bus USB bus
3012  */
3013 static void xhci_bus_poll ( struct usb_bus *bus ) {
3014  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
3015 
3016  /* Poll event ring */
3017  xhci_event_poll ( xhci );
3018 }
3019 
3020 /******************************************************************************
3021  *
3022  * Hub operations
3023  *
3024  ******************************************************************************
3025  */
3026 
3027 /**
3028  * Open hub
3029  *
3030  * @v hub USB hub
3031  * @ret rc Return status code
3032  */
3033 static int xhci_hub_open ( struct usb_hub *hub ) {
3034  struct xhci_slot *slot;
3035 
3036  /* Do nothing if this is the root hub */
3037  if ( ! hub->usb )
3038  return 0;
3039 
3040  /* Get device slot */
3041  slot = usb_get_hostdata ( hub->usb );
3042 
3043  /* Update device slot hub parameters. We don't inform the
3044  * hardware of this information until the hub's interrupt
3045  * endpoint is opened, since the only mechanism for so doing
3046  * provided by the xHCI specification is a Configure Endpoint
3047  * command, and we can't issue that command until we have a
3048  * non-EP0 endpoint to configure.
3049  */
3050  slot->ports = hub->ports;
3051 
3052  return 0;
3053 }
3054 
3055 /**
3056  * Close hub
3057  *
3058  * @v hub USB hub
3059  */
3060 static void xhci_hub_close ( struct usb_hub *hub __unused ) {
3061 
3062  /* Nothing to do */
3063 }
3064 
3065 /******************************************************************************
3066  *
3067  * Root hub operations
3068  *
3069  ******************************************************************************
3070  */
3071 
3072 /**
3073  * Open root hub
3074  *
3075  * @v hub USB hub
3076  * @ret rc Return status code
3077  */
3078 static int xhci_root_open ( struct usb_hub *hub ) {
3079  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3080  struct usb_port *port;
3081  uint32_t portsc;
3082  unsigned int i;
3083 
3084  /* Enable power to all ports */
3085  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3086  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
3087  portsc &= XHCI_PORTSC_PRESERVE;
3088  portsc |= XHCI_PORTSC_PP;
3089  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
3090  }
3091 
3092  /* xHCI spec requires us to potentially wait 20ms after
3093  * enabling power to a port.
3094  */
3095  mdelay ( XHCI_PORT_POWER_DELAY_MS );
3096 
3097  /* USB3 ports may power up as Disabled */
3098  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3099  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
3100  port = usb_port ( hub, i );
3101  if ( ( port->protocol >= USB_PROTO_3_0 ) &&
3102  ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
3103  XHCI_PORTSC_PLS_DISABLED ) ) {
3104  /* Force link state to RxDetect */
3105  portsc &= XHCI_PORTSC_PRESERVE;
3106  portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
3107  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
3108  }
3109  }
3110 
3111  /* Some xHCI cards seem to require an additional delay after
3112  * setting the link state to RxDetect.
3113  */
3114  mdelay ( XHCI_LINK_STATE_DELAY_MS );
3115 
3116  return 0;
3117 }
3118 
3119 /**
3120  * Close root hub
3121  *
3122  * @v hub USB hub
3123  */
3124 static void xhci_root_close ( struct usb_hub *hub __unused ) {
3125 
3126  /* Nothing to do */
3127 }
3128 
3129 /**
3130  * Enable port
3131  *
3132  * @v hub USB hub
3133  * @v port USB port
3134  * @ret rc Return status code
3135  */
3136 static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
3137  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3138  uint32_t portsc;
3139  unsigned int i;
3140 
3141  /* Reset port */
3142  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3143  portsc &= XHCI_PORTSC_PRESERVE;
3144  portsc |= XHCI_PORTSC_PR;
3145  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3146 
3147  /* Wait for port to become enabled */
3148  for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
3149 
3150  /* Check port status */
3151  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3152  if ( portsc & XHCI_PORTSC_PED )
3153  return 0;
3154 
3155  /* Delay */
3156  mdelay ( 1 );
3157  }
3158 
3159  DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n",
3160  xhci->name, port->address );
3161  return -ETIMEDOUT;
3162 }
3163 
3164 /**
3165  * Disable port
3166  *
3167  * @v hub USB hub
3168  * @v port USB port
3169  * @ret rc Return status code
3170  */
3171 static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
3172  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3173  uint32_t portsc;
3174 
3175  /* Disable port */
3176  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3177  portsc &= XHCI_PORTSC_PRESERVE;
3178  portsc |= XHCI_PORTSC_PED;
3179  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3180 
3181  /* Allow time for link state to stabilise */
3182  mdelay ( XHCI_LINK_STATE_DELAY_MS );
3183 
3184  /* Set link state to RxDetect for USB3 ports */
3185  if ( port->protocol >= USB_PROTO_3_0 ) {
3186  portsc &= XHCI_PORTSC_PRESERVE;
3187  portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS );
3188  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3189  }
3190 
3191  /* Allow time for link state to stabilise */
3192  mdelay ( XHCI_LINK_STATE_DELAY_MS );
3193 
3194  return 0;
3195 }
3196 
3197 /**
3198  * Update root hub port speed
3199  *
3200  * @v hub USB hub
3201  * @v port USB port
3202  * @ret rc Return status code
3203  */
3204 static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
3205  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3206  uint32_t portsc;
3207  unsigned int psiv;
3208  int ccs;
3209  int ped;
3210  int csc;
3211  int speed;
3212  int rc;
3213 
3214  /* Read port status */
3215  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3216  DBGC2 ( xhci, "XHCI %s-%d status is %08x\n",
3217  xhci->name, port->address, portsc );
3218  ccs = ( portsc & XHCI_PORTSC_CCS );
3219  ped = ( portsc & XHCI_PORTSC_PED );
3220  csc = ( portsc & XHCI_PORTSC_CSC );
3221  psiv = XHCI_PORTSC_PSIV ( portsc );
3222 
3223  /* Record disconnections and clear changes */
3224  port->disconnected |= csc;
3225  portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
3226  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3227 
3228  /* Port speed is not valid unless port is connected */
3229  if ( ! ccs ) {
3230  port->speed = USB_SPEED_NONE;
3231  return 0;
3232  }
3233 
3234  /* For USB2 ports, the PSIV field is not valid until the port
3235  * completes reset and becomes enabled.
3236  */
3237  if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
3238  port->speed = USB_SPEED_FULL;
3239  return 0;
3240  }
3241 
3242  /* Get port speed and map to generic USB speed */
3243  speed = xhci_port_speed ( xhci, port->address, psiv );
3244  if ( speed < 0 ) {
3245  rc = speed;
3246  return rc;
3247  }
3248 
3249  port->speed = speed;
3250  return 0;
3251 }
3252 
3253 /**
3254  * Clear transaction translator buffer
3255  *
3256  * @v hub USB hub
3257  * @v port USB port
3258  * @v ep USB endpoint
3259  * @ret rc Return status code
3260  */
3261 static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
3262  struct usb_endpoint *ep ) {
3263  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3264 
3265  /* Should never be called; this is a root hub */
3266  DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name,
3267  port->address, ep->usb->name, usb_endpoint_name ( ep ) );
3268 
3269  return -ENOTSUP;
3270 }
3271 
3272 /******************************************************************************
3273  *
3274  * Hardware-independent interface
3275  *
3276  ******************************************************************************
3277  */
3278 
3279 /** USB host controller operations */
3280 static struct usb_host_operations xhci_operations = {
3281  .endpoint = {
3282  .open = xhci_endpoint_open,
3283  .close = xhci_endpoint_close,
3284  .reset = xhci_endpoint_reset,
3285  .mtu = xhci_endpoint_mtu,
3286  .message = xhci_endpoint_message,
3287  .stream = xhci_endpoint_stream,
3288  },
3289  .device = {
3290  .open = xhci_device_open,
3291  .close = xhci_device_close,
3292  .address = xhci_device_address,
3293  },
3294  .bus = {
3295  .open = xhci_bus_open,
3296  .close = xhci_bus_close,
3297  .poll = xhci_bus_poll,
3298  },
3299  .hub = {
3300  .open = xhci_hub_open,
3301  .close = xhci_hub_close,
3302  },
3303  .root = {
3304  .open = xhci_root_open,
3305  .close = xhci_root_close,
3306  .enable = xhci_root_enable,
3307  .disable = xhci_root_disable,
3308  .speed = xhci_root_speed,
3309  .clear_tt = xhci_root_clear_tt,
3310  },
3311 };
3312 
3313 /**
3314  * Register xHCI controller
3315  *
3316  * @v xhci xHCI device
3317  * @ret rc Return status code
3318  */
3319 int xhci_register ( struct xhci_device *xhci ) {
3320  struct usb_port *port;
3321  unsigned int i;
3322  int rc;
3323 
3324  /* Reset device */
3325  if ( ( rc = xhci_reset ( xhci ) ) != 0 )
3326  goto err_reset;
3327 
3328  /* Allocate USB bus */
3329  xhci->bus = alloc_usb_bus ( xhci->dev, xhci->ports, XHCI_MTU,
3330  &xhci_operations );
3331  if ( ! xhci->bus ) {
3332  rc = -ENOMEM;
3333  goto err_alloc_bus;
3334  }
3335  usb_bus_set_hostdata ( xhci->bus, xhci );
3336  usb_hub_set_drvdata ( xhci->bus->hub, xhci );
3337 
3338  /* Set port protocols */
3339  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3340  port = usb_port ( xhci->bus->hub, i );
3341  port->protocol = xhci_port_protocol ( xhci, i );
3342  }
3343 
3344  /* Register USB bus */
3345  if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
3346  goto err_register;
3347 
3348  return 0;
3349 
3350  unregister_usb_bus ( xhci->bus );
3351  err_register:
3352  free_usb_bus ( xhci->bus );
3353  err_alloc_bus:
3354  xhci_reset ( xhci );
3355  err_reset:
3356  return rc;
3357 }
3358 
3359 /**
3360  * Unregister xHCI controller
3361  *
3362  * @v xhci xHCI device
3363  */
3364 void xhci_unregister ( struct xhci_device *xhci ) {
3365  struct usb_bus *bus = xhci->bus;
3366 
3367  /* Unregister and free USB bus */
3368  unregister_usb_bus ( bus );
3369  free_usb_bus ( bus );
3370 
3371  /* Reset device */
3372  xhci_reset ( xhci );
3373 }
3374 
3375 /******************************************************************************
3376  *
3377  * PCI interface
3378  *
3379  ******************************************************************************
3380  */
3381 
3382 /**
3383  * Fix Intel PCH-specific quirks
3384  *
3385  * @v xhci xHCI device
3386  * @v pci PCI device
3387  */
3388 static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) {
3389  struct xhci_pch *pch = &xhci->pch;
3390  uint32_t xusb2pr;
3391  uint32_t xusb2prm;
3392  uint32_t usb3pssen;
3393  uint32_t usb3prm;
3394 
3395  /* Enable SuperSpeed capability. Do this before rerouting
3396  * USB2 ports, so that USB3 devices connect at SuperSpeed.
3397  */
3398  pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen );
3399  pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm );
3400  if ( usb3prm & ~usb3pssen ) {
3401  DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n",
3402  xhci->name, ( usb3prm & ~usb3pssen ) );
3403  }
3404  pch->usb3pssen = usb3pssen;
3405  usb3pssen |= usb3prm;
3406  pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen );
3407 
3408  /* Route USB2 ports from EHCI to xHCI */
3409  pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr );
3410  pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm );
3411  if ( xusb2prm & ~xusb2pr ) {
3412  DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n",
3413  xhci->name, ( xusb2prm & ~xusb2pr ) );
3414  }
3415  pch->xusb2pr = xusb2pr;
3416  xusb2pr |= xusb2prm;
3417  pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr );
3418 }
3419 
3420 /**
3421  * Undo Intel PCH-specific quirk fixes
3422  *
3423  * @v xhci xHCI device
3424  * @v pci PCI device
3425  */
3426 static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) {
3427  struct xhci_pch *pch = &xhci->pch;
3428 
3429  /* Restore USB2 port routing to original state */
3430  pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr );
3431 
3432  /* Restore SuperSpeed capability to original state */
3433  pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen );
3434 }
3435 
3436 /**
3437  * Probe PCI device
3438  *
3439  * @v pci PCI device
3440  * @ret rc Return status code
3441  */
3442 static int xhci_probe ( struct pci_device *pci ) {
3443  struct xhci_device *xhci;
3444  unsigned long bar_start;
3445  size_t bar_size;
3446  int rc;
3447 
3448  /* Allocate and initialise structure */
3449  xhci = zalloc ( sizeof ( *xhci ) );
3450  if ( ! xhci ) {
3451  rc = -ENOMEM;
3452  goto err_alloc;
3453  }
3454  xhci->dev = &pci->dev;
3455  xhci->dma = &pci->dma;
3456  xhci->quirks = pci->id->driver_data;
3457 
3458  /* Fix up PCI device */
3459  adjust_pci_device ( pci );
3460 
3461  /* Map registers */
3462  bar_start = pci_bar_start ( pci, XHCI_BAR );
3463  bar_size = pci_bar_size ( pci, XHCI_BAR );
3464  xhci->regs = pci_ioremap ( pci, bar_start, bar_size );
3465  if ( ! xhci->regs ) {
3466  rc = -ENODEV;
3467  goto err_ioremap;
3468  }
3469 
3470  /* Initialise xHCI device */
3471  xhci_init ( xhci );
3472 
3473  /* Initialise USB legacy support and claim ownership */
3474  xhci_legacy_init ( xhci );
3475  xhci_legacy_claim ( xhci );
3476 
3477  /* Fix Intel PCH-specific quirks, if applicable */
3478  if ( xhci->quirks & XHCI_PCH )
3479  xhci_pch_fix ( xhci, pci );
3480 
3481  /* Register xHCI device */
3482  if ( ( rc = xhci_register ( xhci ) ) != 0 )
3483  goto err_register;
3484 
3485  pci_set_drvdata ( pci, xhci );
3486  return 0;
3487 
3488  xhci_unregister ( xhci );
3489  err_register:
3490  if ( xhci->quirks & XHCI_PCH )
3491  xhci_pch_undo ( xhci, pci );
3492  xhci_legacy_release ( xhci );
3493  iounmap ( xhci->regs );
3494  err_ioremap:
3495  free ( xhci );
3496  err_alloc:
3497  return rc;
3498 }
3499 
3500 /**
3501  * Remove PCI device
3502  *
3503  * @v pci PCI device
3504  */
3505 static void xhci_remove ( struct pci_device *pci ) {
3506  struct xhci_device *xhci = pci_get_drvdata ( pci );
3507  uint16_t command;
3508 
3509  /* Some systems are observed to disable bus mastering on
3510  * Thunderbolt controllers before we get a chance to shut
3511  * down. Detect this and avoid attempting any DMA operations,
3512  * which are guaranteed to fail and may end up spuriously
3513  * completing after the operating system kernel starts up.
3514  */
3515  pci_read_config_word ( pci, PCI_COMMAND, &command );
3516  if ( ! ( command & PCI_COMMAND_MASTER ) ) {
3517  DBGC ( xhci, "XHCI %s DMA was disabled\n", xhci->name );
3518  xhci_fail ( xhci );
3519  }
3520 
3521  /* Unregister xHCI controller */
3522  xhci_unregister ( xhci );
3523 
3524  /* Undo any PCH-specific fixes */
3525  if ( xhci->quirks & XHCI_PCH )
3526  xhci_pch_undo ( xhci, pci );
3527 
3528  /* Release ownership back to BIOS */
3529  xhci_legacy_release ( xhci );
3530 
3531  /* Unmap registers */
3532  iounmap ( xhci->regs );
3533 
3534  /* Free device */
3535  free ( xhci );
3536 }
3537 
3538 /** XHCI PCI device IDs */
3539 static struct pci_device_id xhci_ids[] = {
3540  PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ),
3541  PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ),
3542  PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
3543 };
3544 
3545 /** XHCI PCI driver */
3546 struct pci_driver xhci_driver __pci_driver = {
3547  .ids = xhci_ids,
3548  .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
3549  .class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
3550  PCI_CLASS_SERIAL_USB_XHCI ),
3551  .probe = xhci_probe,
3552  .remove = xhci_remove,
3553 };
3554 
3555 /**
3556  * Prepare for exit
3557  *
3558  * @v booting System is shutting down for OS boot
3559  */
3560 static void xhci_shutdown ( int booting ) {
3561  /* If we are shutting down to boot an OS, then prevent the
3562  * release of ownership back to BIOS.
3563  */
3564  xhci_legacy_prevent_release = booting;
3565 }
3566 
3567 /** Startup/shutdown function */
3568 struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = {
3569  .name = "xhci",
3570  .shutdown = xhci_shutdown,
3571 };
XHCI_BAD_PSIV
#define XHCI_BAD_PSIV
Invalid protocol speed ID values quirk.
Definition: xhci.h:1042
xhci_endpoint_context
An endpoint context.
Definition: xhci.h:769
XHCI_CMPLT_SHORT
Short packet.
Definition: xhci.h:637
iob_pull
#define iob_pull(iobuf, len)
Definition: iobuf.h:107
xhci_endpoint_context::burst
uint8_t burst
Maximum burst size.
Definition: xhci.h:781
__attribute__
#define __attribute__(x)
Definition: compiler.h:10
xhci_event_ring
An event ring.
Definition: xhci.h:869
XHCI_USBLEGSUP_MAX_WAIT_MS
#define XHCI_USBLEGSUP_MAX_WAIT_MS
Maximum time to wait for BIOS to release ownership.
Definition: xhci.h:984
XHCI_SUPPORTED_PORTS
#define XHCI_SUPPORTED_PORTS
Supported protocol ports.
Definition: xhci.h:113
xhci_trb_disable_slot
A disable slot transfer request block.
Definition: xhci.h:485
xhci_root_speed
static int xhci_root_speed(struct usb_hub *hub, struct usb_port *port)
Update root hub port speed.
Definition: xhci.c:3204
XHCI_SUPPORTED_SLOT_TYPE
#define XHCI_SUPPORTED_SLOT_TYPE(slot)
Supported protocol slot type.
Definition: xhci.h:128
usb_endpoint_set_hostdata
static void usb_endpoint_set_hostdata(struct usb_endpoint *ep, void *priv)
Set USB endpoint host controller private data.
Definition: usb.h:576
xhci_command_free
static void xhci_command_free(struct xhci_device *xhci)
Free command ring.
Definition: xhci.c:1487
XHCI_TRB_TC
#define XHCI_TRB_TC
Transfer request block toggle cycle bit flag.
Definition: xhci.h:332
xhci_endpoint_context::type
uint8_t type
Endpoint type.
Definition: xhci.h:779
PCI_CLASS_ID
#define PCI_CLASS_ID(base, sub, progif)
Construct PCI class ID.
Definition: pci.h:203
rc
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
XHCI_STATUS_IN
#define XHCI_STATUS_IN
Input status direction.
Definition: xhci.h:440
name
const char * name
Definition: ath9k_hw.c:1986
uint16_t
unsigned short uint16_t
Definition: stdint.h:11
DMA_TX
#define DMA_TX
Device will read data from host memory.
Definition: dma.h:135
xhci_trb_status
A status stage transfer request block.
Definition: xhci.h:421
wmb
wmb()
xhci_trb::context
struct xhci_trb_context context
Input context TRB.
Definition: xhci.h:703
xhci_trb_complete::command
uint64_t command
Command TRB pointer.
Definition: xhci.h:614
xhci_trb_common
A transfer request block.
Definition: xhci.h:315
readb
uint8_t readb(volatile uint8_t *io_addr)
Read byte from memory-mapped device.
xhci_trb_ring::dbval
uint32_t dbval
Doorbell register value.
Definition: xhci.h:865
xhci_dequeue
static struct io_buffer * xhci_dequeue(struct xhci_trb_ring *ring)
Dequeue a transfer request block.
Definition: xhci.c:1359
pci_device::dma
struct dma_device dma
DMA device.
Definition: pci.h:215
xhci_trb::common
struct xhci_trb_common common
Common fields.
Definition: xhci.h:687
xhci_root_clear_tt
static int xhci_root_clear_tt(struct usb_hub *hub, struct usb_port *port, struct usb_endpoint *ep)
Clear transaction translator buffer.
Definition: xhci.c:3261
xhci_evaluate_context_input
static void xhci_evaluate_context_input(struct xhci_device *xhci, struct xhci_slot *slot __unused, struct xhci_endpoint *endpoint, void *input)
Populate evaluate context input context.
Definition: xhci.c:2269
__profiler
static struct profiler xhci_message_profiler __profiler
Message transfer profiler.
Definition: xhci.c:48
xhci_device::ports
unsigned int ports
Number of ports.
Definition: xhci.h:1093
XHCI_CONFIG_MAX_SLOTS_EN
#define XHCI_CONFIG_MAX_SLOTS_EN(slots)
Maximum device slots enabled.
Definition: xhci.h:197
xhci_scratchpad::buffer
void * buffer
Scratchpad buffer area.
Definition: xhci.h:1057
xhci_stop
static int xhci_stop(struct xhci_device *xhci)
Stop xHCI device.
Definition: xhci.c:1110
xhci_device::dcbaa
struct xhci_dcbaa dcbaa
Device context base address array.
Definition: xhci.h:1109
FILE_SECBOOT
FILE_SECBOOT(PERMITTED)
xhci_device_open
static int xhci_device_open(struct usb_device *usb)
Open device.
Definition: xhci.c:2778
pci_driver
A PCI driver.
Definition: pci.h:252
EBUSY
#define EBUSY
Device or resource busy.
Definition: errno.h:339
xhci_register
int xhci_register(struct xhci_device *xhci)
Register xHCI controller.
Definition: xhci.c:3319
normal
struct arbelprm_completion_queue_entry normal
Definition: arbel.h:11
XHCI_TRB_ENABLE_SLOT
#define XHCI_TRB_ENABLE_SLOT
An enable slot transfer request block.
Definition: xhci.h:482
reg
static unsigned int unsigned int reg
Definition: myson.h:162
xhci_ring_consumed
static physaddr_t xhci_ring_consumed(struct xhci_trb_ring *ring)
Calculate physical address of most recently consumed TRB.
Definition: xhci.h:936
XHCI_OP_USBSTS
#define XHCI_OP_USBSTS
USB status register.
Definition: xhci.h:164
XHCI_USBLEGSUP_BIOS_OWNED
#define XHCI_USBLEGSUP_BIOS_OWNED
USB legacy support BIOS ownership flag.
Definition: xhci.h:89
XHCI_PORTSC_PSIV
#define XHCI_PORTSC_PSIV(portsc)
Port speed ID value.
Definition: xhci.h:234
usb_hub
A USB hub.
Definition: usb.h:841
xhci_address_device
static int xhci_address_device(struct xhci_device *xhci, struct xhci_slot *slot)
Address device.
Definition: xhci.c:2114
XHCI_CTX_END
#define XHCI_CTX_END
End of contexts.
Definition: xhci.h:954
XHCI_ICI
#define XHCI_ICI(ctx)
Input context index.
Definition: xhci.h:960
xhci_hub_close
static void xhci_hub_close(struct usb_hub *hub __unused)
Close hub.
Definition: xhci.c:3060
XHCI_XECP_NEXT
#define XHCI_XECP_NEXT(xecp)
Next xHCI extended capability pointer.
Definition: xhci.h:80
xhci_extended_capability
static unsigned int xhci_extended_capability(struct xhci_device *xhci, unsigned int id, unsigned int offset)
Find extended capability.
Definition: xhci.c:330
xhci_slot::xhci
struct xhci_device * xhci
xHCI device
Definition: xhci.h:1136
XHCI_TRB_SET_TR_DEQUEUE_POINTER
#define XHCI_TRB_SET_TR_DEQUEUE_POINTER
A set transfer ring dequeue pointer transfer request block.
Definition: xhci.h:583
xhci_trb_ring::cons
unsigned int cons
Consumer counter.
Definition: xhci.h:844
xhci_device_context_offset
static size_t xhci_device_context_offset(struct xhci_device *xhci, unsigned int ctx)
Calculate device context offset.
Definition: xhci.c:419
xhci_device::run
void * run
Runtime registers.
Definition: xhci.h:1084
xhci_device
An xHCI device.
Definition: xhci.h:1067
xhci_trb_set_tr_dequeue_pointer
A set transfer ring dequeue pointer transfer request block.
Definition: xhci.h:567
alloc_usb_bus
struct usb_bus * alloc_usb_bus(struct device *dev, unsigned int ports, size_t mtu, struct usb_host_operations *op)
Allocate USB bus.
Definition: usb.c:2095
errno.h
Error codes.
USB_SPEED_LOW
Low speed (1.5Mbps)
Definition: usb.h:49
XHCI_PORT_RESET_MAX_WAIT_MS
#define XHCI_PORT_RESET_MAX_WAIT_MS
Maximum time to wait for a port reset to complete.
Definition: xhci.h:1016
USB_ENDPOINT_ATTR_TYPE_MASK
#define USB_ENDPOINT_ATTR_TYPE_MASK
Endpoint attribute transfer type mask.
Definition: usb.h:281
xhci_trb_ring::len
size_t len
Length of transfer request blocks.
Definition: xhci.h:856
XHCI_SUPPORTED_SLOT
#define XHCI_SUPPORTED_SLOT
Supported protocol slot.
Definition: xhci.h:125
xhci_device::regs
void * regs
Registers.
Definition: xhci.h:1069
xhci_control_context
An input control context.
Definition: xhci.h:721
XHCI_HCSPARAMS1_PORTS
#define XHCI_HCSPARAMS1_PORTS(params)
Number of ports.
Definition: xhci.h:49
command
A command-line command.
Definition: command.h:10
xhci_align
static size_t xhci_align(size_t len)
Calculate buffer alignment.
Definition: xhci.c:400
pci_device_id::driver_data
unsigned long driver_data
Arbitrary driver data.
Definition: pci.h:183
xhci_trb_enable_slot::slot
uint8_t slot
Slot type.
Definition: xhci.h:476
xhci_device::command
struct xhci_trb_ring command
Command ring.
Definition: xhci.h:1115
xhci_ring_fill
static unsigned int xhci_ring_fill(struct xhci_trb_ring *ring)
Calculate space used in TRB ring.
Definition: xhci.h:898
xhci_trb_complete
A command completion event transfer request block.
Definition: xhci.h:612
XHCI_TRB_DISABLE_SLOT
#define XHCI_TRB_DISABLE_SLOT
A disable slot transfer request block.
Definition: xhci.h:501
xhci_endpoint_context::state
uint8_t state
Endpoint state.
Definition: xhci.h:771
pci_driver::ids
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:254
usb_set_hostdata
static void usb_set_hostdata(struct usb_device *usb, void *priv)
Set USB device host controller private data.
Definition: usb.h:784
type
uint32_t type
Operating system type.
Definition: ena.h:12
XHCI_PCH_XUSB2PR
#define XHCI_PCH_XUSB2PR
Intel PCH USB2 port routing register.
Definition: xhci.h:1030
xhci_trb::transfer
struct xhci_trb_transfer transfer
Transfer event.
Definition: xhci.h:711
EPIPE
#define EPIPE
Broken pipe.
Definition: errno.h:620
xhci_trb_host_controller
A port status change transfer request block.
Definition: xhci.h:664
xhci_endpoint_context::trb_len
uint16_t trb_len
Average TRB length.
Definition: xhci.h:787
xhci_device::pending
union xhci_trb * pending
Current command (if any)
Definition: xhci.h:1119
XHCI_SUPPORTED_PSI_VALUE
#define XHCI_SUPPORTED_PSI_VALUE(psi)
Supported protocol PSI value.
Definition: xhci.h:134
DMA_RX
#define DMA_RX
Device will write data to host memory.
Definition: dma.h:138
XHCI_EP_TYPE
#define XHCI_EP_TYPE(type)
Endpoint type.
Definition: xhci.h:812
xhci_slot::endpoint
struct xhci_endpoint * endpoint[XHCI_CTX_END]
Endpoints, indexed by context ID.
Definition: xhci.h:1158
readl
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
xhci_device::dev
struct device * dev
Underlying hardware device.
Definition: xhci.h:1071
xhci_set_tr_dequeue_pointer
static int xhci_set_tr_dequeue_pointer(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Set transfer ring dequeue pointer.
Definition: xhci.c:2393
XHCI_XECP_ID_SUPPORTED
#define XHCI_XECP_ID_SUPPORTED
Supported protocol extended capability.
Definition: xhci.h:101
DBGC
#define DBGC(...)
Definition: compiler.h:505
xhci_trb_common::type
uint8_t type
Type.
Definition: xhci.h:323
xhci_trb_normal::flags
uint8_t flags
Flags.
Definition: xhci.h:356
xhci_trb_link::flags
uint8_t flags
Flags.
Definition: xhci.h:452
xhci_context
static int xhci_context(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, unsigned int type, void(*populate)(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input))
Issue context-based command and wait for completion.
Definition: xhci.c:2022
pci_bar_size
unsigned long pci_bar_size(struct pci_device *pci, unsigned int reg)
Get the size of a PCI BAR.
Definition: pci.c:164
xhci_trb_ring::map
struct dma_mapping map
DMA mapping.
Definition: xhci.h:858
__pci_driver
struct pci_driver xhci_driver __pci_driver
XHCI PCI driver.
Definition: xhci.c:3546
XHCI_EP_TYPE_CONTROL
#define XHCI_EP_TYPE_CONTROL
Control endpoint type.
Definition: xhci.h:815
xhci_trb_host_controller::code
uint8_t code
Completion code.
Definition: xhci.h:670
xhci_trb::disable
struct xhci_trb_disable_slot disable
Disable slot TRB.
Definition: xhci.h:701
device::name
char name[40]
Name.
Definition: device.h:79
XHCI_HCSPARAMS1_INTRS
#define XHCI_HCSPARAMS1_INTRS(params)
Number of interrupters.
Definition: xhci.h:46
input
Definition: bnxt_hsi.h:68
index
long index
Definition: bigint.h:65
ENOENT
#define ENOENT
No such file or directory.
Definition: errno.h:515
xhci_trb_enable_slot
An enable slot transfer request block.
Definition: xhci.h:466
usb_port::usb
struct usb_device * usb
Currently attached device (if in use)
Definition: usb.h:835
XHCI_SETUP_OUT
#define XHCI_SETUP_OUT
Setup stage output data direction.
Definition: xhci.h:393
xhci_deconfigure_endpoint_input
static void xhci_deconfigure_endpoint_input(struct xhci_device *xhci __unused, struct xhci_slot *slot __unused, struct xhci_endpoint *endpoint, void *input)
Populate deconfigure endpoint input context.
Definition: xhci.c:2215
XHCI_OP_PORTHLPMC
#define XHCI_OP_PORTHLPMC(port)
Port hardware link power management control register.
Definition: xhci.h:293
xhci_device::bus
struct usb_bus * bus
USB bus.
Definition: xhci.h:1127
xhci_trb::enable
struct xhci_trb_enable_slot enable
Enable slot TRB.
Definition: xhci.h:699
dma_set_mask_64bit
static __always_inline void dma_set_mask_64bit(struct dma_device *dma)
Set 64-bit addressable space mask.
Definition: dma.h:467
xhci_trb_disable_slot::slot
uint8_t slot
Slot ID.
Definition: xhci.h:497
xhci_ring_free
static void xhci_ring_free(struct xhci_trb_ring *ring)
Free transfer request block ring.
Definition: xhci.c:1290
xhci_scratchpad::buffer_map
struct dma_mapping buffer_map
Buffer DMA mapping.
Definition: xhci.h:1059
xhci_configure_endpoint
static int xhci_configure_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Configure endpoint.
Definition: xhci.c:2187
cpu_to_le64
#define cpu_to_le64(value)
Definition: byteswap.h:109
xhci_scratchpad::count
unsigned int count
Number of page-sized scratchpad buffers.
Definition: xhci.h:1055
pci_read_config_word
int pci_read_config_word(struct pci_device *pci, unsigned int where, uint16_t *value)
Read 16-bit word from PCI configuration space.
xhci_command_alloc
static int xhci_command_alloc(struct xhci_device *xhci)
Allocate command ring.
Definition: xhci.c:1456
xhci_trb_data
A data stage transfer request block.
Definition: xhci.h:396
xhci_trb_transfer::code
uint8_t code
Completion code.
Definition: xhci.h:597
xhci_pch
Intel PCH quirk.
Definition: xhci.h:1019
xhci_trb_ring::db
void * db
Doorbell register.
Definition: xhci.h:863
XHCI_TRB_RESET_ENDPOINT
#define XHCI_TRB_RESET_ENDPOINT
A reset endpoint transfer request block.
Definition: xhci.h:545
XHCI_SPEED_FULL
Full speed (12Mbps)
Definition: xhci.h:145
PCI_COMMAND
#define PCI_COMMAND
PCI command.
Definition: pci.h:26
XHCI_TRANSFER_TRBS_LOG2
#define XHCI_TRANSFER_TRBS_LOG2
Number of TRBs in a transfer ring.
Definition: xhci.h:978
XHCI_DATA_IN
#define XHCI_DATA_IN
Input data direction.
Definition: xhci.h:415
xhci_trb_common::flags
uint8_t flags
Flags.
Definition: xhci.h:321
XHCI_TRB_PORT_STATUS
#define XHCI_TRB_PORT_STATUS
A port status change transfer request block.
Definition: xhci.h:661
iob_map
static __always_inline int iob_map(struct io_buffer *iobuf, struct dma_device *dma, size_t len, int flags)
Map I/O buffer for DMA.
Definition: iobuf.h:231
XHCI_PORTSC_CSC
#define XHCI_PORTSC_CSC
Connect status change.
Definition: xhci.h:249
xhci_endpoint
An xHCI endpoint.
Definition: xhci.h:1162
xhci_trb_reset_endpoint::type
uint8_t type
Type.
Definition: xhci.h:537
xhci_trb_setup::direction
uint8_t direction
Transfer direction.
Definition: xhci.h:381
ctx
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
profiler
A data structure for storing profiling information.
Definition: profile.h:27
XHCI_CMPLT_SUCCESS
Success.
Definition: xhci.h:635
xhci_endpoint_stream
static int xhci_endpoint_stream(struct usb_endpoint *ep, struct io_buffer *iobuf, int zlp)
Enqueue stream transfer.
Definition: xhci.c:2695
USB_SPEED_SUPER
Super speed (5Gbps)
Definition: usb.h:55
XHCI_RESET_MAX_WAIT_MS
#define XHCI_RESET_MAX_WAIT_MS
Maximum time to wait for reset to complete.
Definition: xhci.h:996
profile_stop
static void profile_stop(struct profiler *profiler)
Stop profiling.
Definition: profile.h:174
XHCI_COMMAND_ABORT_DELAY_MS
#define XHCI_COMMAND_ABORT_DELAY_MS
Time to delay after aborting a command.
Definition: xhci.h:1010
XHCI_OP_PORTLI
#define XHCI_OP_PORTLI(port)
Port link info register.
Definition: xhci.h:290
XHCI_PORTSC_PRESERVE
#define XHCI_PORTSC_PRESERVE
Port status and control bits which should be preserved.
Definition: xhci.h:284
xhci_trb_transfer::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:603
adjust_pci_device
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:241
startup_fn::name
const char * name
Definition: init.h:44
xhci_trb_set_tr_dequeue_pointer::dequeue
uint64_t dequeue
Dequeue pointer.
Definition: xhci.h:569
pci_device::dev
struct device dev
Generic device.
Definition: pci.h:213
XHCI_TRB_STATUS
#define XHCI_TRB_STATUS
A status stage transfer request block.
Definition: xhci.h:437
xhci_ids
static struct pci_device_id xhci_ids[]
XHCI PCI device IDs.
Definition: xhci.c:3539
xhci_device::slots
unsigned int slots
Number of device slots.
Definition: xhci.h:1089
xhci_control_context::add
uint32_t add
Add context flags.
Definition: xhci.h:725
enabled
uint16_t enabled
Single-entry bitmask of the enabled option value.
Definition: ena.h:14
xhci_endpoint_context::dequeue
uint64_t dequeue
Transfer ring dequeue pointer.
Definition: xhci.h:785
xhci_device::scratch
struct xhci_scratchpad scratch
Scratchpad buffer.
Definition: xhci.h:1112
XHCI_USBLEGSUP_OS
#define XHCI_USBLEGSUP_OS
USB legacy support OS owned semaphore.
Definition: xhci.h:92
usb_device::address
unsigned int address
Device address, if assigned.
Definition: usb.h:733
xhci_ring_remaining
static unsigned int xhci_ring_remaining(struct xhci_trb_ring *ring)
Calculate space remaining in TRB ring.
Definition: xhci.h:913
PCI_CLASS_SERIAL_USB_XHCI
#define PCI_CLASS_SERIAL_USB_XHCI
xHCI USB controller
Definition: pci.h:141
xhci_scratchpad::array
uint64_t * array
Scratchpad array.
Definition: xhci.h:1061
xhci_endpoint_mtu
static int xhci_endpoint_mtu(struct usb_endpoint *ep)
Update MTU.
Definition: xhci.c:2576
writeb
void writeb(uint8_t data, volatile uint8_t *io_addr)
Write byte to memory-mapped device.
usb_device::speed
unsigned int speed
Device speed.
Definition: usb.h:729
xhci_slot_context::address
uint8_t address
USB address.
Definition: xhci.h:755
ECANCELED
#define ECANCELED
Operation canceled.
Definition: errno.h:344
XHCI_TRB_ADDRESS_DEVICE
#define XHCI_TRB_ADDRESS_DEVICE
An address device transfer request block.
Definition: xhci.h:520
ENOTSUP
#define ENOTSUP
Operation not supported.
Definition: errno.h:590
xhci_slot_context::ports
uint8_t ports
Number of downstream ports.
Definition: xhci.h:747
STARTUP_LATE
#define STARTUP_LATE
Late startup.
Definition: init.h:66
usb_port_changed
void usb_port_changed(struct usb_port *port)
Report port status change.
Definition: usb.c:1858
PCI_CLASS_SERIAL
#define PCI_CLASS_SERIAL
Definition: Pci22.h:267
ECODE
#define ECODE(code)
Definition: xhci.c:234
malloc.h
Dynamic memory allocation.
xhci_port_psiv
static int xhci_port_psiv(struct xhci_device *xhci, unsigned int port, unsigned int speed)
Find protocol speed ID value.
Definition: xhci.c:855
XHCI_DATA_OUT
#define XHCI_DATA_OUT
Output data direction.
Definition: xhci.h:418
xhci_transfer
static void xhci_transfer(struct xhci_device *xhci, struct xhci_trb_transfer *trb)
Handle transfer event.
Definition: xhci.c:1596
usb_bus_set_hostdata
static void usb_bus_set_hostdata(struct usb_bus *bus, void *priv)
Set USB bus host controller private data.
Definition: usb.h:1051
xhci_endpoint::type
unsigned int type
Endpoint type.
Definition: xhci.h:1172
xhci_trb::host
struct xhci_trb_host_controller host
Host controller event.
Definition: xhci.h:717
XHCI_EP_DCS
#define XHCI_EP_DCS
Endpoint dequeue cycle state.
Definition: xhci.h:824
xhci_trb_context::input
uint64_t input
Input context.
Definition: xhci.h:506
XHCI_EP_TYPE_IN
#define XHCI_EP_TYPE_IN
Input endpoint type.
Definition: xhci.h:818
xhci_speed_name
static const char * xhci_speed_name(uint32_t psi)
Transcribe port speed (for debugging)
Definition: xhci.c:649
xhci_root_enable
static int xhci_root_enable(struct usb_hub *hub, struct usb_port *port)
Enable port.
Definition: xhci.c:3136
XHCI_PORTSC_PP
#define XHCI_PORTSC_PP
Port power.
Definition: xhci.h:228
startup_fn
A startup/shutdown function.
Definition: init.h:43
PCI_COMMAND_MASTER
#define PCI_COMMAND_MASTER
Bus master.
Definition: pci.h:29
xhci_bus_poll
static void xhci_bus_poll(struct usb_bus *bus)
Poll USB bus.
Definition: xhci.c:3013
XHCI_PORTSC_CCS
#define XHCI_PORTSC_CCS
Current connect status.
Definition: xhci.h:207
xhci_ring_reset
static void xhci_ring_reset(struct xhci_trb_ring *ring)
Reset transfer request block ring.
Definition: xhci.c:1274
usb_port
A USB port.
Definition: usb.h:813
dma_free
void dma_free(struct dma_mapping *map, void *addr, size_t len)
Unmap and free DMA-coherent buffer.
pci_set_drvdata
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:366
rmb
#define rmb()
Definition: io.h:545
ENOMEM
#define ENOMEM
Not enough space.
Definition: errno.h:535
usb_endpoint
A USB endpoint.
Definition: usb.h:404
XHCI_COMMAND_MAX_WAIT_MS
#define XHCI_COMMAND_MAX_WAIT_MS
Maximum time to wait for a command to complete.
Definition: xhci.h:1004
usb_hub::usb
struct usb_device * usb
Underlying USB device, if any.
Definition: usb.h:847
xhci_scratchpad::array_map
struct dma_mapping array_map
Array DMA mapping.
Definition: xhci.h:1063
xhci_enable_slot
static int xhci_enable_slot(struct xhci_device *xhci, unsigned int type)
Enable slot.
Definition: xhci.c:1956
xhci_trb_stop_endpoint::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:558
XHCI_OP_CRCR
#define XHCI_OP_CRCR
Command ring control register.
Definition: xhci.h:179
xhci_trb_ring::trb
union xhci_trb * trb
Transfer request blocks.
Definition: xhci.h:854
memcpy
void * memcpy(void *dest, const void *src, size_t len) __nonnull
XHCI_TRB_COMPLETE
#define XHCI_TRB_COMPLETE
A command completion event transfer request block.
Definition: xhci.h:630
xhci_trb_transfer::slot
uint8_t slot
Slot ID.
Definition: xhci.h:605
xhci_init
void xhci_init(struct xhci_device *xhci)
Initialise device.
Definition: xhci.c:264
profiler::name
const char * name
Name.
Definition: profile.h:29
xhci_probe
static int xhci_probe(struct pci_device *pci)
Probe PCI device.
Definition: xhci.c:3442
xhci_device::pch
struct xhci_pch pch
Intel PCH quirk.
Definition: xhci.h:1130
port
u8 port
Port number.
Definition: CIB_PRM.h:31
XHCI_CAP_RTSOFF
#define XHCI_CAP_RTSOFF
Runtime register space offset.
Definition: xhci.h:74
xhci_trb_reset_endpoint::slot
uint8_t slot
Slot ID.
Definition: xhci.h:541
xhci_trb_reset_endpoint
A reset endpoint transfer request block.
Definition: xhci.h:529
xhci_port_slot_type
static int xhci_port_slot_type(struct xhci_device *xhci, unsigned int port)
Find port slot type.
Definition: xhci.c:767
unregister_usb_bus
void unregister_usb_bus(struct usb_bus *bus)
Unregister USB bus.
Definition: usb.c:2171
usb_device::port
struct usb_port * port
USB port.
Definition: usb.h:727
xhci_event_ring::trb
union xhci_trb * trb
Transfer request blocks.
Definition: xhci.h:877
xhci_trb_stop_endpoint::slot
uint8_t slot
Slot ID.
Definition: xhci.h:560
xhci_control_context::drop
uint32_t drop
Drop context flags.
Definition: xhci.h:723
assert
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
XHCI_EP_TYPE_PERIODIC
#define XHCI_EP_TYPE_PERIODIC
Periodic endpoint type.
Definition: xhci.h:821
USB_PROTO_3_0
USB 3.0.
Definition: usb.h:25
xhci_device_close
static void xhci_device_close(struct usb_device *usb)
Close device.
Definition: xhci.c:2861
pci_read_config_dword
int pci_read_config_dword(struct pci_device *pci, unsigned int where, uint32_t *value)
Read 32-bit dword from PCI configuration space.
XHCI_TRB_DATA
#define XHCI_TRB_DATA
A data stage transfer request block.
Definition: xhci.h:412
xhci_trb::stop
struct xhci_trb_stop_endpoint stop
Stop endpoint TRB.
Definition: xhci.h:707
XHCI_MTU
#define XHCI_MTU
Maximum transfer size.
Definition: xhci.h:28
USB_SPEED_NONE
Not connected.
Definition: usb.h:47
xhci_trb::template
struct xhci_trb_template template
Template.
Definition: xhci.h:685
xhci_dump_port
static void xhci_dump_port(struct xhci_device *xhci, unsigned int port)
Dump port registers.
Definition: xhci.c:490
xhci_endpoint::ctx
unsigned int ctx
Context index.
Definition: xhci.h:1170
xhci_trb_setup::len
uint32_t len
Length.
Definition: xhci.h:375
value
pseudo_bit_t value[0x00020]
Definition: arbel.h:13
DBGC_HDA
#define DBGC_HDA(...)
Definition: compiler.h:506
__unused
#define __unused
Declare a variable or data structure as unused.
Definition: compiler.h:573
xhci_slot::usb
struct usb_device * usb
USB device.
Definition: xhci.h:1138
xhci_device_address
static int xhci_device_address(struct usb_device *usb)
Assign device address.
Definition: xhci.c:2898
xhci_pch::xusb2pr
uint32_t xusb2pr
USB2 port routing register original value.
Definition: xhci.h:1021
iob_unmap
static __always_inline void iob_unmap(struct io_buffer *iobuf)
Unmap I/O buffer for DMA.
Definition: iobuf.h:279
XHCI_PORTSC_CHANGE
#define XHCI_PORTSC_CHANGE
Port status change mask.
Definition: xhci.h:270
len
ring len
Length.
Definition: dwmac.h:231
writel
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
xhci_slot_context::port
uint8_t port
Root hub port number.
Definition: xhci.h:745
XHCI_SLOT_INFO
#define XHCI_SLOT_INFO(entries, hub, speed, route)
Construct slot context device info.
Definition: xhci.h:765
XHCI_TRB_TRANSFER
#define XHCI_TRB_TRANSFER
A transfer event transfer request block.
Definition: xhci.h:609
XHCI_USBLEGSUP_CTLSTS
#define XHCI_USBLEGSUP_CTLSTS
USB legacy support control/status.
Definition: xhci.h:98
xhci_trb_setup
A setup stage transfer request block.
Definition: xhci.h:371
xhci_endpoint_context::mtu
uint16_t mtu
Maximum packet size.
Definition: xhci.h:783
xhci_trb_transfer
A transfer event transfer request block.
Definition: xhci.h:589
usb_endpoint::burst
unsigned int burst
Maximum burst size.
Definition: usb.h:414
xhci_bus_close
static void xhci_bus_close(struct usb_bus *bus)
Close USB bus.
Definition: xhci.c:2991
XHCI_PCH
#define XHCI_PCH
Intel PCH quirk flag.
Definition: xhci.h:1027
XHCI_TRB_NOP_CMD
#define XHCI_TRB_NOP_CMD
A no-op command transfer request block.
Definition: xhci.h:586
XHCI_TRB_SETUP
#define XHCI_TRB_SETUP
A setup stage transfer request block.
Definition: xhci.h:387
iob_dma
static __always_inline physaddr_t iob_dma(struct io_buffer *iobuf)
Get I/O buffer DMA address.
Definition: iobuf.h:268
xhci_root_open
static int xhci_root_open(struct usb_hub *hub)
Open root hub.
Definition: xhci.c:3078
link
u32 link
Link to next descriptor.
Definition: ar9003_mac.h:25
count
static unsigned int count
Number of entries.
Definition: dwmac.h:225
pci_bar_start
unsigned long pci_bar_start(struct pci_device *pci, unsigned int reg)
Find the start of a PCI BAR.
Definition: pci.c:97
xhci_trb_setup::type
uint8_t type
Type.
Definition: xhci.h:379
xhci_pch::usb3pssen
uint32_t usb3pssen
USB3 port SuperSpeed enable register original value.
Definition: xhci.h:1023
xhci_fail
static int xhci_fail(struct xhci_device *xhci)
Mark xHCI device as permanently failed.
Definition: xhci.c:1180
USB_DIR_IN
#define USB_DIR_IN
Data transfer is from device to host.
Definition: usb.h:98
profile_start
static void profile_start(struct profiler *profiler)
Start profiling.
Definition: profile.h:161
usb_device::name
char name[32]
Name.
Definition: usb.h:725
xhci_legacy_claim
static void xhci_legacy_claim(struct xhci_device *xhci)
Claim ownership from BIOS.
Definition: xhci.c:571
usb_hub::ports
unsigned int ports
Number of ports.
Definition: usb.h:851
xhci_event_free
static void xhci_event_free(struct xhci_device *xhci)
Free event ring.
Definition: xhci.c:1570
profile.h
Profiling.
USB_SPEED
#define USB_SPEED(mantissa, exponent)
Define a USB speed.
Definition: usb.h:36
xhci_slot_context::info
uint32_t info
Device info.
Definition: xhci.h:741
xhci_endpoint::slot
struct xhci_slot * slot
xHCI slot
Definition: xhci.h:1166
XHCI_SUPPORTED_PORTS_PSIC
#define XHCI_SUPPORTED_PORTS_PSIC(ports)
Supported protocol PSI count.
Definition: xhci.h:122
xhci_endpoint::context
struct xhci_endpoint_context * context
Endpoint context.
Definition: xhci.h:1176
cpu_to_le32
#define cpu_to_le32(value)
Definition: byteswap.h:108
id
uint8_t id
Request identifier.
Definition: ena.h:12
usb_device
A USB device.
Definition: usb.h:723
revision
uint32_t revision
Entry point revision.
Definition: ib_mad.h:21
xhci_trb_setup::flags
uint8_t flags
Flags.
Definition: xhci.h:377
XHCI_PORTSC_LWS
#define XHCI_PORTSC_LWS
Port link state write strobe.
Definition: xhci.h:243
XHCI_DCI
#define XHCI_DCI(ctx)
Device context index.
Definition: xhci.h:957
xhci_deconfigure_endpoint
static int xhci_deconfigure_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Deconfigure endpoint.
Definition: xhci.c:2241
xhci_slot
An xHCI device slot.
Definition: xhci.h:1134
XHCI_OP_DCBAAP
#define XHCI_OP_DCBAAP
Device context base address array pointer.
Definition: xhci.h:191
xhci_trb_context
A context transfer request block.
Definition: xhci.h:504
XHCI_TRB_STOP_ENDPOINT
#define XHCI_TRB_STOP_ENDPOINT
A stop endpoint transfer request block.
Definition: xhci.h:564
XHCI_TRB_HOST_CONTROLLER
#define XHCI_TRB_HOST_CONTROLLER
A port status change transfer request block.
Definition: xhci.h:680
iob_unput
#define iob_unput(iobuf, len)
Definition: iobuf.h:140
XHCI_SUPPORTED_PSI
#define XHCI_SUPPORTED_PSI(index)
Supported protocol PSI.
Definition: xhci.h:131
xhci_bus_open
static int xhci_bus_open(struct usb_bus *bus)
Open USB bus.
Definition: xhci.c:2940
strerror
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:79
xhci_enqueue_multi
static int xhci_enqueue_multi(struct xhci_trb_ring *ring, struct io_buffer *iobuf, const union xhci_trb *trbs, unsigned int count)
Enqueue multiple transfer request blocks.
Definition: xhci.c:1391
free
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:55
xhci_shutdown
static void xhci_shutdown(int booting)
Prepare for exit.
Definition: xhci.c:3560
xhci_root_disable
static int xhci_root_disable(struct usb_hub *hub, struct usb_port *port)
Disable port.
Definition: xhci.c:3171
xhci_trb_template::status
uint32_t status
Status.
Definition: xhci.h:309
zalloc
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:662
XHCI_XECP_ID
#define XHCI_XECP_ID(xecp)
xHCI extended capability ID
Definition: xhci.h:77
cons
uint16_t cons
Consumer index.
Definition: ena.h:22
xhci_device::failed
int failed
Command mechanism has permanently failed.
Definition: xhci.h:1121
PCI_CLASS_SERIAL_USB
#define PCI_CLASS_SERIAL_USB
Definition: Pci22.h:273
xhci_trb_link
A link transfer request block.
Definition: xhci.h:446
xhci_trb
A transfer request block.
Definition: xhci.h:683
xhci_dcbaa_free
static void xhci_dcbaa_free(struct xhci_device *xhci)
Free device context base address array.
Definition: xhci.c:959
usb_route_string
unsigned int usb_route_string(struct usb_device *usb)
Get USB route string.
Definition: usb.c:2336
dma_umalloc
void * dma_umalloc(struct dma_device *dma, struct dma_mapping *map, size_t len, size_t align)
Allocate and map DMA-coherent buffer from external (user) memory.
xhci_device::name
const char * name
Name.
Definition: xhci.h:1075
pci.h
PCI bus.
pci_device
A PCI device.
Definition: pci.h:211
xhci_trb_set_tr_dequeue_pointer::slot
uint8_t slot
Slot ID.
Definition: xhci.h:579
iob_len
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:160
xhci_root_close
static void xhci_root_close(struct usb_hub *hub __unused)
Close root hub.
Definition: xhci.c:3124
xhci_slot::id
unsigned int id
Slot ID.
Definition: xhci.h:1140
addr
uint32_t addr
Buffer address.
Definition: dwmac.h:20
XHCI_USBLEGSUP_BIOS
#define XHCI_USBLEGSUP_BIOS
USB legacy support BIOS owned semaphore.
Definition: xhci.h:86
XHCI_USBLEGSUP_OS_OWNED
#define XHCI_USBLEGSUP_OS_OWNED
USB legacy support OS ownership flag.
Definition: xhci.h:95
XHCI_OP_PORTPMSC
#define XHCI_OP_PORTPMSC(port)
Port power management status and control register.
Definition: xhci.h:287
xhci_dcbaa::context
uint64_t * context
Context base addresses.
Definition: xhci.h:1047
ENODEV
#define ENODEV
No such device.
Definition: errno.h:510
XHCI_HCCPARAMS1_XECP
#define XHCI_HCCPARAMS1_XECP(params)
xHCI extended capabilities pointer
Definition: xhci.h:68
xhci_scratchpad_alloc
static int xhci_scratchpad_alloc(struct xhci_device *xhci)
Allocate scratchpad buffers.
Definition: xhci.c:988
XHCI_BAR
#define XHCI_BAR
xHCI PCI BAR
Definition: xhci.h:31
XHCI_HCCPARAMS1_ADDR64
#define XHCI_HCCPARAMS1_ADDR64(params)
64-bit addressing capability
Definition: xhci.h:62
XHCI_CAP_CAPLENGTH
#define XHCI_CAP_CAPLENGTH
Capability register length.
Definition: xhci.h:34
xhci_dump
static void xhci_dump(struct xhci_device *xhci)
Dump host controller registers.
Definition: xhci.c:449
uint8_t
unsigned char uint8_t
Definition: stdint.h:10
nop
uint8_t nop
Definition: tcp.h:14
slot
uint8_t slot
Slot.
Definition: edd.h:17
xhci_endpoint_open
static int xhci_endpoint_open(struct usb_endpoint *ep)
Open endpoint.
Definition: xhci.c:2442
xhci_remove
static void xhci_remove(struct pci_device *pci)
Remove PCI device.
Definition: xhci.c:3505
xhci_device::intrs
unsigned int intrs
Number of interrupters.
Definition: xhci.h:1091
xhci_writeq
static int xhci_writeq(struct xhci_device *xhci, physaddr_t value, void *reg)
Write potentially 64-bit register.
Definition: xhci.c:366
usb_get_hostdata
static void * usb_get_hostdata(struct usb_device *usb)
Get USB device host controller private data.
Definition: usb.h:795
xhci_trb_ring::prod
unsigned int prod
Producer counter.
Definition: xhci.h:842
xhci_trb_disable_slot::type
uint8_t type
Type.
Definition: xhci.h:493
XHCI_EP0_TRB_LEN
#define XHCI_EP0_TRB_LEN
Control endpoint average TRB length.
Definition: xhci.h:827
xhci_trb::complete
struct xhci_trb_complete complete
Command completion event.
Definition: xhci.h:713
xhci_trb::dequeue
struct xhci_trb_set_tr_dequeue_pointer dequeue
Set transfer ring dequeue pointer TRB.
Definition: xhci.h:709
XHCI_SUPPORTED_PORTS_COUNT
#define XHCI_SUPPORTED_PORTS_COUNT(ports)
Supported protocol port count.
Definition: xhci.h:119
pci_device_id
A PCI device ID list entry.
Definition: pci.h:175
command
static int command
Definition: epic100.c:68
dma_ufree
void dma_ufree(struct dma_mapping *map, void *addr, size_t len)
Unmap and free DMA-coherent buffer from external (user) memory.
xhci_endpoint_message
static int xhci_endpoint_message(struct usb_endpoint *ep, struct io_buffer *iobuf)
Enqueue message transfer.
Definition: xhci.c:2596
le16_to_cpu
#define le16_to_cpu(value)
Definition: byteswap.h:113
XHCI_HCSPARAMS1_SLOTS
#define XHCI_HCSPARAMS1_SLOTS(params)
Number of device slots.
Definition: xhci.h:43
uint32_t
unsigned int uint32_t
Definition: stdint.h:12
dma_alloc
void * dma_alloc(struct dma_device *dma, struct dma_mapping *map, size_t len, size_t align)
Allocate and map DMA-coherent buffer.
next
uint32_t next
Next descriptor address.
Definition: dwmac.h:22
usb_setup_packet
A USB setup data packet.
Definition: usb.h:83
usb_endpoint_name
const char * usb_endpoint_name(struct usb_endpoint *ep)
Get USB endpoint name (for debugging)
Definition: usb.c:221
xhci_scratchpad
Scratchpad buffer.
Definition: xhci.h:1053
xhci_device::cap
void * cap
Capability registers.
Definition: xhci.h:1080
usb_root_hub_port
struct usb_port * usb_root_hub_port(struct usb_device *usb)
Get USB root hub port.
Definition: usb.c:2356
xhci_device::xecp
unsigned int xecp
xHCI extended capabilities offset
Definition: xhci.h:1100
xhci_abort
static void xhci_abort(struct xhci_device *xhci)
Abort command.
Definition: xhci.c:1814
XHCI_OP_PAGESIZE
#define XHCI_OP_PAGESIZE
Page size register.
Definition: xhci.h:170
XHCI_RUN_ERSTBA
#define XHCI_RUN_ERSTBA(intr)
Event ring segment table base address register.
Definition: xhci.h:299
XHCI_SUPPORTED_REVISION
#define XHCI_SUPPORTED_REVISION
Supported protocol revision.
Definition: xhci.h:104
xhci_trb_normal
A normal transfer request block.
Definition: xhci.h:350
XHCI_TRB_IDT
#define XHCI_TRB_IDT
Transfer request block immediate data flag.
Definition: xhci.h:341
XHCI_XECP_ID_LEGACY
#define XHCI_XECP_ID_LEGACY
USB legacy support extended capability.
Definition: xhci.h:83
xhci_device::addr64
int addr64
64-bit addressing capability
Definition: xhci.h:1096
usb_endpoint::mtu
size_t mtu
Maximum transfer size.
Definition: usb.h:412
XHCI_USBCMD_HCRST
#define XHCI_USBCMD_HCRST
Host controller reset.
Definition: xhci.h:161
status
uint8_t status
Status.
Definition: ena.h:16
xhci_trb_port_status
A port status change transfer request block.
Definition: xhci.h:643
usb_setup_packet::request
uint16_t request
Request.
Definition: usb.h:85
xhci_trb_port_status::port
uint8_t port
Port ID.
Definition: xhci.h:647
xhci_trb::normal
struct xhci_trb_normal normal
Normal TRB.
Definition: xhci.h:689
__startup_fn
struct startup_fn xhci_startup __startup_fn(STARTUP_LATE)
Startup/shutdown function.
xhci_configure_endpoint_input
static void xhci_configure_endpoint_input(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input)
Populate configure endpoint input context.
Definition: xhci.c:2147
xhci_reset
static int xhci_reset(struct xhci_device *xhci)
Reset xHCI device.
Definition: xhci.c:1142
usb_transaction_translator
struct usb_port * usb_transaction_translator(struct usb_device *usb)
Get USB transaction translator.
Definition: usb.c:2372
physaddr_t
unsigned long physaddr_t
Definition: stdint.h:20
xhci_run
static void xhci_run(struct xhci_device *xhci)
Start xHCI device.
Definition: xhci.c:1088
pci_get_drvdata
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:376
XHCI_ENDPOINT_RUNNING
Endpoint is running.
Definition: xhci.h:799
XHCI_DBVAL
#define XHCI_DBVAL(target, stream)
Calculate doorbell register value.
Definition: xhci.h:889
xhci_dequeue_multi
static struct io_buffer * xhci_dequeue_multi(struct xhci_trb_ring *ring)
Dequeue multiple transfer request blocks.
Definition: xhci.c:1420
XHCI_PAGESIZE
#define XHCI_PAGESIZE(pagesize)
Page size.
Definition: xhci.h:173
mdelay
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:79
xhci_trb_stop_endpoint::type
uint8_t type
Type.
Definition: xhci.h:556
XHCI_PORTSC_PLS_RXDETECT
#define XHCI_PORTSC_PLS_RXDETECT
RxDetect port link state.
Definition: xhci.h:222
xhci_trb::reset
struct xhci_trb_reset_endpoint reset
Reset endpoint TRB.
Definition: xhci.h:705
XHCI_TRB_CONFIGURE_ENDPOINT
#define XHCI_TRB_CONFIGURE_ENDPOINT
A configure endpoint transfer request block.
Definition: xhci.h:523
XHCI_SUPPORTED_PORTS_OFFSET
#define XHCI_SUPPORTED_PORTS_OFFSET(ports)
Supported protocol port offset.
Definition: xhci.h:116
XHCI_CMPLT_CMD_STOPPED
Command ring stopped.
Definition: xhci.h:639
xhci_supported_protocol
static unsigned int xhci_supported_protocol(struct xhci_device *xhci, unsigned int port)
Find supported protocol extended capability for a port.
Definition: xhci.c:672
XHCI_OP_CONFIG
#define XHCI_OP_CONFIG
Configure register.
Definition: xhci.h:194
xhci.h
USB eXtensible Host Controller Interface (xHCI) driver.
FILE_LICENCE
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
XHCI_PCH_USB3PSSEN
#define XHCI_PCH_USB3PSSEN
Intel PCH SuperSpeed enable register.
Definition: xhci.h:1036
XHCI_OP_PORTSC
#define XHCI_OP_PORTSC(port)
Port status and control register.
Definition: xhci.h:204
usb_bus_get_hostdata
static void * usb_bus_get_hostdata(struct usb_bus *bus)
Get USB bus host controller private data.
Definition: usb.h:1062
xhci_disable_slot
static int xhci_disable_slot(struct xhci_device *xhci, unsigned int slot)
Disable slot.
Definition: xhci.c:1990
xhci_slot_context::tt_id
uint8_t tt_id
TT hub slot ID.
Definition: xhci.h:749
xhci_pch_fix
static void xhci_pch_fix(struct xhci_device *xhci, struct pci_device *pci)
Fix Intel PCH-specific quirks.
Definition: xhci.c:3388
xhci_endpoint::ep
struct usb_endpoint * ep
USB endpoint.
Definition: xhci.h:1168
xhci_trb_template::control
uint32_t control
Control.
Definition: xhci.h:311
map
static __always_inline int struct dma_mapping * map
Definition: dma.h:184
XHCI_CTX
#define XHCI_CTX(address)
Calculate context index from USB endpoint address.
Definition: xhci.h:946
XHCI_STOP_MAX_WAIT_MS
#define XHCI_STOP_MAX_WAIT_MS
Maximum time to wait for host controller to stop.
Definition: xhci.h:990
XHCI_TRB_NORMAL
#define XHCI_TRB_NORMAL
A normal transfer request block.
Definition: xhci.h:364
ENOBUFS
#define ENOBUFS
No buffer space available.
Definition: errno.h:499
DBGC2
#define DBGC2(...)
Definition: compiler.h:522
xhci_device::pagesize
size_t pagesize
Page size.
Definition: xhci.h:1103
usb.h
Universal Serial Bus (USB)
XHCI_PORT_POWER_DELAY_MS
#define XHCI_PORT_POWER_DELAY_MS
Time to delay after enabling power to a port.
Definition: xhci.h:231
register_usb_bus
int register_usb_bus(struct usb_bus *bus)
Register USB bus.
Definition: usb.c:2131
xhci_event_poll
static void xhci_event_poll(struct xhci_device *xhci)
Poll event ring.
Definition: xhci.c:1745
XHCI_CRCR_RCS
#define XHCI_CRCR_RCS
Command ring cycle state.
Definition: xhci.h:182
xhci_trb_stop_endpoint
A stop endpoint transfer request block.
Definition: xhci.h:548
xhci_trb_ring::mask
unsigned int mask
Ring counter mask.
Definition: xhci.h:848
xhci_scratchpad_free
static void xhci_scratchpad_free(struct xhci_device *xhci)
Free scratchpad buffers.
Definition: xhci.c:1054
XHCI_TRB_TYPE_MASK
#define XHCI_TRB_TYPE_MASK
Transfer request block type mask.
Definition: xhci.h:347
usb_complete
static void usb_complete(struct usb_endpoint *ep, struct io_buffer *iobuf)
Complete transfer (without error)
Definition: usb.h:1087
xhci_endpoint_context::interval
uint8_t interval
Polling interval.
Definition: xhci.h:775
xhci_slot_context::tt_port
uint8_t tt_port
TT port number.
Definition: xhci.h:751
usb_port::address
unsigned int address
Port address.
Definition: usb.h:817
io_buffer::data
void * data
Start of data.
Definition: iobuf.h:53
xhci_host_controller
static void xhci_host_controller(struct xhci_device *xhci, struct xhci_trb_host_controller *trb)
Handle host controller event.
Definition: xhci.c:1730
usb_port::hub
struct usb_hub * hub
USB hub.
Definition: usb.h:815
xhci_dcbaa::map
struct dma_mapping map
DMA mapping.
Definition: xhci.h:1049
xhci_trb_ring::iobuf
struct io_buffer ** iobuf
I/O buffers.
Definition: xhci.h:851
XHCI_TRB_EVALUATE_CONTEXT
#define XHCI_TRB_EVALUATE_CONTEXT
An evaluate context transfer request block.
Definition: xhci.h:526
xhci_complete
static void xhci_complete(struct xhci_device *xhci, struct xhci_trb_complete *trb)
Handle command completion event.
Definition: xhci.c:1669
XHCI_CAP_DBOFF
#define XHCI_CAP_DBOFF
Doorbell offset.
Definition: xhci.h:71
USB_SPEED_HIGH
High speed (480Mbps)
Definition: usb.h:53
usb_device::ep
struct usb_endpoint * ep[32]
Endpoint list.
Definition: usb.h:745
xhci_port_speed
static int xhci_port_speed(struct xhci_device *xhci, unsigned int port, unsigned int psiv)
Find port speed.
Definition: xhci.c:791
xhci_endpoint::ring
struct xhci_trb_ring ring
Transfer ring.
Definition: xhci.h:1178
usb_endpoint::interval
unsigned int interval
Interval (in microframes)
Definition: usb.h:416
pci_device::id
struct pci_device_id * id
Driver device ID.
Definition: pci.h:248
xhci_stop_endpoint
static int xhci_stop_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Stop endpoint.
Definition: xhci.c:2360
xhci_input_context_offset
static size_t xhci_input_context_offset(struct xhci_device *xhci, unsigned int ctx)
Calculate input context offset.
Definition: xhci.c:431
XHCI_HCCPARAMS1_CSZ_SHIFT
#define XHCI_HCCPARAMS1_CSZ_SHIFT(params)
Context size shift.
Definition: xhci.h:65
XHCI_CRCR_CA
#define XHCI_CRCR_CA
Command abort.
Definition: xhci.h:185
usb_host_operations::endpoint
struct usb_endpoint_host_operations endpoint
Endpoint operations.
Definition: usb.h:1033
pci_write_config_dword
int pci_write_config_dword(struct pci_device *pci, unsigned int where, uint32_t value)
Write 32-bit dword to PCI configuration space.
XHCI_PCH_USB3PRM
#define XHCI_PCH_USB3PRM
Intel PCH USB3 port routing mask register.
Definition: xhci.h:1039
xhci_trb_set_tr_dequeue_pointer::type
uint8_t type
Type.
Definition: xhci.h:575
cpu_to_le16
#define cpu_to_le16(value)
Definition: byteswap.h:107
xhci_pch_undo
static void xhci_pch_undo(struct xhci_device *xhci, struct pci_device *pci)
Undo Intel PCH-specific quirk fixes.
Definition: xhci.c:3426
XHCI_USBCMD_RUN
#define XHCI_USBCMD_RUN
Run/stop.
Definition: xhci.h:158
dest
if(len >=6 *4) __asm__ __volatile__("movsl" if(len >=5 *4) __asm__ __volatile__("movsl" if(len >=4 *4) __asm__ __volatile__("movsl" if(len >=3 *4) __asm__ __volatile__("movsl" if(len >=2 *4) __asm__ __volatile__("movsl" if(len >=1 *4) __asm__ __volatile__("movsl" if((len % 4) >=2) __asm__ __volatile__("movsw" if((len % 2) >=1) __asm__ __volatile__("movsb" return dest
Definition: string.h:151
iounmap
void iounmap(volatile const void *io_addr)
Unmap I/O address.
data
uint8_t data[48]
Additional event data.
Definition: ena.h:22
XHCI_TRB_CH
#define XHCI_TRB_CH
Transfer request block chain flag.
Definition: xhci.h:335
XHCI_CAP_HCSPARAMS1
#define XHCI_CAP_HCSPARAMS1
Structural parameters 1.
Definition: xhci.h:40
usb_host_operations
USB host controller operations.
Definition: usb.h:1031
USB_ENDPOINT_ATTR_CONTROL
#define USB_ENDPOINT_ATTR_CONTROL
Control endpoint transfer type.
Definition: usb.h:287
xhci_trb::port
struct xhci_trb_port_status port
Port status changed event.
Definition: xhci.h:715
xhci_device::dma
struct dma_device * dma
DMA device.
Definition: xhci.h:1073
xhci_endpoint_reset
static int xhci_endpoint_reset(struct usb_endpoint *ep)
Reset endpoint.
Definition: xhci.c:2548
xhci_enqueue
static int xhci_enqueue(struct xhci_trb_ring *ring, struct io_buffer *iobuf, const union xhci_trb *trb)
Enqueue a transfer request block.
Definition: xhci.c:1316
xhci_evaluate_context
static int xhci_evaluate_context(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Evaluate context.
Definition: xhci.c:2300
xhci_doorbell
static void xhci_doorbell(struct xhci_trb_ring *ring)
Ring doorbell register.
Definition: xhci.c:1437
xhci_port_protocol
static unsigned int xhci_port_protocol(struct xhci_device *xhci, unsigned int port)
Find port protocol.
Definition: xhci.c:706
xhci_trb_transfer::residual
uint16_t residual
Residual transfer length.
Definition: xhci.h:593
XHCI_SPEED_SUPER
Super speed.
Definition: xhci.h:151
XHCI_PORTSC_PR
#define XHCI_PORTSC_PR
Port reset.
Definition: xhci.h:213
xhci_device::legacy
unsigned int legacy
USB legacy support capability (if present and enabled)
Definition: xhci.h:1106
raw
__be32 raw[7]
Definition: CIB_PRM.h:28
usb_bus::hub
struct usb_hub * hub
Root hub.
Definition: usb.h:995
XHCI_CRCR_CRR
#define XHCI_CRCR_CRR
Command ring running.
Definition: xhci.h:188
XHCI_CONFIG_MAX_SLOTS_EN_MASK
#define XHCI_CONFIG_MAX_SLOTS_EN_MASK
Maximum device slots enabled mask.
Definition: xhci.h:200
XHCI_PORTSC_PED
#define XHCI_PORTSC_PED
Port enabled.
Definition: xhci.h:210
xhci_trb_ring::link
struct xhci_trb_link * link
Link TRB (if applicable)
Definition: xhci.h:860
XHCI_CAP_HCCPARAMS1
#define XHCI_CAP_HCCPARAMS1
Capability parameters.
Definition: xhci.h:59
snprintf
int snprintf(char *buf, size_t size, const char *fmt,...)
Write a formatted string to a buffer.
Definition: vsprintf.c:383
XHCI_SPEED_LOW
Low speed (1.5Mbps)
Definition: xhci.h:147
xhci_trb_ring::shift
unsigned int shift
Ring size (log2)
Definition: xhci.h:846
xhci_trb_ring
A transfer request block command/transfer ring.
Definition: xhci.h:840
usb_hub_set_drvdata
static void usb_hub_set_drvdata(struct usb_hub *hub, void *priv)
Set USB hub driver private data.
Definition: usb.h:937
usb_endpoint::usb
struct usb_device * usb
USB device.
Definition: usb.h:406
xhci_address_device_input
static void xhci_address_device_input(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input)
Populate address device input context.
Definition: xhci.c:2072
xhci_trb_transfer::transfer
uint64_t transfer
Transfer TRB pointer.
Definition: xhci.h:591
usb_port
static struct usb_port * usb_port(struct usb_hub *hub, unsigned int address)
Get USB port.
Definition: usb.h:960
xhci_device::csz_shift
unsigned int csz_shift
Context size shift.
Definition: xhci.h:1098
XHCI_EVENT_TRBS_LOG2
#define XHCI_EVENT_TRBS_LOG2
Number of TRBs in the event ring.
Definition: xhci.h:972
xhci_trb_setup::packet
struct usb_setup_packet packet
Setup packet.
Definition: xhci.h:373
XHCI_SUPPORTED_PSI_MANTISSA
#define XHCI_SUPPORTED_PSI_MANTISSA(psi)
Supported protocol PSI mantissa.
Definition: xhci.h:137
XHCI_MIN_ALIGN
#define XHCI_MIN_ALIGN
Minimum alignment required for data structures.
Definition: xhci.h:25
protocol
uint16_t protocol
Protocol ID.
Definition: stp.h:19
xhci_device::db
void * db
Doorbell registers.
Definition: xhci.h:1086
XHCI_TRB_LINK
#define XHCI_TRB_LINK
A link transfer request block.
Definition: xhci.h:460
xhci_endpoint::interval
unsigned int interval
Endpoint interval.
Definition: xhci.h:1174
XHCI_SETUP_IN
#define XHCI_SETUP_IN
Setup stage input data direction.
Definition: xhci.h:390
XHCI_PORTSC_PLS_DISABLED
#define XHCI_PORTSC_PLS_DISABLED
Disabled port link state.
Definition: xhci.h:219
xhci_reset_endpoint
static int xhci_reset_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Reset endpoint.
Definition: xhci.c:2328
DBG_EXTRA
#define DBG_EXTRA
Definition: compiler.h:319
usb_endpoint_host_operations::open
int(* open)(struct usb_endpoint *ep)
Open endpoint.
Definition: usb.h:450
XHCI_SUPPORTED_REVISION_VER
#define XHCI_SUPPORTED_REVISION_VER(revision)
Supported protocol minor revision.
Definition: xhci.h:107
XHCI_OP_USBCMD
#define XHCI_OP_USBCMD
USB command register.
Definition: xhci.h:155
offset
uint16_t offset
Offset to command line.
Definition: bzimage.h:8
xhci_trb_reset_endpoint::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:539
XHCI_RUN_ERSTSZ
#define XHCI_RUN_ERSTSZ(intr)
Event ring segment table size register.
Definition: xhci.h:296
XHCI_HCSPARAMS2_SCRATCHPADS
#define XHCI_HCSPARAMS2_SCRATCHPADS(params)
Number of page-sized scratchpad buffers.
Definition: xhci.h:55
XHCI_STATUS_OUT
#define XHCI_STATUS_OUT
Output status direction.
Definition: xhci.h:443
xhci_trb_complete::code
uint8_t code
Completion code.
Definition: xhci.h:618
supported
uint16_t supported
Bitmask of supported option values.
Definition: ena.h:12
pci_ioremap
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
XHCI_PCH_XUSB2PRM
#define XHCI_PCH_XUSB2PRM
Intel PCH USB2 port routing mask register.
Definition: xhci.h:1033
XHCI_USBSTS_HCH
#define XHCI_USBSTS_HCH
Host controller halted.
Definition: xhci.h:167
usb_hub::bus
struct usb_bus * bus
USB bus.
Definition: usb.h:845
fls
#define fls(x)
Find last (i.e.
Definition: strings.h:167
le64_to_cpu
#define le64_to_cpu(value)
Definition: byteswap.h:115
xhci_trb::link
struct xhci_trb_link link
Link TRB.
Definition: xhci.h:697
xhci_legacy_init
static void xhci_legacy_init(struct xhci_device *xhci)
Initialise USB legacy support.
Definition: xhci.c:540
DBG_LOG
#define DBG_LOG
Definition: compiler.h:317
xhci_device::slot
struct xhci_slot ** slot
Device slots, indexed by slot ID.
Definition: xhci.h:1124
XHCI_TRB_C
#define XHCI_TRB_C
Transfer request block cycle bit flag.
Definition: xhci.h:329
XHCI_CAP_HCSPARAMS2
#define XHCI_CAP_HCSPARAMS2
Structural parameters 2.
Definition: xhci.h:52
XHCI_SPEED_HIGH
High speed (480Mbps)
Definition: xhci.h:149
dma_mapping
A DMA mapping.
Definition: dma.h:33
xhci_event_alloc
static int xhci_event_alloc(struct xhci_device *xhci)
Allocate event ring.
Definition: xhci.c:1505
xhci_command
static int xhci_command(struct xhci_device *xhci, union xhci_trb *trb)
Issue command and wait for completion.
Definition: xhci.c:1857
xhci_device::quirks
unsigned int quirks
Quirks.
Definition: xhci.h:1077
NULL
#define NULL
NULL pointer (VOID *)
Definition: Base.h:322
xhci_trb_template::parameter
uint64_t parameter
Parameter.
Definition: xhci.h:307
XHCI_SUPPORTED_PSI_EXPONENT
#define XHCI_SUPPORTED_PSI_EXPONENT(psi)
Supported protocol PSI exponent.
Definition: xhci.h:140
xhci_endpoint::xhci
struct xhci_device * xhci
xHCI device
Definition: xhci.h:1164
ETIMEDOUT
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:670
string.h
String functions.
PCI_ROM
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:308
xhci_device::event
struct xhci_event_ring event
Event ring.
Definition: xhci.h:1117
xhci_endpoint_count
static unsigned int xhci_endpoint_count(size_t len, int zlp)
Calculate number of TRBs.
Definition: xhci.c:2674
usb_bus
A USB bus.
Definition: usb.h:966
XHCI_LINK_STATE_DELAY_MS
#define XHCI_LINK_STATE_DELAY_MS
Time to delay after writing the port link state.
Definition: xhci.h:246
usb_endpoint::attributes
unsigned int attributes
Attributes.
Definition: usb.h:410
xhci_legacy_prevent_release
static int xhci_legacy_prevent_release
Prevent the release of ownership back to BIOS.
Definition: xhci.c:533
XHCI_OP_DNCTRL
#define XHCI_OP_DNCTRL
Device notifcation control register.
Definition: xhci.h:176
bus
uint8_t bus
Bus.
Definition: edd.h:15
USB_SPEED_FULL
Full speed (12Mbps)
Definition: usb.h:51
writeq
void writeq(uint64_t data, volatile uint64_t *io_addr)
Write 64-bit qword to memory-mapped device.
dma
physaddr_t dma(struct dma_mapping *map, void *addr)
Get DMA address from virtual address.
xhci_device::op
void * op
Operational registers.
Definition: xhci.h:1082
usb_endpoint::address
unsigned int address
Endpoint address.
Definition: usb.h:408
XHCI_RUN_ERDP
#define XHCI_RUN_ERDP(intr)
Event ring dequeue pointer register.
Definition: xhci.h:302
xhci_trb_enable_slot::type
uint8_t type
Type.
Definition: xhci.h:474
free_usb_bus
void free_usb_bus(struct usb_bus *bus)
Free USB bus.
Definition: usb.c:2195
xhci_nop
static int xhci_nop(struct xhci_device *xhci)
Issue NOP and wait for completion.
Definition: xhci.c:1928
xhci_unregister
void xhci_unregister(struct xhci_device *xhci)
Unregister xHCI controller.
Definition: xhci.c:3364
XHCI_CTX_EP0
#define XHCI_CTX_EP0
Endpoint zero context index.
Definition: xhci.h:951
xhci_slot_context
A slot context.
Definition: xhci.h:739
XHCI_PORTSC_PLS_MASK
#define XHCI_PORTSC_PLS_MASK
Port link state mask.
Definition: xhci.h:225
xhci_trb_set_tr_dequeue_pointer::endpoint
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:577
XHCI_TRB_IOC
#define XHCI_TRB_IOC
Transfer request block interrupt on completion flag.
Definition: xhci.h:338
xhci_hub_open
static int xhci_hub_open(struct usb_hub *hub)
Open hub.
Definition: xhci.c:3033
xhci_trb_context::type
uint8_t type
Type.
Definition: xhci.h:512
xhci_port_status
static void xhci_port_status(struct xhci_device *xhci, struct xhci_trb_port_status *trb)
Handle port status event.
Definition: xhci.c:1706
xhci_ring_alloc
static int xhci_ring_alloc(struct xhci_device *xhci, struct xhci_trb_ring *ring, unsigned int shift, unsigned int slot, unsigned int target, unsigned int stream)
Allocate transfer request block ring.
Definition: xhci.c:1217
usb_complete_err
void usb_complete_err(struct usb_endpoint *ep, struct io_buffer *iobuf, int rc)
Complete transfer (possibly with error)
Definition: usb.c:587
XHCI_ENDPOINT_STATE_MASK
#define XHCI_ENDPOINT_STATE_MASK
Endpoint state mask.
Definition: xhci.h:809
usb_hub_get_drvdata
static void * usb_hub_get_drvdata(struct usb_hub *hub)
Get USB hub driver private data.
Definition: usb.h:948
XHCI_CMD_TRBS_LOG2
#define XHCI_CMD_TRBS_LOG2
Number of TRBs (excluding Link TRB) in the command ring.
Definition: xhci.h:966
xhci_operations
static struct usb_host_operations xhci_operations
USB host controller operations.
Definition: xhci.c:3280
usb_endpoint_get_hostdata
static void * usb_endpoint_get_hostdata(struct usb_endpoint *ep)
Get USB endpoint host controller private data.
Definition: usb.h:587
strings.h
String functions.
xhci_legacy_release
static void xhci_legacy_release(struct xhci_device *xhci)
Release ownership back to BIOS.
Definition: xhci.c:618
xhci_trb_context::slot
uint8_t slot
Slot ID.
Definition: xhci.h:516
xhci_dcbaa_alloc
static int xhci_dcbaa_alloc(struct xhci_device *xhci)
Allocate device context base address array.
Definition: xhci.c:917
XHCI_SUPPORTED_NAME
#define XHCI_SUPPORTED_NAME
Supported protocol name.
Definition: xhci.h:110
memset
void * memset(void *dest, int character, size_t len) __nonnull
XHCI_CTX_SLOT
#define XHCI_CTX_SLOT
Slot context index.
Definition: xhci.h:943
xhci_endpoint_close
static void xhci_endpoint_close(struct usb_endpoint *ep)
Close endpoint.
Definition: xhci.c:2518
io_buffer
A persistent I/O buffer.
Definition: iobuf.h:38
Generated by   doxygen 1.8.15