iPXE
xhci.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <unistd.h>
29 #include <string.h>
30 #include <strings.h>
31 #include <errno.h>
32 #include <byteswap.h>
33 #include <ipxe/malloc.h>
34 #include <ipxe/pci.h>
35 #include <ipxe/usb.h>
36 #include <ipxe/init.h>
37 #include <ipxe/profile.h>
38 #include "xhci.h"
39 
40 /** @file
41  *
42  * USB eXtensible Host Controller Interface (xHCI) driver
43  *
44  */
45 
46 /** Message transfer profiler */
47 static struct profiler xhci_message_profiler __profiler =
48  { .name = "xhci.message" };
49 
50 /** Stream transfer profiler */
51 static struct profiler xhci_stream_profiler __profiler =
52  { .name = "xhci.stream" };
53 
54 /** Event ring profiler */
55 static struct profiler xhci_event_profiler __profiler =
56  { .name = "xhci.event" };
57 
58 /** Transfer event profiler */
59 static struct profiler xhci_transfer_profiler __profiler =
60  { .name = "xhci.transfer" };
61 
62 /* Disambiguate the various error causes */
63 #define EIO_DATA \
64  __einfo_error ( EINFO_EIO_DATA )
65 #define EINFO_EIO_DATA \
66  __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \
67  "Data buffer error" )
68 #define EIO_BABBLE \
69  __einfo_error ( EINFO_EIO_BABBLE )
70 #define EINFO_EIO_BABBLE \
71  __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \
72  "Babble detected" )
73 #define EIO_USB \
74  __einfo_error ( EINFO_EIO_USB )
75 #define EINFO_EIO_USB \
76  __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \
77  "USB transaction error" )
78 #define EIO_TRB \
79  __einfo_error ( EINFO_EIO_TRB )
80 #define EINFO_EIO_TRB \
81  __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \
82  "TRB error" )
83 #define EIO_STALL \
84  __einfo_error ( EINFO_EIO_STALL )
85 #define EINFO_EIO_STALL \
86  __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \
87  "Stall error" )
88 #define EIO_RESOURCE \
89  __einfo_error ( EINFO_EIO_RESOURCE )
90 #define EINFO_EIO_RESOURCE \
91  __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \
92  "Resource error" )
93 #define EIO_BANDWIDTH \
94  __einfo_error ( EINFO_EIO_BANDWIDTH )
95 #define EINFO_EIO_BANDWIDTH \
96  __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \
97  "Bandwidth error" )
98 #define EIO_NO_SLOTS \
99  __einfo_error ( EINFO_EIO_NO_SLOTS )
100 #define EINFO_EIO_NO_SLOTS \
101  __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \
102  "No slots available" )
103 #define EIO_STREAM_TYPE \
104  __einfo_error ( EINFO_EIO_STREAM_TYPE )
105 #define EINFO_EIO_STREAM_TYPE \
106  __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \
107  "Invalid stream type" )
108 #define EIO_SLOT \
109  __einfo_error ( EINFO_EIO_SLOT )
110 #define EINFO_EIO_SLOT \
111  __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \
112  "Slot not enabled" )
113 #define EIO_ENDPOINT \
114  __einfo_error ( EINFO_EIO_ENDPOINT )
115 #define EINFO_EIO_ENDPOINT \
116  __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \
117  "Endpoint not enabled" )
118 #define EIO_SHORT \
119  __einfo_error ( EINFO_EIO_SHORT )
120 #define EINFO_EIO_SHORT \
121  __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \
122  "Short packet" )
123 #define EIO_UNDERRUN \
124  __einfo_error ( EINFO_EIO_UNDERRUN )
125 #define EINFO_EIO_UNDERRUN \
126  __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \
127  "Ring underrun" )
128 #define EIO_OVERRUN \
129  __einfo_error ( EINFO_EIO_OVERRUN )
130 #define EINFO_EIO_OVERRUN \
131  __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \
132  "Ring overrun" )
133 #define EIO_VF_RING_FULL \
134  __einfo_error ( EINFO_EIO_VF_RING_FULL )
135 #define EINFO_EIO_VF_RING_FULL \
136  __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \
137  "Virtual function event ring full" )
138 #define EIO_PARAMETER \
139  __einfo_error ( EINFO_EIO_PARAMETER )
140 #define EINFO_EIO_PARAMETER \
141  __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \
142  "Parameter error" )
143 #define EIO_BANDWIDTH_OVERRUN \
144  __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
145 #define EINFO_EIO_BANDWIDTH_OVERRUN \
146  __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \
147  "Bandwidth overrun" )
148 #define EIO_CONTEXT \
149  __einfo_error ( EINFO_EIO_CONTEXT )
150 #define EINFO_EIO_CONTEXT \
151  __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \
152  "Context state error" )
153 #define EIO_NO_PING \
154  __einfo_error ( EINFO_EIO_NO_PING )
155 #define EINFO_EIO_NO_PING \
156  __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \
157  "No ping response" )
158 #define EIO_RING_FULL \
159  __einfo_error ( EINFO_EIO_RING_FULL )
160 #define EINFO_EIO_RING_FULL \
161  __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \
162  "Event ring full" )
163 #define EIO_INCOMPATIBLE \
164  __einfo_error ( EINFO_EIO_INCOMPATIBLE )
165 #define EINFO_EIO_INCOMPATIBLE \
166  __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \
167  "Incompatible device" )
168 #define EIO_MISSED \
169  __einfo_error ( EINFO_EIO_MISSED )
170 #define EINFO_EIO_MISSED \
171  __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \
172  "Missed service error" )
173 #define EIO_CMD_STOPPED \
174  __einfo_error ( EINFO_EIO_CMD_STOPPED )
175 #define EINFO_EIO_CMD_STOPPED \
176  __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \
177  "Command ring stopped" )
178 #define EIO_CMD_ABORTED \
179  __einfo_error ( EINFO_EIO_CMD_ABORTED )
180 #define EINFO_EIO_CMD_ABORTED \
181  __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \
182  "Command aborted" )
183 #define EIO_STOP \
184  __einfo_error ( EINFO_EIO_STOP )
185 #define EINFO_EIO_STOP \
186  __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \
187  "Stopped" )
188 #define EIO_STOP_LEN \
189  __einfo_error ( EINFO_EIO_STOP_LEN )
190 #define EINFO_EIO_STOP_LEN \
191  __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \
192  "Stopped - length invalid" )
193 #define EIO_STOP_SHORT \
194  __einfo_error ( EINFO_EIO_STOP_SHORT )
195 #define EINFO_EIO_STOP_SHORT \
196  __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \
197  "Stopped - short packet" )
198 #define EIO_LATENCY \
199  __einfo_error ( EINFO_EIO_LATENCY )
200 #define EINFO_EIO_LATENCY \
201  __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \
202  "Maximum exit latency too large" )
203 #define EIO_ISOCH \
204  __einfo_error ( EINFO_EIO_ISOCH )
205 #define EINFO_EIO_ISOCH \
206  __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \
207  "Isochronous buffer overrun" )
208 #define EPROTO_LOST \
209  __einfo_error ( EINFO_EPROTO_LOST )
210 #define EINFO_EPROTO_LOST \
211  __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \
212  "Event lost" )
213 #define EPROTO_UNDEFINED \
214  __einfo_error ( EINFO_EPROTO_UNDEFINED )
215 #define EINFO_EPROTO_UNDEFINED \
216  __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \
217  "Undefined error" )
218 #define EPROTO_STREAM_ID \
219  __einfo_error ( EINFO_EPROTO_STREAM_ID )
220 #define EINFO_EPROTO_STREAM_ID \
221  __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \
222  "Invalid stream ID" )
223 #define EPROTO_SECONDARY \
224  __einfo_error ( EINFO_EPROTO_SECONDARY )
225 #define EINFO_EPROTO_SECONDARY \
226  __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \
227  "Secondary bandwidth error" )
228 #define EPROTO_SPLIT \
229  __einfo_error ( EINFO_EPROTO_SPLIT )
230 #define EINFO_EPROTO_SPLIT \
231  __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \
232  "Split transaction error" )
233 #define ECODE(code) \
234  ( ( (code) < 32 ) ? \
235  EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \
236  EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \
237  EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \
238  EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \
239  EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \
240  EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \
241  EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \
242  EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \
243  EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \
244  EIO_ISOCH ) : \
245  ( (code) < 64 ) ? \
246  EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \
247  EPROTO_UNDEFINED, EPROTO_STREAM_ID, \
248  EPROTO_SECONDARY, EPROTO_SPLIT ) : \
249  EFAULT )
250 
251 /******************************************************************************
252  *
253  * Register access
254  *
255  ******************************************************************************
256  */
257 
258 /**
259  * Initialise device
260  *
261  * @v xhci xHCI device
262  * @v regs MMIO registers
263  */
264 static void xhci_init ( struct xhci_device *xhci, void *regs ) {
265  uint32_t hcsparams1;
266  uint32_t hcsparams2;
267  uint32_t hccparams1;
268  uint32_t pagesize;
269  size_t caplength;
270  size_t rtsoff;
271  size_t dboff;
272 
273  /* Locate capability, operational, runtime, and doorbell registers */
274  xhci->cap = regs;
275  caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
276  rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
277  dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
278  xhci->op = ( xhci->cap + caplength );
279  xhci->run = ( xhci->cap + rtsoff );
280  xhci->db = ( xhci->cap + dboff );
281  DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n",
282  xhci->name, virt_to_phys ( xhci->cap ),
283  virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ),
284  virt_to_phys ( xhci->db ) );
285 
286  /* Read structural parameters 1 */
287  hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
288  xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
289  xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
290  xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
291  DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n",
292  xhci->name, xhci->slots, xhci->intrs, xhci->ports );
293 
294  /* Read structural parameters 2 */
295  hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
296  xhci->scratch.count = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
297  DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
298  xhci->name, xhci->scratch.count );
299 
300  /* Read capability parameters 1 */
301  hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
302  xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
303  xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
304  xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
305 
306  /* Read page size */
307  pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
308  xhci->pagesize = XHCI_PAGESIZE ( pagesize );
309  assert ( xhci->pagesize != 0 );
310  assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
311  DBGC2 ( xhci, "XHCI %s page size %zd bytes\n",
312  xhci->name, xhci->pagesize );
313 }
314 
315 /**
316  * Find extended capability
317  *
318  * @v xhci xHCI device
319  * @v id Capability ID
320  * @v offset Offset to previous extended capability instance, or zero
321  * @ret offset Offset to extended capability, or zero if not found
322  */
323 static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
324  unsigned int id,
325  unsigned int offset ) {
326  uint32_t xecp;
327  unsigned int next;
328 
329  /* Locate the extended capability */
330  while ( 1 ) {
331 
332  /* Locate first or next capability as applicable */
333  if ( offset ) {
334  xecp = readl ( xhci->cap + offset );
335  next = XHCI_XECP_NEXT ( xecp );
336  } else {
337  next = xhci->xecp;
338  }
339  if ( ! next )
340  return 0;
341  offset += next;
342 
343  /* Check if this is the requested capability */
344  xecp = readl ( xhci->cap + offset );
345  if ( XHCI_XECP_ID ( xecp ) == id )
346  return offset;
347  }
348 }
349 
350 /**
351  * Write potentially 64-bit register
352  *
353  * @v xhci xHCI device
354  * @v value Value
355  * @v reg Register address
356  * @ret rc Return status code
357  */
358 static inline __attribute__ (( always_inline )) int
359 xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
360 
361  /* If this is a 32-bit build, then this can never fail
362  * (allowing the compiler to optimise out the error path).
363  */
364  if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
365  writel ( value, reg );
366  writel ( 0, ( reg + sizeof ( uint32_t ) ) );
367  return 0;
368  }
369 
370  /* If the device does not support 64-bit addresses and this
371  * address is outside the 32-bit address space, then fail.
372  */
373  if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
374  DBGC ( xhci, "XHCI %s cannot access address %lx\n",
375  xhci->name, value );
376  return -ENOTSUP;
377  }
378 
379  /* If this is a 64-bit build, then writeq() is available */
380  writeq ( value, reg );
381  return 0;
382 }
383 
384 /**
385  * Calculate buffer alignment
386  *
387  * @v len Length
388  * @ret align Buffer alignment
389  *
390  * Determine alignment required for a buffer which must be aligned to
391  * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
392  */
393 static inline size_t xhci_align ( size_t len ) {
394  size_t align;
395 
396  /* Align to own length (rounded up to a power of two) */
397  align = ( 1 << fls ( len - 1 ) );
398 
399  /* Round up to XHCI_MIN_ALIGN if needed */
400  if ( align < XHCI_MIN_ALIGN )
402 
403  return align;
404 }
405 
406 /**
407  * Calculate device context offset
408  *
409  * @v xhci xHCI device
410  * @v ctx Context index
411  */
412 static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
413  unsigned int ctx ) {
414 
415  return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
416 }
417 
418 /**
419  * Calculate input context offset
420  *
421  * @v xhci xHCI device
422  * @v ctx Context index
423  */
424 static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
425  unsigned int ctx ) {
426 
427  return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
428 }
429 
430 /******************************************************************************
431  *
432  * Diagnostics
433  *
434  ******************************************************************************
435  */
436 
437 /**
438  * Dump host controller registers
439  *
440  * @v xhci xHCI device
441  */
442 static inline void xhci_dump ( struct xhci_device *xhci ) {
443  uint32_t usbcmd;
444  uint32_t usbsts;
445  uint32_t pagesize;
446  uint32_t dnctrl;
447  uint32_t config;
448 
449  /* Do nothing unless debugging is enabled */
450  if ( ! DBG_LOG )
451  return;
452 
453  /* Dump USBCMD */
454  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
455  DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd,
456  ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
457  ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
458 
459  /* Dump USBSTS */
460  usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
461  DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts,
462  ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
463 
464  /* Dump PAGESIZE */
465  pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
466  DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize );
467 
468  /* Dump DNCTRL */
469  dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
470  DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl );
471 
472  /* Dump CONFIG */
473  config = readl ( xhci->op + XHCI_OP_CONFIG );
474  DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config );
475 }
476 
477 /**
478  * Dump port registers
479  *
480  * @v xhci xHCI device
481  * @v port Port number
482  */
483 static inline void xhci_dump_port ( struct xhci_device *xhci,
484  unsigned int port ) {
485  uint32_t portsc;
486  uint32_t portpmsc;
487  uint32_t portli;
488  uint32_t porthlpmc;
489 
490  /* Do nothing unless debugging is enabled */
491  if ( ! DBG_LOG )
492  return;
493 
494  /* Dump PORTSC */
495  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
496  DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n",
497  xhci->name, port, portsc,
498  ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
499  ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
500  ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
501  ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
502  XHCI_PORTSC_PSIV ( portsc ) );
503 
504  /* Dump PORTPMSC */
505  portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
506  DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc );
507 
508  /* Dump PORTLI */
509  portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
510  DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli );
511 
512  /* Dump PORTHLPMC */
513  porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
514  DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n",
515  xhci->name, port, porthlpmc );
516 }
517 
518 /******************************************************************************
519  *
520  * USB legacy support
521  *
522  ******************************************************************************
523  */
524 
525 /** Prevent the release of ownership back to BIOS */
527 
528 /**
529  * Initialise USB legacy support
530  *
531  * @v xhci xHCI device
532  */
533 static void xhci_legacy_init ( struct xhci_device *xhci ) {
534  unsigned int legacy;
535  uint8_t bios;
536 
537  /* Locate USB legacy support capability (if present) */
538  legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
539  if ( ! legacy ) {
540  /* Not an error; capability may not be present */
541  DBGC ( xhci, "XHCI %s has no USB legacy support capability\n",
542  xhci->name );
543  return;
544  }
545 
546  /* Check if legacy USB support is enabled */
547  bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
548  if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
549  /* Not an error; already owned by OS */
550  DBGC ( xhci, "XHCI %s USB legacy support already disabled\n",
551  xhci->name );
552  return;
553  }
554 
555  /* Record presence of USB legacy support capability */
556  xhci->legacy = legacy;
557 }
558 
559 /**
560  * Claim ownership from BIOS
561  *
562  * @v xhci xHCI device
563  */
564 static void xhci_legacy_claim ( struct xhci_device *xhci ) {
565  uint32_t ctlsts;
566  uint8_t bios;
567  unsigned int i;
568 
569  /* Do nothing unless legacy support capability is present */
570  if ( ! xhci->legacy )
571  return;
572 
573  /* Claim ownership */
575  xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
576 
577  /* Wait for BIOS to release ownership */
578  for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
579 
580  /* Check if BIOS has released ownership */
581  bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
582  if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
583  DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n",
584  xhci->name );
585  ctlsts = readl ( xhci->cap + xhci->legacy +
587  if ( ctlsts ) {
588  DBGC ( xhci, "XHCI %s warning: BIOS retained "
589  "SMIs: %08x\n", xhci->name, ctlsts );
590  }
591  return;
592  }
593 
594  /* Delay */
595  mdelay ( 1 );
596  }
597 
598  /* BIOS did not release ownership. Claim it forcibly by
599  * disabling all SMIs.
600  */
601  DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly "
602  "disabling SMIs\n", xhci->name );
603  writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS );
604 }
605 
606 /**
607  * Release ownership back to BIOS
608  *
609  * @v xhci xHCI device
610  */
611 static void xhci_legacy_release ( struct xhci_device *xhci ) {
612 
613  /* Do nothing unless legacy support capability is present */
614  if ( ! xhci->legacy )
615  return;
616 
617  /* Do nothing if releasing ownership is prevented */
619  DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n",
620  xhci->name );
621  return;
622  }
623 
624  /* Release ownership */
625  writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
626  DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name );
627 }
628 
629 /******************************************************************************
630  *
631  * Supported protocols
632  *
633  ******************************************************************************
634  */
635 
636 /**
637  * Transcribe port speed (for debugging)
638  *
639  * @v psi Protocol speed ID
640  * @ret speed Transcribed speed
641  */
642 static inline const char * xhci_speed_name ( uint32_t psi ) {
643  static const char *exponents[4] = { "", "k", "M", "G" };
644  static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
645  unsigned int mantissa;
646  unsigned int exponent;
647 
648  /* Extract mantissa and exponent */
649  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
650  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
651 
652  /* Transcribe speed */
653  snprintf ( buf, sizeof ( buf ), "%d%sbps",
654  mantissa, exponents[exponent] );
655  return buf;
656 }
657 
658 /**
659  * Find supported protocol extended capability for a port
660  *
661  * @v xhci xHCI device
662  * @v port Port number
663  * @ret supported Offset to extended capability, or zero if not found
664  */
665 static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
666  unsigned int port ) {
667  unsigned int supported = 0;
668  unsigned int offset;
669  unsigned int count;
670  uint32_t ports;
671 
672  /* Iterate over all supported protocol structures */
673  while ( ( supported = xhci_extended_capability ( xhci,
675  supported ) ) ) {
676 
677  /* Determine port range */
678  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
680  count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
681 
682  /* Check if port lies within this range */
683  if ( ( port - offset ) < count )
684  return supported;
685  }
686 
687  DBGC ( xhci, "XHCI %s-%d has no supported protocol\n",
688  xhci->name, port );
689  return 0;
690 }
691 
692 /**
693  * Find port protocol
694  *
695  * @v xhci xHCI device
696  * @v port Port number
697  * @ret protocol USB protocol, or zero if not found
698  */
699 static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
700  unsigned int port ) {
701  unsigned int supported = xhci_supported_protocol ( xhci, port );
702  union {
703  uint32_t raw;
704  char text[5];
705  } name;
706  unsigned int protocol;
707  unsigned int type;
708  unsigned int psic;
709  unsigned int psiv;
710  unsigned int i;
712  uint32_t ports;
713  uint32_t slot;
714  uint32_t psi;
715 
716  /* Fail if there is no supported protocol */
717  if ( ! supported )
718  return 0;
719 
720  /* Determine protocol version */
721  revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
723 
724  /* Describe port protocol */
725  if ( DBG_EXTRA ) {
726  name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
728  name.text[4] = '\0';
729  slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
731  DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d",
732  xhci->name, port, name.text, protocol, type );
733  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
734  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
735  if ( psic ) {
736  DBGC2 ( xhci, " speeds" );
737  for ( i = 0 ; i < psic ; i++ ) {
738  psi = readl ( xhci->cap + supported +
739  XHCI_SUPPORTED_PSI ( i ) );
740  psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
741  DBGC2 ( xhci, " %d:%s", psiv,
742  xhci_speed_name ( psi ) );
743  }
744  }
745  if ( xhci->quirks & XHCI_BAD_PSIV )
746  DBGC2 ( xhci, " (ignored)" );
747  DBGC2 ( xhci, "\n" );
748  }
749 
750  return protocol;
751 }
752 
753 /**
754  * Find port slot type
755  *
756  * @v xhci xHCI device
757  * @v port Port number
758  * @ret type Slot type, or negative error
759  */
760 static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
761  unsigned int supported = xhci_supported_protocol ( xhci, port );
762  unsigned int type;
763  uint32_t slot;
764 
765  /* Fail if there is no supported protocol */
766  if ( ! supported )
767  return -ENOTSUP;
768 
769  /* Get slot type */
770  slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
772 
773  return type;
774 }
775 
776 /**
777  * Find port speed
778  *
779  * @v xhci xHCI device
780  * @v port Port number
781  * @v psiv Protocol speed ID value
782  * @ret speed Port speed, or negative error
783  */
784 static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
785  unsigned int psiv ) {
786  unsigned int supported = xhci_supported_protocol ( xhci, port );
787  unsigned int psic;
788  unsigned int mantissa;
789  unsigned int exponent;
790  unsigned int speed;
791  unsigned int i;
792  uint32_t ports;
793  uint32_t psi;
794 
795  /* Fail if there is no supported protocol */
796  if ( ! supported )
797  return -ENOTSUP;
798 
799  /* Get protocol speed ID count */
800  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
801  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
802 
803  /* Use protocol speed ID table unless device is known to be faulty */
804  if ( ! ( xhci->quirks & XHCI_BAD_PSIV ) ) {
805 
806  /* Iterate over PSI dwords looking for a match */
807  for ( i = 0 ; i < psic ; i++ ) {
808  psi = readl ( xhci->cap + supported +
809  XHCI_SUPPORTED_PSI ( i ) );
810  if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
811  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
812  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
813  speed = USB_SPEED ( mantissa, exponent );
814  return speed;
815  }
816  }
817 
818  /* Record device as faulty if no match is found */
819  if ( psic != 0 ) {
820  DBGC ( xhci, "XHCI %s-%d spurious PSI value %d: "
821  "assuming PSI table is invalid\n",
822  xhci->name, port, psiv );
823  xhci->quirks |= XHCI_BAD_PSIV;
824  }
825  }
826 
827  /* Use the default mappings */
828  switch ( psiv ) {
829  case XHCI_SPEED_LOW : return USB_SPEED_LOW;
830  case XHCI_SPEED_FULL : return USB_SPEED_FULL;
831  case XHCI_SPEED_HIGH : return USB_SPEED_HIGH;
832  case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
833  default:
834  DBGC ( xhci, "XHCI %s-%d unrecognised PSI value %d\n",
835  xhci->name, port, psiv );
836  return -ENOTSUP;
837  }
838 }
839 
840 /**
841  * Find protocol speed ID value
842  *
843  * @v xhci xHCI device
844  * @v port Port number
845  * @v speed USB speed
846  * @ret psiv Protocol speed ID value, or negative error
847  */
848 static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
849  unsigned int speed ) {
850  unsigned int supported = xhci_supported_protocol ( xhci, port );
851  unsigned int psic;
852  unsigned int mantissa;
853  unsigned int exponent;
854  unsigned int psiv;
855  unsigned int i;
856  uint32_t ports;
857  uint32_t psi;
858 
859  /* Fail if there is no supported protocol */
860  if ( ! supported )
861  return -ENOTSUP;
862 
863  /* Get protocol speed ID count */
864  ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
865  psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
866 
867  /* Use the default mappings if applicable */
868  if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
869  switch ( speed ) {
870  case USB_SPEED_LOW : return XHCI_SPEED_LOW;
871  case USB_SPEED_FULL : return XHCI_SPEED_FULL;
872  case USB_SPEED_HIGH : return XHCI_SPEED_HIGH;
873  case USB_SPEED_SUPER : return XHCI_SPEED_SUPER;
874  default:
875  DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n",
876  xhci->name, port, speed );
877  return -ENOTSUP;
878  }
879  }
880 
881  /* Iterate over PSI dwords looking for a match */
882  for ( i = 0 ; i < psic ; i++ ) {
883  psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
884  mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
885  exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
886  if ( speed == USB_SPEED ( mantissa, exponent ) ) {
887  psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
888  return psiv;
889  }
890  }
891 
892  DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n",
893  xhci->name, port, speed );
894  return -ENOENT;
895 }
896 
897 /******************************************************************************
898  *
899  * Device context base address array
900  *
901  ******************************************************************************
902  */
903 
904 /**
905  * Allocate device context base address array
906  *
907  * @v xhci xHCI device
908  * @ret rc Return status code
909  */
910 static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
911  size_t len;
912  physaddr_t dcbaap;
913  int rc;
914 
915  /* Allocate and initialise structure. Must be at least
916  * 64-byte aligned and must not cross a page boundary, so
917  * align on its own size (rounded up to a power of two and
918  * with a minimum of 64 bytes).
919  */
920  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
921  xhci->dcbaa.context = dma_alloc ( xhci->dma, &xhci->dcbaa.map, len,
922  xhci_align ( len ) );
923  if ( ! xhci->dcbaa.context ) {
924  DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
925  rc = -ENOMEM;
926  goto err_alloc;
927  }
928  memset ( xhci->dcbaa.context, 0, len );
929 
930  /* Program DCBAA pointer */
931  dcbaap = dma ( &xhci->dcbaa.map, xhci->dcbaa.context );
932  if ( ( rc = xhci_writeq ( xhci, dcbaap,
933  xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
934  goto err_writeq;
935 
936  DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n", xhci->name,
937  virt_to_phys ( xhci->dcbaa.context ),
938  ( virt_to_phys ( xhci->dcbaa.context ) + len ) );
939  return 0;
940 
941  err_writeq:
942  dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
943  err_alloc:
944  return rc;
945 }
946 
947 /**
948  * Free device context base address array
949  *
950  * @v xhci xHCI device
951  */
952 static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
953  size_t len;
954  unsigned int i;
955 
956  /* Sanity check */
957  for ( i = 0 ; i <= xhci->slots ; i++ )
958  assert ( xhci->dcbaa.context[i] == 0 );
959 
960  /* Clear DCBAA pointer */
961  xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
962 
963  /* Free DCBAA */
964  len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa.context[0] ) );
965  dma_free ( &xhci->dcbaa.map, xhci->dcbaa.context, len );
966 }
967 
968 /******************************************************************************
969  *
970  * Scratchpad buffers
971  *
972  ******************************************************************************
973  */
974 
975 /**
976  * Allocate scratchpad buffers
977  *
978  * @v xhci xHCI device
979  * @ret rc Return status code
980  */
981 static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
982  struct xhci_scratchpad *scratch = &xhci->scratch;
983  size_t buffer_len;
984  size_t array_len;
986  unsigned int i;
987  int rc;
988 
989  /* Do nothing if no scratchpad buffers are used */
990  if ( ! scratch->count )
991  return 0;
992 
993  /* Allocate scratchpad buffers */
994  buffer_len = ( scratch->count * xhci->pagesize );
995  scratch->buffer = dma_umalloc ( xhci->dma, &scratch->buffer_map,
996  buffer_len, xhci->pagesize );
997  if ( ! scratch->buffer ) {
998  DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
999  xhci->name );
1000  rc = -ENOMEM;
1001  goto err_alloc;
1002  }
1003  memset_user ( scratch->buffer, 0, 0, buffer_len );
1004 
1005  /* Allocate scratchpad array */
1006  array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
1007  scratch->array = dma_alloc ( xhci->dma, &scratch->array_map,
1008  array_len, xhci_align ( array_len ) );
1009  if ( ! scratch->array ) {
1010  DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
1011  "array\n", xhci->name );
1012  rc = -ENOMEM;
1013  goto err_alloc_array;
1014  }
1015 
1016  /* Populate scratchpad array */
1017  addr = dma_phys ( &scratch->buffer_map,
1018  user_to_phys ( scratch->buffer, 0 ) );
1019  for ( i = 0 ; i < scratch->count ; i++ ) {
1020  scratch->array[i] = cpu_to_le64 ( addr );
1021  addr += xhci->pagesize;
1022  }
1023 
1024  /* Set scratchpad array pointer */
1025  assert ( xhci->dcbaa.context != NULL );
1026  xhci->dcbaa.context[0] = cpu_to_le64 ( dma ( &scratch->array_map,
1027  scratch->array ) );
1028 
1029  DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
1030  xhci->name, user_to_phys ( scratch->buffer, 0 ),
1031  user_to_phys ( scratch->buffer, buffer_len ),
1032  virt_to_phys ( scratch->array ),
1033  ( virt_to_phys ( scratch->array ) + array_len ) );
1034  return 0;
1035 
1036  dma_free ( &scratch->array_map, scratch->array, array_len );
1037  err_alloc_array:
1038  dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
1039  err_alloc:
1040  return rc;
1041 }
1042 
1043 /**
1044  * Free scratchpad buffers
1045  *
1046  * @v xhci xHCI device
1047  */
1048 static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
1049  struct xhci_scratchpad *scratch = &xhci->scratch;
1050  size_t array_len;
1051  size_t buffer_len;
1052 
1053  /* Do nothing if no scratchpad buffers are used */
1054  if ( ! scratch->count )
1055  return;
1056 
1057  /* Clear scratchpad array pointer */
1058  assert ( xhci->dcbaa.context != NULL );
1059  xhci->dcbaa.context[0] = 0;
1060 
1061  /* Free scratchpad array */
1062  array_len = ( scratch->count * sizeof ( scratch->array[0] ) );
1063  dma_free ( &scratch->array_map, scratch->array, array_len );
1064 
1065  /* Free scratchpad buffers */
1066  buffer_len = ( scratch->count * xhci->pagesize );
1067  dma_ufree ( &scratch->buffer_map, scratch->buffer, buffer_len );
1068 }
1069 
1070 /******************************************************************************
1071  *
1072  * Run / stop / reset
1073  *
1074  ******************************************************************************
1075  */
1076 
1077 /**
1078  * Start xHCI device
1079  *
1080  * @v xhci xHCI device
1081  */
1082 static void xhci_run ( struct xhci_device *xhci ) {
1083  uint32_t config;
1084  uint32_t usbcmd;
1085 
1086  /* Configure number of device slots */
1087  config = readl ( xhci->op + XHCI_OP_CONFIG );
1088  config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
1089  config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
1090  writel ( config, xhci->op + XHCI_OP_CONFIG );
1091 
1092  /* Set run/stop bit */
1093  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1094  usbcmd |= XHCI_USBCMD_RUN;
1095  writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
1096 }
1097 
1098 /**
1099  * Stop xHCI device
1100  *
1101  * @v xhci xHCI device
1102  * @ret rc Return status code
1103  */
1104 static int xhci_stop ( struct xhci_device *xhci ) {
1105  uint32_t usbcmd;
1106  uint32_t usbsts;
1107  unsigned int i;
1108 
1109  /* Clear run/stop bit */
1110  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1111  usbcmd &= ~XHCI_USBCMD_RUN;
1112  writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
1113 
1114  /* Wait for device to stop */
1115  for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
1116 
1117  /* Check if device is stopped */
1118  usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
1119  if ( usbsts & XHCI_USBSTS_HCH )
1120  return 0;
1121 
1122  /* Delay */
1123  mdelay ( 1 );
1124  }
1125 
1126  DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name );
1127  return -ETIMEDOUT;
1128 }
1129 
1130 /**
1131  * Reset xHCI device
1132  *
1133  * @v xhci xHCI device
1134  * @ret rc Return status code
1135  */
1136 static int xhci_reset ( struct xhci_device *xhci ) {
1137  uint32_t usbcmd;
1138  unsigned int i;
1139  int rc;
1140 
1141  /* The xHCI specification states that resetting a running
1142  * device may result in undefined behaviour, so try stopping
1143  * it first.
1144  */
1145  if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
1146  /* Ignore errors and attempt to reset the device anyway */
1147  }
1148 
1149  /* Reset device */
1151 
1152  /* Wait for reset to complete */
1153  for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
1154 
1155  /* Check if reset is complete */
1156  usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
1157  if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
1158  return 0;
1159 
1160  /* Delay */
1161  mdelay ( 1 );
1162  }
1163 
1164  DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name );
1165  return -ETIMEDOUT;
1166 }
1167 
1168 /******************************************************************************
1169  *
1170  * Transfer request blocks
1171  *
1172  ******************************************************************************
1173  */
1174 
1175 /**
1176  * Allocate transfer request block ring
1177  *
1178  * @v xhci xHCI device
1179  * @v ring TRB ring
1180  * @v shift Ring size (log2)
1181  * @v slot Device slot
1182  * @v target Doorbell target
1183  * @v stream Doorbell stream ID
1184  * @ret rc Return status code
1185  */
1186 static int xhci_ring_alloc ( struct xhci_device *xhci,
1187  struct xhci_trb_ring *ring,
1188  unsigned int shift, unsigned int slot,
1189  unsigned int target, unsigned int stream ) {
1190  struct xhci_trb_link *link;
1191  unsigned int count;
1192  int rc;
1193 
1194  /* Sanity check */
1195  assert ( shift > 0 );
1196 
1197  /* Initialise structure */
1198  memset ( ring, 0, sizeof ( *ring ) );
1199  ring->shift = shift;
1200  count = ( 1U << shift );
1201  ring->mask = ( count - 1 );
1202  ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
1203  ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
1204  ring->dbval = XHCI_DBVAL ( target, stream );
1205 
1206  /* Allocate I/O buffers */
1207  ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
1208  if ( ! ring->iobuf ) {
1209  rc = -ENOMEM;
1210  goto err_alloc_iobuf;
1211  }
1212 
1213  /* Allocate TRBs */
1214  ring->trb = dma_alloc ( xhci->dma, &ring->map, ring->len,
1215  xhci_align ( ring->len ) );
1216  if ( ! ring->trb ) {
1217  rc = -ENOMEM;
1218  goto err_alloc_trb;
1219  }
1220  memset ( ring->trb, 0, ring->len );
1221 
1222  /* Initialise Link TRB */
1223  link = &ring->trb[count].link;
1224  link->next = cpu_to_le64 ( dma ( &ring->map, ring->trb ) );
1225  link->flags = XHCI_TRB_TC;
1226  link->type = XHCI_TRB_LINK;
1227  ring->link = link;
1228 
1229  return 0;
1230 
1231  dma_free ( &ring->map, ring->trb, ring->len );
1232  err_alloc_trb:
1233  free ( ring->iobuf );
1234  err_alloc_iobuf:
1235  return rc;
1236 }
1237 
1238 /**
1239  * Reset transfer request block ring
1240  *
1241  * @v ring TRB ring
1242  */
1243 static void xhci_ring_reset ( struct xhci_trb_ring *ring ) {
1244  unsigned int count = ( 1U << ring->shift );
1245 
1246  /* Reset producer and consumer counters */
1247  ring->prod = 0;
1248  ring->cons = 0;
1249 
1250  /* Reset TRBs (except Link TRB) */
1251  memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) );
1252 }
1253 
1254 /**
1255  * Free transfer request block ring
1256  *
1257  * @v ring TRB ring
1258  */
1259 static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
1260  unsigned int count = ( 1U << ring->shift );
1261  unsigned int i;
1262 
1263  /* Sanity checks */
1264  assert ( ring->cons == ring->prod );
1265  for ( i = 0 ; i < count ; i++ )
1266  assert ( ring->iobuf[i] == NULL );
1267 
1268  /* Free TRBs */
1269  dma_free ( &ring->map, ring->trb, ring->len );
1270 
1271  /* Free I/O buffers */
1272  free ( ring->iobuf );
1273 }
1274 
1275 /**
1276  * Enqueue a transfer request block
1277  *
1278  * @v ring TRB ring
1279  * @v iobuf I/O buffer (if any)
1280  * @v trb Transfer request block (with empty Cycle flag)
1281  * @ret rc Return status code
1282  *
1283  * This operation does not implicitly ring the doorbell register.
1284  */
1285 static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
1286  const union xhci_trb *trb ) {
1287  union xhci_trb *dest;
1288  unsigned int prod;
1289  unsigned int mask;
1290  unsigned int index;
1291  unsigned int cycle;
1292 
1293  /* Sanity check */
1294  assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
1295 
1296  /* Fail if ring is full */
1297  if ( ! xhci_ring_remaining ( ring ) )
1298  return -ENOBUFS;
1299 
1300  /* Update producer counter (and link TRB, if applicable) */
1301  prod = ring->prod++;
1302  mask = ring->mask;
1303  cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
1304  index = ( prod & mask );
1305  if ( index == 0 )
1306  ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
1307 
1308  /* Record I/O buffer */
1309  ring->iobuf[index] = iobuf;
1310 
1311  /* Enqueue TRB */
1312  dest = &ring->trb[index];
1313  dest->template.parameter = trb->template.parameter;
1314  dest->template.status = trb->template.status;
1315  wmb();
1316  dest->template.control = ( trb->template.control |
1317  cpu_to_le32 ( cycle ) );
1318 
1319  return 0;
1320 }
1321 
1322 /**
1323  * Dequeue a transfer request block
1324  *
1325  * @v ring TRB ring
1326  * @ret iobuf I/O buffer
1327  */
1328 static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
1329  struct io_buffer *iobuf;
1330  unsigned int cons;
1331  unsigned int mask;
1332  unsigned int index;
1333 
1334  /* Sanity check */
1335  assert ( xhci_ring_fill ( ring ) != 0 );
1336 
1337  /* Update consumer counter */
1338  cons = ring->cons++;
1339  mask = ring->mask;
1340  index = ( cons & mask );
1341 
1342  /* Retrieve I/O buffer */
1343  iobuf = ring->iobuf[index];
1344  ring->iobuf[index] = NULL;
1345 
1346  return iobuf;
1347 }
1348 
1349 /**
1350  * Enqueue multiple transfer request blocks
1351  *
1352  * @v ring TRB ring
1353  * @v iobuf I/O buffer
1354  * @v trbs Transfer request blocks (with empty Cycle flag)
1355  * @v count Number of transfer request blocks
1356  * @ret rc Return status code
1357  *
1358  * This operation does not implicitly ring the doorbell register.
1359  */
1360 static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
1361  struct io_buffer *iobuf,
1362  const union xhci_trb *trbs,
1363  unsigned int count ) {
1364  const union xhci_trb *trb = trbs;
1365  int rc;
1366 
1367  /* Sanity check */
1368  assert ( iobuf != NULL );
1369 
1370  /* Fail if ring does not have sufficient space */
1371  if ( xhci_ring_remaining ( ring ) < count )
1372  return -ENOBUFS;
1373 
1374  /* Enqueue each TRB, recording the I/O buffer with the final TRB */
1375  while ( count-- ) {
1376  rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
1377  assert ( rc == 0 ); /* Should never be able to fail */
1378  }
1379 
1380  return 0;
1381 }
1382 
1383 /**
1384  * Dequeue multiple transfer request blocks
1385  *
1386  * @v ring TRB ring
1387  * @ret iobuf I/O buffer
1388  */
1389 static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
1390  struct io_buffer *iobuf;
1391 
1392  /* Dequeue TRBs until we reach the final TRB for an I/O buffer */
1393  do {
1394  iobuf = xhci_dequeue ( ring );
1395  } while ( iobuf == NULL );
1396 
1397  return iobuf;
1398 }
1399 
1400 /**
1401  * Ring doorbell register
1402  *
1403  * @v ring TRB ring
1404  */
1405 static inline __attribute__ (( always_inline )) void
1406 xhci_doorbell ( struct xhci_trb_ring *ring ) {
1407 
1408  wmb();
1409  writel ( ring->dbval, ring->db );
1410 }
1411 
1412 /******************************************************************************
1413  *
1414  * Command and event rings
1415  *
1416  ******************************************************************************
1417  */
1418 
1419 /**
1420  * Allocate command ring
1421  *
1422  * @v xhci xHCI device
1423  * @ret rc Return status code
1424  */
1425 static int xhci_command_alloc ( struct xhci_device *xhci ) {
1426  physaddr_t crp;
1427  int rc;
1428 
1429  /* Allocate TRB ring */
1430  if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
1431  0, 0, 0 ) ) != 0 )
1432  goto err_ring_alloc;
1433 
1434  /* Program command ring control register */
1435  crp = dma ( &xhci->command.map, xhci->command.trb );
1436  if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
1437  xhci->op + XHCI_OP_CRCR ) ) != 0 )
1438  goto err_writeq;
1439 
1440  DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n", xhci->name,
1441  virt_to_phys ( xhci->command.trb ),
1442  ( virt_to_phys ( xhci->command.trb ) + xhci->command.len ) );
1443  return 0;
1444 
1445  err_writeq:
1446  xhci_ring_free ( &xhci->command );
1447  err_ring_alloc:
1448  return rc;
1449 }
1450 
1451 /**
1452  * Free command ring
1453  *
1454  * @v xhci xHCI device
1455  */
1456 static void xhci_command_free ( struct xhci_device *xhci ) {
1457 
1458  /* Sanity check */
1459  assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
1460 
1461  /* Clear command ring control register */
1462  xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
1463 
1464  /* Free TRB ring */
1465  xhci_ring_free ( &xhci->command );
1466 }
1467 
1468 /**
1469  * Allocate event ring
1470  *
1471  * @v xhci xHCI device
1472  * @ret rc Return status code
1473  */
1474 static int xhci_event_alloc ( struct xhci_device *xhci ) {
1475  struct xhci_event_ring *event = &xhci->event;
1476  unsigned int count;
1477  size_t len;
1478  int rc;
1479 
1480  /* Allocate event ring */
1481  count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
1482  len = ( count * sizeof ( event->trb[0] ) );
1483  event->trb = dma_alloc ( xhci->dma, &event->trb_map, len,
1484  xhci_align ( len ) );
1485  if ( ! event->trb ) {
1486  rc = -ENOMEM;
1487  goto err_alloc_trb;
1488  }
1489  memset ( event->trb, 0, len );
1490 
1491  /* Allocate event ring segment table */
1492  event->segment = dma_alloc ( xhci->dma, &event->segment_map,
1493  sizeof ( event->segment[0] ),
1494  xhci_align ( sizeof (event->segment[0])));
1495  if ( ! event->segment ) {
1496  rc = -ENOMEM;
1497  goto err_alloc_segment;
1498  }
1499  memset ( event->segment, 0, sizeof ( event->segment[0] ) );
1500  event->segment[0].base = cpu_to_le64 ( dma ( &event->trb_map,
1501  event->trb ) );
1502  event->segment[0].count = cpu_to_le32 ( count );
1503 
1504  /* Program event ring registers */
1505  writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
1506  if ( ( rc = xhci_writeq ( xhci, dma ( &event->trb_map, event->trb ),
1507  xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
1508  goto err_writeq_erdp;
1509  if ( ( rc = xhci_writeq ( xhci,
1510  dma ( &event->segment_map, event->segment ),
1511  xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
1512  goto err_writeq_erstba;
1513 
1514  DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
1515  xhci->name, virt_to_phys ( event->trb ),
1516  ( virt_to_phys ( event->trb ) + len ),
1517  virt_to_phys ( event->segment ),
1518  ( virt_to_phys ( event->segment ) +
1519  sizeof ( event->segment[0] ) ) );
1520  return 0;
1521 
1522  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
1523  err_writeq_erstba:
1524  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
1525  err_writeq_erdp:
1526  dma_free ( &event->segment_map, event->segment,
1527  sizeof ( event->segment[0] ) );
1528  err_alloc_segment:
1529  dma_free ( &event->trb_map, event->trb, len );
1530  err_alloc_trb:
1531  return rc;
1532 }
1533 
1534 /**
1535  * Free event ring
1536  *
1537  * @v xhci xHCI device
1538  */
1539 static void xhci_event_free ( struct xhci_device *xhci ) {
1540  struct xhci_event_ring *event = &xhci->event;
1541  unsigned int count;
1542  size_t len;
1543 
1544  /* Clear event ring registers */
1545  writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
1546  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
1547  xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
1548 
1549  /* Free event ring segment table */
1550  dma_free ( &event->segment_map, event->segment,
1551  sizeof ( event->segment[0] ) );
1552 
1553  /* Free event ring */
1554  count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
1555  len = ( count * sizeof ( event->trb[0] ) );
1556  dma_free ( &event->trb_map, event->trb, len );
1557 }
1558 
1559 /**
1560  * Handle transfer event
1561  *
1562  * @v xhci xHCI device
1563  * @v trb Transfer event TRB
1564  */
1565 static void xhci_transfer ( struct xhci_device *xhci,
1566  struct xhci_trb_transfer *trb ) {
1567  struct xhci_slot *slot;
1568  struct xhci_endpoint *endpoint;
1569  struct io_buffer *iobuf;
1570  int rc;
1571 
1572  /* Profile transfer events */
1573  profile_start ( &xhci_transfer_profiler );
1574 
1575  /* Identify slot */
1576  if ( ( trb->slot > xhci->slots ) ||
1577  ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) {
1578  DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n",
1579  xhci->name, trb->slot );
1580  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1581  return;
1582  }
1583 
1584  /* Identify endpoint */
1585  if ( ( trb->endpoint >= XHCI_CTX_END ) ||
1586  ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) {
1587  DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid "
1588  "%d:\n", xhci->name, slot->id, trb->endpoint );
1589  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1590  return;
1591  }
1592 
1593  /* Dequeue TRB(s) */
1594  iobuf = xhci_dequeue_multi ( &endpoint->ring );
1595  assert ( iobuf != NULL );
1596 
1597  /* Unmap I/O buffer */
1598  iob_unmap ( iobuf );
1599 
1600  /* Check for errors */
1601  if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
1602  ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
1603 
1604  /* Construct error */
1605  rc = -ECODE ( trb->code );
1606  DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n",
1607  xhci->name, slot->id, endpoint->ctx, trb->code,
1608  strerror ( rc ) );
1609  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1610 
1611  /* Sanity check */
1612  assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
1613  != XHCI_ENDPOINT_RUNNING );
1614 
1615  /* Report failure to USB core */
1616  usb_complete_err ( endpoint->ep, iobuf, rc );
1617  return;
1618  }
1619 
1620  /* Record actual transfer size */
1621  iob_unput ( iobuf, le16_to_cpu ( trb->residual ) );
1622 
1623  /* Sanity check (for successful completions only) */
1624  assert ( xhci_ring_consumed ( &endpoint->ring ) ==
1625  le64_to_cpu ( trb->transfer ) );
1626 
1627  /* Report completion to USB core */
1628  usb_complete ( endpoint->ep, iobuf );
1629  profile_stop ( &xhci_transfer_profiler );
1630 }
1631 
1632 /**
1633  * Handle command completion event
1634  *
1635  * @v xhci xHCI device
1636  * @v trb Command completion event
1637  */
1638 static void xhci_complete ( struct xhci_device *xhci,
1639  struct xhci_trb_complete *trb ) {
1640  int rc;
1641 
1642  /* Ignore "command ring stopped" notifications */
1643  if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) {
1644  DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name );
1645  return;
1646  }
1647 
1648  /* Ignore unexpected completions */
1649  if ( ! xhci->pending ) {
1650  rc = -ECODE ( trb->code );
1651  DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n",
1652  xhci->name, trb->code, strerror ( rc ) );
1653  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1654  return;
1655  }
1656 
1657  /* Dequeue command TRB */
1658  xhci_dequeue ( &xhci->command );
1659 
1660  /* Sanity check */
1661  assert ( xhci_ring_consumed ( &xhci->command ) ==
1662  le64_to_cpu ( trb->command ) );
1663 
1664  /* Record completion */
1665  memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) );
1666  xhci->pending = NULL;
1667 }
1668 
1669 /**
1670  * Handle port status event
1671  *
1672  * @v xhci xHCI device
1673  * @v trb Port status event
1674  */
1675 static void xhci_port_status ( struct xhci_device *xhci,
1676  struct xhci_trb_port_status *trb ) {
1677  struct usb_port *port = usb_port ( xhci->bus->hub, trb->port );
1678  uint32_t portsc;
1679 
1680  /* Sanity check */
1681  assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) );
1682 
1683  /* Record disconnections and clear changes */
1684  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) );
1685  port->disconnected |= ( portsc & XHCI_PORTSC_CSC );
1686  portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
1687  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) );
1688 
1689  /* Report port status change */
1690  usb_port_changed ( port );
1691 }
1692 
1693 /**
1694  * Handle host controller event
1695  *
1696  * @v xhci xHCI device
1697  * @v trb Host controller event
1698  */
1699 static void xhci_host_controller ( struct xhci_device *xhci,
1700  struct xhci_trb_host_controller *trb ) {
1701  int rc;
1702 
1703  /* Construct error */
1704  rc = -ECODE ( trb->code );
1705  DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n",
1706  xhci->name, trb->code, strerror ( rc ) );
1707 }
1708 
1709 /**
1710  * Poll event ring
1711  *
1712  * @v xhci xHCI device
1713  */
1714 static void xhci_event_poll ( struct xhci_device *xhci ) {
1715  struct xhci_event_ring *event = &xhci->event;
1716  union xhci_trb *trb;
1717  unsigned int shift = XHCI_EVENT_TRBS_LOG2;
1718  unsigned int count = ( 1 << shift );
1719  unsigned int mask = ( count - 1 );
1720  unsigned int consumed;
1721  unsigned int type;
1722 
1723  /* Poll for events */
1724  profile_start ( &xhci_event_profiler );
1725  for ( consumed = 0 ; ; consumed++ ) {
1726 
1727  /* Stop if we reach an empty TRB */
1728  rmb();
1729  trb = &event->trb[ event->cons & mask ];
1730  if ( ! ( ( trb->common.flags ^
1731  ( event->cons >> shift ) ) & XHCI_TRB_C ) )
1732  break;
1733 
1734  /* Consume this TRB */
1735  event->cons++;
1736 
1737  /* Handle TRB */
1738  type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
1739  switch ( type ) {
1740 
1741  case XHCI_TRB_TRANSFER :
1742  xhci_transfer ( xhci, &trb->transfer );
1743  break;
1744 
1745  case XHCI_TRB_COMPLETE :
1746  xhci_complete ( xhci, &trb->complete );
1747  break;
1748 
1749  case XHCI_TRB_PORT_STATUS:
1750  xhci_port_status ( xhci, &trb->port );
1751  break;
1752 
1754  xhci_host_controller ( xhci, &trb->host );
1755  break;
1756 
1757  default:
1758  DBGC ( xhci, "XHCI %s unrecognised event %#x\n:",
1759  xhci->name, ( event->cons - 1 ) );
1760  DBGC_HDA ( xhci, virt_to_phys ( trb ),
1761  trb, sizeof ( *trb ) );
1762  break;
1763  }
1764  }
1765 
1766  /* Update dequeue pointer if applicable */
1767  if ( consumed ) {
1768  xhci_writeq ( xhci, dma ( &event->trb_map, trb ),
1769  xhci->run + XHCI_RUN_ERDP ( 0 ) );
1770  profile_stop ( &xhci_event_profiler );
1771  }
1772 }
1773 
1774 /**
1775  * Abort command
1776  *
1777  * @v xhci xHCI device
1778  */
1779 static void xhci_abort ( struct xhci_device *xhci ) {
1780  physaddr_t crp;
1781 
1782  /* Abort the command */
1783  DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
1784  xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR );
1785 
1786  /* Allow time for command to abort */
1788 
1789  /* Sanity check */
1790  assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
1791 
1792  /* Consume (and ignore) any final command status */
1793  xhci_event_poll ( xhci );
1794 
1795  /* Reset the command ring control register */
1796  xhci_ring_reset ( &xhci->command );
1797  crp = dma ( &xhci->command.map, xhci->command.trb );
1798  xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
1799 }
1800 
1801 /**
1802  * Issue command and wait for completion
1803  *
1804  * @v xhci xHCI device
1805  * @v trb Transfer request block (with empty Cycle flag)
1806  * @ret rc Return status code
1807  *
1808  * On a successful completion, the TRB will be overwritten with the
1809  * completion.
1810  */
1811 static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
1812  struct xhci_trb_complete *complete = &trb->complete;
1813  unsigned int i;
1814  int rc;
1815 
1816  /* Sanity check */
1817  if ( xhci->pending ) {
1818  DBGC ( xhci, "XHCI %s command ring busy\n", xhci->name );
1819  rc = -EBUSY;
1820  goto err_pending;
1821  }
1822 
1823  /* Record the pending command */
1824  xhci->pending = trb;
1825 
1826  /* Enqueue the command */
1827  if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
1828  goto err_enqueue;
1829 
1830  /* Ring the command doorbell */
1831  xhci_doorbell ( &xhci->command );
1832 
1833  /* Wait for the command to complete */
1834  for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
1835 
1836  /* Poll event ring */
1837  xhci_event_poll ( xhci );
1838 
1839  /* Check for completion */
1840  if ( ! xhci->pending ) {
1841  if ( complete->code != XHCI_CMPLT_SUCCESS ) {
1842  rc = -ECODE ( complete->code );
1843  DBGC ( xhci, "XHCI %s command failed (code "
1844  "%d): %s\n", xhci->name, complete->code,
1845  strerror ( rc ) );
1846  DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
1847  return rc;
1848  }
1849  return 0;
1850  }
1851 
1852  /* Delay */
1853  mdelay ( 1 );
1854  }
1855 
1856  /* Timeout */
1857  DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name );
1858  rc = -ETIMEDOUT;
1859 
1860  /* Abort command */
1861  xhci_abort ( xhci );
1862 
1863  err_enqueue:
1864  xhci->pending = NULL;
1865  err_pending:
1866  return rc;
1867 }
1868 
1869 /**
1870  * Issue NOP and wait for completion
1871  *
1872  * @v xhci xHCI device
1873  * @ret rc Return status code
1874  */
1875 static inline int xhci_nop ( struct xhci_device *xhci ) {
1876  union xhci_trb trb;
1877  struct xhci_trb_common *nop = &trb.common;
1878  int rc;
1879 
1880  /* Construct command */
1881  memset ( nop, 0, sizeof ( *nop ) );
1882  nop->flags = XHCI_TRB_IOC;
1883  nop->type = XHCI_TRB_NOP_CMD;
1884 
1885  /* Issue command and wait for completion */
1886  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1887  DBGC ( xhci, "XHCI %s NOP failed: %s\n",
1888  xhci->name, strerror ( rc ) );
1889  return rc;
1890  }
1891 
1892  DBGC2 ( xhci, "XHCI %s NOP completed successfully\n", xhci->name );
1893  return 0;
1894 }
1895 
1896 /**
1897  * Enable slot
1898  *
1899  * @v xhci xHCI device
1900  * @v type Slot type
1901  * @ret slot Device slot ID, or negative error
1902  */
1903 static inline int xhci_enable_slot ( struct xhci_device *xhci,
1904  unsigned int type ) {
1905  union xhci_trb trb;
1906  struct xhci_trb_enable_slot *enable = &trb.enable;
1907  struct xhci_trb_complete *enabled = &trb.complete;
1908  unsigned int slot;
1909  int rc;
1910 
1911  /* Construct command */
1912  memset ( enable, 0, sizeof ( *enable ) );
1913  enable->slot = type;
1914  enable->type = XHCI_TRB_ENABLE_SLOT;
1915 
1916  /* Issue command and wait for completion */
1917  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1918  DBGC ( xhci, "XHCI %s could not enable new slot: %s\n",
1919  xhci->name, strerror ( rc ) );
1920  return rc;
1921  }
1922 
1923  /* Extract slot number */
1924  slot = enabled->slot;
1925 
1926  DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot );
1927  return slot;
1928 }
1929 
1930 /**
1931  * Disable slot
1932  *
1933  * @v xhci xHCI device
1934  * @v slot Device slot
1935  * @ret rc Return status code
1936  */
1937 static inline int xhci_disable_slot ( struct xhci_device *xhci,
1938  unsigned int slot ) {
1939  union xhci_trb trb;
1940  struct xhci_trb_disable_slot *disable = &trb.disable;
1941  int rc;
1942 
1943  /* Construct command */
1944  memset ( disable, 0, sizeof ( *disable ) );
1945  disable->type = XHCI_TRB_DISABLE_SLOT;
1946  disable->slot = slot;
1947 
1948  /* Issue command and wait for completion */
1949  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
1950  DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n",
1951  xhci->name, slot, strerror ( rc ) );
1952  return rc;
1953  }
1954 
1955  DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot );
1956  return 0;
1957 }
1958 
1959 /**
1960  * Issue context-based command and wait for completion
1961  *
1962  * @v xhci xHCI device
1963  * @v slot Device slot
1964  * @v endpoint Endpoint
1965  * @v type TRB type
1966  * @v populate Input context populater
1967  * @ret rc Return status code
1968  */
1969 static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
1970  struct xhci_endpoint *endpoint, unsigned int type,
1971  void ( * populate ) ( struct xhci_device *xhci,
1972  struct xhci_slot *slot,
1973  struct xhci_endpoint *endpoint,
1974  void *input ) ) {
1975  union xhci_trb trb;
1976  struct xhci_trb_context *context = &trb.context;
1977  struct dma_mapping map;
1978  size_t len;
1979  void *input;
1980  int rc;
1981 
1982  /* Allocate an input context */
1983  memset ( &map, 0, sizeof ( map ) );
1985  input = dma_alloc ( xhci->dma, &map, len, xhci_align ( len ) );
1986  if ( ! input ) {
1987  rc = -ENOMEM;
1988  goto err_alloc;
1989  }
1990  memset ( input, 0, len );
1991 
1992  /* Populate input context */
1993  populate ( xhci, slot, endpoint, input );
1994 
1995  /* Construct command */
1996  memset ( context, 0, sizeof ( *context ) );
1997  context->type = type;
1998  context->input = cpu_to_le64 ( dma ( &map, input ) );
1999  context->slot = slot->id;
2000 
2001  /* Issue command and wait for completion */
2002  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
2003  goto err_command;
2004 
2005  err_command:
2006  dma_free ( &map, input, len );
2007  err_alloc:
2008  return rc;
2009 }
2010 
2011 /**
2012  * Populate address device input context
2013  *
2014  * @v xhci xHCI device
2015  * @v slot Device slot
2016  * @v endpoint Endpoint
2017  * @v input Input context
2018  */
2019 static void xhci_address_device_input ( struct xhci_device *xhci,
2020  struct xhci_slot *slot,
2021  struct xhci_endpoint *endpoint,
2022  void *input ) {
2023  struct xhci_trb_ring *ring = &endpoint->ring;
2024  struct xhci_control_context *control_ctx;
2025  struct xhci_slot_context *slot_ctx;
2026  struct xhci_endpoint_context *ep_ctx;
2027 
2028  /* Sanity checks */
2029  assert ( endpoint->ctx == XHCI_CTX_EP0 );
2030 
2031  /* Populate control context */
2032  control_ctx = input;
2033  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2034  ( 1 << XHCI_CTX_EP0 ) );
2035 
2036  /* Populate slot context */
2037  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2038  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
2039  slot->route ) );
2040  slot_ctx->port = slot->port;
2041  slot_ctx->tt_id = slot->tt_id;
2042  slot_ctx->tt_port = slot->tt_port;
2043 
2044  /* Populate control endpoint context */
2045  ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
2046  ep_ctx->type = XHCI_EP_TYPE_CONTROL;
2047  ep_ctx->burst = endpoint->ep->burst;
2048  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2049  ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
2050  XHCI_EP_DCS );
2051  ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
2052 }
2053 
2054 /**
2055  * Address device
2056  *
2057  * @v xhci xHCI device
2058  * @v slot Device slot
2059  * @ret rc Return status code
2060  */
2061 static inline int xhci_address_device ( struct xhci_device *xhci,
2062  struct xhci_slot *slot ) {
2063  struct usb_device *usb = slot->usb;
2064  struct xhci_slot_context *slot_ctx;
2065  int rc;
2066 
2067  /* Assign device address */
2068  if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
2070  xhci_address_device_input ) ) != 0 ) {
2071  DBGC ( xhci, "XHCI %s slot %d could not assign address: %s\n",
2072  xhci->name, slot->id, strerror ( rc ) );
2073  return rc;
2074  }
2075 
2076  /* Get assigned address */
2077  slot_ctx = ( slot->context +
2079  usb->address = slot_ctx->address;
2080  DBGC2 ( xhci, "XHCI %s slot %d assigned address %d to %s\n",
2081  xhci->name, slot->id, usb->address, usb->name );
2082 
2083  return 0;
2084 }
2085 
2086 /**
2087  * Populate configure endpoint input context
2088  *
2089  * @v xhci xHCI device
2090  * @v slot Device slot
2091  * @v endpoint Endpoint
2092  * @v input Input context
2093  */
2094 static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
2095  struct xhci_slot *slot,
2096  struct xhci_endpoint *endpoint,
2097  void *input ) {
2098  struct xhci_trb_ring *ring = &endpoint->ring;
2099  struct xhci_control_context *control_ctx;
2100  struct xhci_slot_context *slot_ctx;
2101  struct xhci_endpoint_context *ep_ctx;
2102 
2103  /* Populate control context */
2104  control_ctx = input;
2105  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2106  ( 1 << endpoint->ctx ) );
2107 
2108  /* Populate slot context */
2109  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2110  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2111  ( slot->ports ? 1 : 0 ),
2112  slot->psiv, 0 ) );
2113  slot_ctx->ports = slot->ports;
2114 
2115  /* Populate endpoint context */
2116  ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
2117  ep_ctx->interval = endpoint->interval;
2118  ep_ctx->type = endpoint->type;
2119  ep_ctx->burst = endpoint->ep->burst;
2120  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2121  ep_ctx->dequeue = cpu_to_le64 ( dma ( &ring->map, ring->trb ) |
2122  XHCI_EP_DCS );
2123  ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
2124 }
2125 
2126 /**
2127  * Configure endpoint
2128  *
2129  * @v xhci xHCI device
2130  * @v slot Device slot
2131  * @v endpoint Endpoint
2132  * @ret rc Return status code
2133  */
2134 static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
2135  struct xhci_slot *slot,
2136  struct xhci_endpoint *endpoint ) {
2137  int rc;
2138 
2139  /* Configure endpoint */
2140  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2142  xhci_configure_endpoint_input ) ) != 0 ) {
2143  DBGC ( xhci, "XHCI %s slot %d ctx %d could not configure: %s\n",
2144  xhci->name, slot->id, endpoint->ctx, strerror ( rc ) );
2145  return rc;
2146  }
2147 
2148  DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
2149  xhci->name, slot->id, endpoint->ctx );
2150  return 0;
2151 }
2152 
2153 /**
2154  * Populate deconfigure endpoint input context
2155  *
2156  * @v xhci xHCI device
2157  * @v slot Device slot
2158  * @v endpoint Endpoint
2159  * @v input Input context
2160  */
2161 static void
2163  struct xhci_slot *slot __unused,
2164  struct xhci_endpoint *endpoint,
2165  void *input ) {
2166  struct xhci_control_context *control_ctx;
2167  struct xhci_slot_context *slot_ctx;
2168 
2169  /* Populate control context */
2170  control_ctx = input;
2171  control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
2172  control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
2173 
2174  /* Populate slot context */
2175  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2176  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2177  0, 0, 0 ) );
2178 }
2179 
2180 /**
2181  * Deconfigure endpoint
2182  *
2183  * @v xhci xHCI device
2184  * @v slot Device slot
2185  * @v endpoint Endpoint
2186  * @ret rc Return status code
2187  */
2188 static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
2189  struct xhci_slot *slot,
2190  struct xhci_endpoint *endpoint ) {
2191  int rc;
2192 
2193  /* Deconfigure endpoint */
2194  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2196  xhci_deconfigure_endpoint_input ) ) != 0 ) {
2197  DBGC ( xhci, "XHCI %s slot %d ctx %d could not deconfigure: "
2198  "%s\n", xhci->name, slot->id, endpoint->ctx,
2199  strerror ( rc ) );
2200  return rc;
2201  }
2202 
2203  DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
2204  xhci->name, slot->id, endpoint->ctx );
2205  return 0;
2206 }
2207 
2208 /**
2209  * Populate evaluate context input context
2210  *
2211  * @v xhci xHCI device
2212  * @v slot Device slot
2213  * @v endpoint Endpoint
2214  * @v input Input context
2215  */
2216 static void xhci_evaluate_context_input ( struct xhci_device *xhci,
2217  struct xhci_slot *slot __unused,
2218  struct xhci_endpoint *endpoint,
2219  void *input ) {
2220  struct xhci_control_context *control_ctx;
2221  struct xhci_slot_context *slot_ctx;
2222  struct xhci_endpoint_context *ep_ctx;
2223 
2224  /* Populate control context */
2225  control_ctx = input;
2226  control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
2227  ( 1 << endpoint->ctx ) );
2228 
2229  /* Populate slot context */
2230  slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
2231  slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
2232  0, 0, 0 ) );
2233 
2234  /* Populate endpoint context */
2235  ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
2236  ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
2237 }
2238 
2239 /**
2240  * Evaluate context
2241  *
2242  * @v xhci xHCI device
2243  * @v slot Device slot
2244  * @v endpoint Endpoint
2245  * @ret rc Return status code
2246  */
2247 static inline int xhci_evaluate_context ( struct xhci_device *xhci,
2248  struct xhci_slot *slot,
2249  struct xhci_endpoint *endpoint ) {
2250  int rc;
2251 
2252  /* Configure endpoint */
2253  if ( ( rc = xhci_context ( xhci, slot, endpoint,
2255  xhci_evaluate_context_input ) ) != 0 ) {
2256  DBGC ( xhci, "XHCI %s slot %d ctx %d could not (re-)evaluate: "
2257  "%s\n", xhci->name, slot->id, endpoint->ctx,
2258  strerror ( rc ) );
2259  return rc;
2260  }
2261 
2262  DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
2263  xhci->name, slot->id, endpoint->ctx );
2264  return 0;
2265 }
2266 
2267 /**
2268  * Reset endpoint
2269  *
2270  * @v xhci xHCI device
2271  * @v slot Device slot
2272  * @v endpoint Endpoint
2273  * @ret rc Return status code
2274  */
2275 static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
2276  struct xhci_slot *slot,
2277  struct xhci_endpoint *endpoint ) {
2278  union xhci_trb trb;
2279  struct xhci_trb_reset_endpoint *reset = &trb.reset;
2280  int rc;
2281 
2282  /* Construct command */
2283  memset ( reset, 0, sizeof ( *reset ) );
2284  reset->slot = slot->id;
2285  reset->endpoint = endpoint->ctx;
2286  reset->type = XHCI_TRB_RESET_ENDPOINT;
2287 
2288  /* Issue command and wait for completion */
2289  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2290  DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint "
2291  "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
2292  endpoint->context->state, strerror ( rc ) );
2293  return rc;
2294  }
2295 
2296  return 0;
2297 }
2298 
2299 /**
2300  * Stop endpoint
2301  *
2302  * @v xhci xHCI device
2303  * @v slot Device slot
2304  * @v endpoint Endpoint
2305  * @ret rc Return status code
2306  */
2307 static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
2308  struct xhci_slot *slot,
2309  struct xhci_endpoint *endpoint ) {
2310  union xhci_trb trb;
2311  struct xhci_trb_stop_endpoint *stop = &trb.stop;
2312  int rc;
2313 
2314  /* Construct command */
2315  memset ( stop, 0, sizeof ( *stop ) );
2316  stop->slot = slot->id;
2317  stop->endpoint = endpoint->ctx;
2318  stop->type = XHCI_TRB_STOP_ENDPOINT;
2319 
2320  /* Issue command and wait for completion */
2321  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2322  DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint "
2323  "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
2324  endpoint->context->state, strerror ( rc ) );
2325  return rc;
2326  }
2327 
2328  return 0;
2329 }
2330 
2331 /**
2332  * Set transfer ring dequeue pointer
2333  *
2334  * @v xhci xHCI device
2335  * @v slot Device slot
2336  * @v endpoint Endpoint
2337  * @ret rc Return status code
2338  */
2339 static inline int
2341  struct xhci_slot *slot,
2342  struct xhci_endpoint *endpoint ) {
2343  union xhci_trb trb;
2345  struct xhci_trb_ring *ring = &endpoint->ring;
2346  unsigned int cons;
2347  unsigned int mask;
2348  unsigned int index;
2349  unsigned int dcs;
2350  physaddr_t addr;
2351  int rc;
2352 
2353  /* Construct command */
2354  memset ( dequeue, 0, sizeof ( *dequeue ) );
2355  cons = ring->cons;
2356  mask = ring->mask;
2357  dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
2358  index = ( cons & mask );
2359  addr = dma ( &ring->map, &ring->trb[index] );
2360  dequeue->dequeue = cpu_to_le64 ( addr | dcs );
2361  dequeue->slot = slot->id;
2362  dequeue->endpoint = endpoint->ctx;
2364 
2365  /* Issue command and wait for completion */
2366  if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
2367  DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue "
2368  "pointer in state %d: %s\n", xhci->name, slot->id,
2369  endpoint->ctx, endpoint->context->state, strerror ( rc));
2370  return rc;
2371  }
2372 
2373  return 0;
2374 }
2375 
2376 /******************************************************************************
2377  *
2378  * Endpoint operations
2379  *
2380  ******************************************************************************
2381  */
2382 
2383 /**
2384  * Open endpoint
2385  *
2386  * @v ep USB endpoint
2387  * @ret rc Return status code
2388  */
2389 static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
2390  struct usb_device *usb = ep->usb;
2391  struct xhci_slot *slot = usb_get_hostdata ( usb );
2392  struct xhci_device *xhci = slot->xhci;
2393  struct xhci_endpoint *endpoint;
2394  unsigned int ctx;
2395  unsigned int type;
2396  unsigned int interval;
2397  int rc;
2398 
2399  /* Calculate context index */
2400  ctx = XHCI_CTX ( ep->address );
2401  assert ( slot->endpoint[ctx] == NULL );
2402 
2403  /* Calculate endpoint type */
2407  if ( ep->address & USB_DIR_IN )
2408  type |= XHCI_EP_TYPE_IN;
2409 
2410  /* Calculate interval */
2411  if ( type & XHCI_EP_TYPE_PERIODIC ) {
2412  interval = ( fls ( ep->interval ) - 1 );
2413  } else {
2414  interval = ep->interval;
2415  }
2416 
2417  /* Allocate and initialise structure */
2418  endpoint = zalloc ( sizeof ( *endpoint ) );
2419  if ( ! endpoint ) {
2420  rc = -ENOMEM;
2421  goto err_alloc;
2422  }
2423  usb_endpoint_set_hostdata ( ep, endpoint );
2424  slot->endpoint[ctx] = endpoint;
2425  endpoint->xhci = xhci;
2426  endpoint->slot = slot;
2427  endpoint->ep = ep;
2428  endpoint->ctx = ctx;
2429  endpoint->type = type;
2430  endpoint->interval = interval;
2431  endpoint->context = ( ( ( void * ) slot->context ) +
2433 
2434  /* Allocate transfer ring */
2435  if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
2437  slot->id, ctx, 0 ) ) != 0 )
2438  goto err_ring_alloc;
2439 
2440  /* Configure endpoint, if applicable */
2441  if ( ( ctx != XHCI_CTX_EP0 ) &&
2442  ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
2443  goto err_configure_endpoint;
2444 
2445  DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n",
2446  xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
2447  ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
2448  return 0;
2449 
2450  xhci_deconfigure_endpoint ( xhci, slot, endpoint );
2451  err_configure_endpoint:
2452  xhci_ring_free ( &endpoint->ring );
2453  err_ring_alloc:
2454  slot->endpoint[ctx] = NULL;
2455  free ( endpoint );
2456  err_alloc:
2457  return rc;
2458 }
2459 
2460 /**
2461  * Close endpoint
2462  *
2463  * @v ep USB endpoint
2464  */
2465 static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
2466  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2467  struct xhci_slot *slot = endpoint->slot;
2468  struct xhci_device *xhci = slot->xhci;
2469  struct io_buffer *iobuf;
2470  unsigned int ctx = endpoint->ctx;
2471 
2472  /* Deconfigure endpoint, if applicable */
2473  if ( ctx != XHCI_CTX_EP0 )
2474  xhci_deconfigure_endpoint ( xhci, slot, endpoint );
2475 
2476  /* Cancel any incomplete transfers */
2477  while ( xhci_ring_fill ( &endpoint->ring ) ) {
2478  iobuf = xhci_dequeue_multi ( &endpoint->ring );
2479  iob_unmap ( iobuf );
2480  usb_complete_err ( ep, iobuf, -ECANCELED );
2481  }
2482 
2483  /* Free endpoint */
2484  xhci_ring_free ( &endpoint->ring );
2485  slot->endpoint[ctx] = NULL;
2486  free ( endpoint );
2487 }
2488 
2489 /**
2490  * Reset endpoint
2491  *
2492  * @v ep USB endpoint
2493  * @ret rc Return status code
2494  */
2495 static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
2496  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2497  struct xhci_slot *slot = endpoint->slot;
2498  struct xhci_device *xhci = slot->xhci;
2499  int rc;
2500 
2501  /* Reset endpoint context */
2502  if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
2503  return rc;
2504 
2505  /* Set transfer ring dequeue pointer */
2506  if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
2507  return rc;
2508 
2509  /* Ring doorbell to resume processing */
2510  xhci_doorbell ( &endpoint->ring );
2511 
2512  DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n",
2513  xhci->name, slot->id, endpoint->ctx );
2514  return 0;
2515 }
2516 
2517 /**
2518  * Update MTU
2519  *
2520  * @v ep USB endpoint
2521  * @ret rc Return status code
2522  */
2523 static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
2524  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2525  struct xhci_slot *slot = endpoint->slot;
2526  struct xhci_device *xhci = slot->xhci;
2527  int rc;
2528 
2529  /* Evalulate context */
2530  if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
2531  return rc;
2532 
2533  return 0;
2534 }
2535 
2536 /**
2537  * Enqueue message transfer
2538  *
2539  * @v ep USB endpoint
2540  * @v iobuf I/O buffer
2541  * @ret rc Return status code
2542  */
2543 static int xhci_endpoint_message ( struct usb_endpoint *ep,
2544  struct io_buffer *iobuf ) {
2545  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2546  struct xhci_device *xhci = endpoint->xhci;
2547  struct usb_setup_packet *packet;
2548  unsigned int input;
2549  size_t len;
2550  union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
2551  1 /* status */ ];
2552  union xhci_trb *trb = trbs;
2553  struct xhci_trb_setup *setup;
2554  struct xhci_trb_data *data;
2555  struct xhci_trb_status *status;
2556  int rc;
2557 
2558  /* Profile message transfers */
2559  profile_start ( &xhci_message_profiler );
2560 
2561  /* Construct setup stage TRB */
2562  memset ( trbs, 0, sizeof ( trbs ) );
2563  assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
2564  packet = iobuf->data;
2565  iob_pull ( iobuf, sizeof ( *packet ) );
2566  setup = &(trb++)->setup;
2567  memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
2568  setup->len = cpu_to_le32 ( sizeof ( *packet ) );
2569  setup->flags = XHCI_TRB_IDT;
2570  setup->type = XHCI_TRB_SETUP;
2571  len = iob_len ( iobuf );
2572  input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
2573  if ( len )
2574  setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
2575 
2576  /* Map I/O buffer */
2577  if ( ( rc = iob_map ( iobuf, xhci->dma, len,
2578  ( input ? DMA_RX : DMA_TX ) ) ) != 0 )
2579  goto err_map;
2580 
2581  /* Construct data stage TRB, if applicable */
2582  if ( len ) {
2583  data = &(trb++)->data;
2584  data->data = cpu_to_le64 ( iob_dma ( iobuf ) );
2585  data->len = cpu_to_le32 ( len );
2586  data->type = XHCI_TRB_DATA;
2587  data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
2588  }
2589 
2590  /* Construct status stage TRB */
2591  status = &(trb++)->status;
2592  status->flags = XHCI_TRB_IOC;
2593  status->type = XHCI_TRB_STATUS;
2594  status->direction =
2595  ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
2596 
2597  /* Enqueue TRBs */
2598  if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
2599  ( trb - trbs ) ) ) != 0 )
2600  goto err_enqueue;
2601 
2602  /* Ring the doorbell */
2603  xhci_doorbell ( &endpoint->ring );
2604 
2605  profile_stop ( &xhci_message_profiler );
2606  return 0;
2607 
2608  err_enqueue:
2609  iob_unmap ( iobuf );
2610  err_map:
2611  return rc;
2612 }
2613 
2614 /**
2615  * Calculate number of TRBs
2616  *
2617  * @v len Length of data
2618  * @v zlp Append a zero-length packet
2619  * @ret count Number of transfer descriptors
2620  */
2621 static unsigned int xhci_endpoint_count ( size_t len, int zlp ) {
2622  unsigned int count;
2623 
2624  /* Split into 64kB TRBs */
2625  count = ( ( len + XHCI_MTU - 1 ) / XHCI_MTU );
2626 
2627  /* Append a zero-length TRB if applicable */
2628  if ( zlp || ( count == 0 ) )
2629  count++;
2630 
2631  return count;
2632 }
2633 
2634 /**
2635  * Enqueue stream transfer
2636  *
2637  * @v ep USB endpoint
2638  * @v iobuf I/O buffer
2639  * @v zlp Append a zero-length packet
2640  * @ret rc Return status code
2641  */
2642 static int xhci_endpoint_stream ( struct usb_endpoint *ep,
2643  struct io_buffer *iobuf, int zlp ) {
2644  struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
2645  struct xhci_device *xhci = endpoint->xhci;
2646  size_t len = iob_len ( iobuf );
2647  unsigned int count = xhci_endpoint_count ( len, zlp );
2648  union xhci_trb trbs[count];
2649  union xhci_trb *trb = trbs;
2650  struct xhci_trb_normal *normal;
2651  physaddr_t data;
2652  unsigned int i;
2653  size_t trb_len;
2654  int rc;
2655 
2656  /* Profile stream transfers */
2657  profile_start ( &xhci_stream_profiler );
2658 
2659  /* Map I/O buffer */
2660  if ( ( rc = iob_map ( iobuf, xhci->dma, len,
2661  ( ( ep->address & USB_DIR_IN ) ?
2662  DMA_RX : DMA_TX ) ) ) != 0 )
2663  goto err_map;
2664  data = iob_dma ( iobuf );
2665 
2666  /* Construct normal TRBs */
2667  memset ( &trbs, 0, sizeof ( trbs ) );
2668  for ( i = 0 ; i < count ; i ++ ) {
2669 
2670  /* Calculate TRB length */
2671  trb_len = XHCI_MTU;
2672  if ( trb_len > len )
2673  trb_len = len;
2674 
2675  /* Construct normal TRB */
2676  normal = &trb->normal;
2677  normal->data = cpu_to_le64 ( data );
2678  normal->len = cpu_to_le32 ( trb_len );
2679  normal->type = XHCI_TRB_NORMAL;
2680  normal->flags = XHCI_TRB_CH;
2681 
2682  /* Move to next TRB */
2683  data += trb_len;
2684  len -= trb_len;
2685  trb++;
2686  }
2687 
2688  /* Mark zero-length packet (if present) as a separate transfer */
2689  if ( zlp && ( count > 1 ) )
2690  trb[-2].normal.flags = 0;
2691 
2692  /* Generate completion for final TRB */
2693  trb[-1].normal.flags = XHCI_TRB_IOC;
2694 
2695  /* Enqueue TRBs */
2696  if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
2697  count ) ) != 0 )
2698  goto err_enqueue;
2699 
2700  /* Ring the doorbell */
2701  xhci_doorbell ( &endpoint->ring );
2702 
2703  profile_stop ( &xhci_stream_profiler );
2704  return 0;
2705 
2706  err_enqueue:
2707  iob_unmap ( iobuf );
2708  err_map:
2709  return rc;
2710 }
2711 
2712 /******************************************************************************
2713  *
2714  * Device operations
2715  *
2716  ******************************************************************************
2717  */
2718 
2719 /**
2720  * Open device
2721  *
2722  * @v usb USB device
2723  * @ret rc Return status code
2724  */
2725 static int xhci_device_open ( struct usb_device *usb ) {
2726  struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
2727  struct usb_port *tt = usb_transaction_translator ( usb );
2728  struct xhci_slot *slot;
2729  struct xhci_slot *tt_slot;
2730  size_t len;
2731  int type;
2732  int id;
2733  int rc;
2734 
2735  /* Determine applicable slot type */
2737  if ( type < 0 ) {
2738  rc = type;
2739  DBGC ( xhci, "XHCI %s-%d has no slot type\n",
2740  xhci->name, usb->port->address );
2741  goto err_type;
2742  }
2743 
2744  /* Allocate a device slot number */
2745  id = xhci_enable_slot ( xhci, type );
2746  if ( id < 0 ) {
2747  rc = id;
2748  goto err_enable_slot;
2749  }
2750  assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) );
2751  assert ( xhci->slot[id] == NULL );
2752 
2753  /* Allocate and initialise structure */
2754  slot = zalloc ( sizeof ( *slot ) );
2755  if ( ! slot ) {
2756  rc = -ENOMEM;
2757  goto err_alloc;
2758  }
2759  usb_set_hostdata ( usb, slot );
2760  xhci->slot[id] = slot;
2761  slot->xhci = xhci;
2762  slot->usb = usb;
2763  slot->id = id;
2764  if ( tt ) {
2765  tt_slot = usb_get_hostdata ( tt->hub->usb );
2766  slot->tt_id = tt_slot->id;
2767  slot->tt_port = tt->address;
2768  }
2769 
2770  /* Allocate a device context */
2772  slot->context = dma_alloc ( xhci->dma, &slot->map, len,
2773  xhci_align ( len ) );
2774  if ( ! slot->context ) {
2775  rc = -ENOMEM;
2776  goto err_alloc_context;
2777  }
2778  memset ( slot->context, 0, len );
2779 
2780  /* Set device context base address */
2781  assert ( xhci->dcbaa.context[id] == 0 );
2782  xhci->dcbaa.context[id] = cpu_to_le64 ( dma ( &slot->map,
2783  slot->context ) );
2784 
2785  DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
2786  xhci->name, slot->id, virt_to_phys ( slot->context ),
2787  ( virt_to_phys ( slot->context ) + len ), usb->name );
2788  return 0;
2789 
2790  xhci->dcbaa.context[id] = 0;
2791  dma_free ( &slot->map, slot->context, len );
2792  err_alloc_context:
2793  xhci->slot[id] = NULL;
2794  free ( slot );
2795  err_alloc:
2796  xhci_disable_slot ( xhci, id );
2797  err_enable_slot:
2798  err_type:
2799  return rc;
2800 }
2801 
2802 /**
2803  * Close device
2804  *
2805  * @v usb USB device
2806  */
2807 static void xhci_device_close ( struct usb_device *usb ) {
2808  struct xhci_slot *slot = usb_get_hostdata ( usb );
2809  struct xhci_device *xhci = slot->xhci;
2810  size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
2811  unsigned int id = slot->id;
2812  int rc;
2813 
2814  /* Disable slot */
2815  if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) {
2816  /* Slot is still enabled. Leak the slot context,
2817  * since the controller may still write to this
2818  * memory, and leave the DCBAA entry intact.
2819  *
2820  * If the controller later reports that this same slot
2821  * has been re-enabled, then some assertions will be
2822  * triggered.
2823  */
2824  DBGC ( xhci, "XHCI %s slot %d leaking context memory\n",
2825  xhci->name, slot->id );
2826  slot->context = NULL;
2827  }
2828 
2829  /* Free slot */
2830  if ( slot->context ) {
2831  dma_free ( &slot->map, slot->context, len );
2832  xhci->dcbaa.context[id] = 0;
2833  }
2834  xhci->slot[id] = NULL;
2835  free ( slot );
2836 }
2837 
2838 /**
2839  * Assign device address
2840  *
2841  * @v usb USB device
2842  * @ret rc Return status code
2843  */
2844 static int xhci_device_address ( struct usb_device *usb ) {
2845  struct xhci_slot *slot = usb_get_hostdata ( usb );
2846  struct xhci_device *xhci = slot->xhci;
2847  struct usb_port *root_port;
2848  int psiv;
2849  int rc;
2850 
2851  /* Calculate route string */
2852  slot->route = usb_route_string ( usb );
2853 
2854  /* Calculate root hub port number */
2855  root_port = usb_root_hub_port ( usb );
2856  slot->port = root_port->address;
2857 
2858  /* Calculate protocol speed ID */
2859  psiv = xhci_port_psiv ( xhci, slot->port, usb->speed );
2860  if ( psiv < 0 ) {
2861  rc = psiv;
2862  return rc;
2863  }
2864  slot->psiv = psiv;
2865 
2866  /* Address device */
2867  if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
2868  return rc;
2869 
2870  return 0;
2871 }
2872 
2873 /******************************************************************************
2874  *
2875  * Bus operations
2876  *
2877  ******************************************************************************
2878  */
2879 
2880 /**
2881  * Open USB bus
2882  *
2883  * @v bus USB bus
2884  * @ret rc Return status code
2885  */
2886 static int xhci_bus_open ( struct usb_bus *bus ) {
2887  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2888  int rc;
2889 
2890  /* Allocate device slot array */
2891  xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) );
2892  if ( ! xhci->slot ) {
2893  rc = -ENOMEM;
2894  goto err_slot_alloc;
2895  }
2896 
2897  /* Allocate device context base address array */
2898  if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
2899  goto err_dcbaa_alloc;
2900 
2901  /* Allocate scratchpad buffers */
2902  if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
2903  goto err_scratchpad_alloc;
2904 
2905  /* Allocate command ring */
2906  if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
2907  goto err_command_alloc;
2908 
2909  /* Allocate event ring */
2910  if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
2911  goto err_event_alloc;
2912 
2913  /* Start controller */
2914  xhci_run ( xhci );
2915 
2916  return 0;
2917 
2918  xhci_stop ( xhci );
2919  xhci_event_free ( xhci );
2920  err_event_alloc:
2921  xhci_command_free ( xhci );
2922  err_command_alloc:
2923  xhci_scratchpad_free ( xhci );
2924  err_scratchpad_alloc:
2925  xhci_dcbaa_free ( xhci );
2926  err_dcbaa_alloc:
2927  free ( xhci->slot );
2928  err_slot_alloc:
2929  return rc;
2930 }
2931 
2932 /**
2933  * Close USB bus
2934  *
2935  * @v bus USB bus
2936  */
2937 static void xhci_bus_close ( struct usb_bus *bus ) {
2938  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2939  unsigned int i;
2940 
2941  /* Sanity checks */
2942  assert ( xhci->slot != NULL );
2943  for ( i = 0 ; i <= xhci->slots ; i++ )
2944  assert ( xhci->slot[i] == NULL );
2945 
2946  xhci_stop ( xhci );
2947  xhci_event_free ( xhci );
2948  xhci_command_free ( xhci );
2949  xhci_scratchpad_free ( xhci );
2950  xhci_dcbaa_free ( xhci );
2951  free ( xhci->slot );
2952 }
2953 
2954 /**
2955  * Poll USB bus
2956  *
2957  * @v bus USB bus
2958  */
2959 static void xhci_bus_poll ( struct usb_bus *bus ) {
2960  struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
2961 
2962  /* Poll event ring */
2963  xhci_event_poll ( xhci );
2964 }
2965 
2966 /******************************************************************************
2967  *
2968  * Hub operations
2969  *
2970  ******************************************************************************
2971  */
2972 
2973 /**
2974  * Open hub
2975  *
2976  * @v hub USB hub
2977  * @ret rc Return status code
2978  */
2979 static int xhci_hub_open ( struct usb_hub *hub ) {
2980  struct xhci_slot *slot;
2981 
2982  /* Do nothing if this is the root hub */
2983  if ( ! hub->usb )
2984  return 0;
2985 
2986  /* Get device slot */
2987  slot = usb_get_hostdata ( hub->usb );
2988 
2989  /* Update device slot hub parameters. We don't inform the
2990  * hardware of this information until the hub's interrupt
2991  * endpoint is opened, since the only mechanism for so doing
2992  * provided by the xHCI specification is a Configure Endpoint
2993  * command, and we can't issue that command until we have a
2994  * non-EP0 endpoint to configure.
2995  */
2996  slot->ports = hub->ports;
2997 
2998  return 0;
2999 }
3000 
3001 /**
3002  * Close hub
3003  *
3004  * @v hub USB hub
3005  */
3006 static void xhci_hub_close ( struct usb_hub *hub __unused ) {
3007 
3008  /* Nothing to do */
3009 }
3010 
3011 /******************************************************************************
3012  *
3013  * Root hub operations
3014  *
3015  ******************************************************************************
3016  */
3017 
3018 /**
3019  * Open root hub
3020  *
3021  * @v hub USB hub
3022  * @ret rc Return status code
3023  */
3024 static int xhci_root_open ( struct usb_hub *hub ) {
3025  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3026  struct usb_port *port;
3027  uint32_t portsc;
3028  unsigned int i;
3029 
3030  /* Enable power to all ports */
3031  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3032  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
3033  portsc &= XHCI_PORTSC_PRESERVE;
3034  portsc |= XHCI_PORTSC_PP;
3035  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
3036  }
3037 
3038  /* xHCI spec requires us to potentially wait 20ms after
3039  * enabling power to a port.
3040  */
3042 
3043  /* USB3 ports may power up as Disabled */
3044  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3045  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
3046  port = usb_port ( hub, i );
3047  if ( ( port->protocol >= USB_PROTO_3_0 ) &&
3048  ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
3050  /* Force link state to RxDetect */
3051  portsc &= XHCI_PORTSC_PRESERVE;
3053  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
3054  }
3055  }
3056 
3057  /* Some xHCI cards seem to require an additional delay after
3058  * setting the link state to RxDetect.
3059  */
3061 
3062  return 0;
3063 }
3064 
3065 /**
3066  * Close root hub
3067  *
3068  * @v hub USB hub
3069  */
3070 static void xhci_root_close ( struct usb_hub *hub __unused ) {
3071 
3072  /* Nothing to do */
3073 }
3074 
3075 /**
3076  * Enable port
3077  *
3078  * @v hub USB hub
3079  * @v port USB port
3080  * @ret rc Return status code
3081  */
3082 static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
3083  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3084  uint32_t portsc;
3085  unsigned int i;
3086 
3087  /* Reset port */
3088  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3089  portsc &= XHCI_PORTSC_PRESERVE;
3090  portsc |= XHCI_PORTSC_PR;
3091  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3092 
3093  /* Wait for port to become enabled */
3094  for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
3095 
3096  /* Check port status */
3097  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3098  if ( portsc & XHCI_PORTSC_PED )
3099  return 0;
3100 
3101  /* Delay */
3102  mdelay ( 1 );
3103  }
3104 
3105  DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n",
3106  xhci->name, port->address );
3107  return -ETIMEDOUT;
3108 }
3109 
3110 /**
3111  * Disable port
3112  *
3113  * @v hub USB hub
3114  * @v port USB port
3115  * @ret rc Return status code
3116  */
3117 static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
3118  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3119  uint32_t portsc;
3120 
3121  /* Disable port */
3122  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3123  portsc &= XHCI_PORTSC_PRESERVE;
3124  portsc |= XHCI_PORTSC_PED;
3125  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3126 
3127  /* Allow time for link state to stabilise */
3129 
3130  /* Set link state to RxDetect for USB3 ports */
3131  if ( port->protocol >= USB_PROTO_3_0 ) {
3132  portsc &= XHCI_PORTSC_PRESERVE;
3133  portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS );
3134  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3135  }
3136 
3137  /* Allow time for link state to stabilise */
3139 
3140  return 0;
3141 }
3142 
3143 /**
3144  * Update root hub port speed
3145  *
3146  * @v hub USB hub
3147  * @v port USB port
3148  * @ret rc Return status code
3149  */
3150 static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
3151  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3152  uint32_t portsc;
3153  unsigned int psiv;
3154  int ccs;
3155  int ped;
3156  int csc;
3157  int speed;
3158  int rc;
3159 
3160  /* Read port status */
3161  portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
3162  DBGC2 ( xhci, "XHCI %s-%d status is %08x\n",
3163  xhci->name, port->address, portsc );
3164  ccs = ( portsc & XHCI_PORTSC_CCS );
3165  ped = ( portsc & XHCI_PORTSC_PED );
3166  csc = ( portsc & XHCI_PORTSC_CSC );
3167  psiv = XHCI_PORTSC_PSIV ( portsc );
3168 
3169  /* Record disconnections and clear changes */
3170  port->disconnected |= csc;
3171  portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
3172  writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
3173 
3174  /* Port speed is not valid unless port is connected */
3175  if ( ! ccs ) {
3176  port->speed = USB_SPEED_NONE;
3177  return 0;
3178  }
3179 
3180  /* For USB2 ports, the PSIV field is not valid until the port
3181  * completes reset and becomes enabled.
3182  */
3183  if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
3184  port->speed = USB_SPEED_FULL;
3185  return 0;
3186  }
3187 
3188  /* Get port speed and map to generic USB speed */
3189  speed = xhci_port_speed ( xhci, port->address, psiv );
3190  if ( speed < 0 ) {
3191  rc = speed;
3192  return rc;
3193  }
3194 
3195  port->speed = speed;
3196  return 0;
3197 }
3198 
3199 /**
3200  * Clear transaction translator buffer
3201  *
3202  * @v hub USB hub
3203  * @v port USB port
3204  * @v ep USB endpoint
3205  * @ret rc Return status code
3206  */
3207 static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
3208  struct usb_endpoint *ep ) {
3209  struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
3210 
3211  /* Should never be called; this is a root hub */
3212  DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name,
3213  port->address, ep->usb->name, usb_endpoint_name ( ep ) );
3214 
3215  return -ENOTSUP;
3216 }
3217 
3218 /******************************************************************************
3219  *
3220  * PCI interface
3221  *
3222  ******************************************************************************
3223  */
3224 
3225 /** USB host controller operations */
3227  .endpoint = {
3229  .close = xhci_endpoint_close,
3230  .reset = xhci_endpoint_reset,
3231  .mtu = xhci_endpoint_mtu,
3232  .message = xhci_endpoint_message,
3233  .stream = xhci_endpoint_stream,
3234  },
3235  .device = {
3236  .open = xhci_device_open,
3237  .close = xhci_device_close,
3238  .address = xhci_device_address,
3239  },
3240  .bus = {
3241  .open = xhci_bus_open,
3242  .close = xhci_bus_close,
3243  .poll = xhci_bus_poll,
3244  },
3245  .hub = {
3246  .open = xhci_hub_open,
3247  .close = xhci_hub_close,
3248  },
3249  .root = {
3250  .open = xhci_root_open,
3251  .close = xhci_root_close,
3252  .enable = xhci_root_enable,
3253  .disable = xhci_root_disable,
3254  .speed = xhci_root_speed,
3255  .clear_tt = xhci_root_clear_tt,
3256  },
3257 };
3258 
3259 /**
3260  * Fix Intel PCH-specific quirks
3261  *
3262  * @v xhci xHCI device
3263  * @v pci PCI device
3264  */
3265 static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) {
3266  struct xhci_pch *pch = &xhci->pch;
3267  uint32_t xusb2pr;
3268  uint32_t xusb2prm;
3270  uint32_t usb3prm;
3271 
3272  /* Enable SuperSpeed capability. Do this before rerouting
3273  * USB2 ports, so that USB3 devices connect at SuperSpeed.
3274  */
3276  pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm );
3277  if ( usb3prm & ~usb3pssen ) {
3278  DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n",
3279  xhci->name, ( usb3prm & ~usb3pssen ) );
3280  }
3281  pch->usb3pssen = usb3pssen;
3282  usb3pssen |= usb3prm;
3284 
3285  /* Route USB2 ports from EHCI to xHCI */
3287  pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm );
3288  if ( xusb2prm & ~xusb2pr ) {
3289  DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n",
3290  xhci->name, ( xusb2prm & ~xusb2pr ) );
3291  }
3292  pch->xusb2pr = xusb2pr;
3293  xusb2pr |= xusb2prm;
3295 }
3296 
3297 /**
3298  * Undo Intel PCH-specific quirk fixes
3299  *
3300  * @v xhci xHCI device
3301  * @v pci PCI device
3302  */
3303 static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) {
3304  struct xhci_pch *pch = &xhci->pch;
3305 
3306  /* Restore USB2 port routing to original state */
3308 
3309  /* Restore SuperSpeed capability to original state */
3311 }
3312 
3313 /**
3314  * Probe PCI device
3315  *
3316  * @v pci PCI device
3317  * @ret rc Return status code
3318  */
3319 static int xhci_probe ( struct pci_device *pci ) {
3320  struct xhci_device *xhci;
3321  struct usb_port *port;
3322  unsigned long bar_start;
3323  size_t bar_size;
3324  unsigned int i;
3325  int rc;
3326 
3327  /* Allocate and initialise structure */
3328  xhci = zalloc ( sizeof ( *xhci ) );
3329  if ( ! xhci ) {
3330  rc = -ENOMEM;
3331  goto err_alloc;
3332  }
3333  xhci->name = pci->dev.name;
3334  xhci->quirks = pci->id->driver_data;
3335 
3336  /* Fix up PCI device */
3337  adjust_pci_device ( pci );
3338 
3339  /* Map registers */
3340  bar_start = pci_bar_start ( pci, XHCI_BAR );
3341  bar_size = pci_bar_size ( pci, XHCI_BAR );
3342  xhci->regs = pci_ioremap ( pci, bar_start, bar_size );
3343  if ( ! xhci->regs ) {
3344  rc = -ENODEV;
3345  goto err_ioremap;
3346  }
3347 
3348  /* Initialise xHCI device */
3349  xhci_init ( xhci, xhci->regs );
3350 
3351  /* Configure DMA device */
3352  xhci->dma = &pci->dma;
3353  if ( xhci->addr64 )
3354  dma_set_mask_64bit ( xhci->dma );
3355 
3356  /* Initialise USB legacy support and claim ownership */
3357  xhci_legacy_init ( xhci );
3358  xhci_legacy_claim ( xhci );
3359 
3360  /* Fix Intel PCH-specific quirks, if applicable */
3361  if ( xhci->quirks & XHCI_PCH )
3362  xhci_pch_fix ( xhci, pci );
3363 
3364  /* Reset device */
3365  if ( ( rc = xhci_reset ( xhci ) ) != 0 )
3366  goto err_reset;
3367 
3368  /* Allocate USB bus */
3369  xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports, XHCI_MTU,
3370  &xhci_operations );
3371  if ( ! xhci->bus ) {
3372  rc = -ENOMEM;
3373  goto err_alloc_bus;
3374  }
3375  usb_bus_set_hostdata ( xhci->bus, xhci );
3376  usb_hub_set_drvdata ( xhci->bus->hub, xhci );
3377 
3378  /* Set port protocols */
3379  for ( i = 1 ; i <= xhci->ports ; i++ ) {
3380  port = usb_port ( xhci->bus->hub, i );
3381  port->protocol = xhci_port_protocol ( xhci, i );
3382  }
3383 
3384  /* Register USB bus */
3385  if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
3386  goto err_register;
3387 
3388  pci_set_drvdata ( pci, xhci );
3389  return 0;
3390 
3391  unregister_usb_bus ( xhci->bus );
3392  err_register:
3393  free_usb_bus ( xhci->bus );
3394  err_alloc_bus:
3395  xhci_reset ( xhci );
3396  err_reset:
3397  if ( xhci->quirks & XHCI_PCH )
3398  xhci_pch_undo ( xhci, pci );
3399  xhci_legacy_release ( xhci );
3400  iounmap ( xhci->regs );
3401  err_ioremap:
3402  free ( xhci );
3403  err_alloc:
3404  return rc;
3405 }
3406 
3407 /**
3408  * Remove PCI device
3409  *
3410  * @v pci PCI device
3411  */
3412 static void xhci_remove ( struct pci_device *pci ) {
3413  struct xhci_device *xhci = pci_get_drvdata ( pci );
3414  struct usb_bus *bus = xhci->bus;
3415 
3416  unregister_usb_bus ( bus );
3417  free_usb_bus ( bus );
3418  xhci_reset ( xhci );
3419  if ( xhci->quirks & XHCI_PCH )
3420  xhci_pch_undo ( xhci, pci );
3421  xhci_legacy_release ( xhci );
3422  iounmap ( xhci->regs );
3423  free ( xhci );
3424 }
3425 
3426 /** XHCI PCI device IDs */
3427 static struct pci_device_id xhci_ids[] = {
3428  PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ),
3429  PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ),
3430  PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
3431 };
3432 
3433 /** XHCI PCI driver */
3434 struct pci_driver xhci_driver __pci_driver = {
3435  .ids = xhci_ids,
3436  .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
3439  .probe = xhci_probe,
3440  .remove = xhci_remove,
3441 };
3442 
3443 /**
3444  * Prepare for exit
3445  *
3446  * @v booting System is shutting down for OS boot
3447  */
3448 static void xhci_shutdown ( int booting ) {
3449  /* If we are shutting down to boot an OS, then prevent the
3450  * release of ownership back to BIOS.
3451  */
3452  xhci_legacy_prevent_release = booting;
3453 }
3454 
3455 /** Startup/shutdown function */
3456 struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = {
3457  .name = "xhci",
3458  .shutdown = xhci_shutdown,
3459 };
#define XHCI_BAD_PSIV
Invalid protocol speed ID values quirk.
Definition: xhci.h:1042
An endpoint context.
Definition: xhci.h:769
Short packet.
Definition: xhci.h:637
static __always_inline void struct dma_mapping size_t size_t align
Definition: dma.h:222
#define iob_pull(iobuf, len)
Definition: iobuf.h:102
uint8_t burst
Maximum burst size.
Definition: xhci.h:781
#define __attribute__(x)
Definition: compiler.h:10
An event ring.
Definition: xhci.h:869
#define XHCI_USBLEGSUP_MAX_WAIT_MS
Maximum time to wait for BIOS to release ownership.
Definition: xhci.h:984
#define XHCI_SUPPORTED_PORTS
Supported protocol ports.
Definition: xhci.h:113
A disable slot transfer request block.
Definition: xhci.h:485
static int xhci_root_speed(struct usb_hub *hub, struct usb_port *port)
Update root hub port speed.
Definition: xhci.c:3150
#define XHCI_SUPPORTED_SLOT_TYPE(slot)
Supported protocol slot type.
Definition: xhci.h:128
static void usb_endpoint_set_hostdata(struct usb_endpoint *ep, void *priv)
Set USB endpoint host controller private data.
Definition: usb.h:561
static void xhci_command_free(struct xhci_device *xhci)
Free command ring.
Definition: xhci.c:1456
#define XHCI_TRB_TC
Transfer request block toggle cycle bit flag.
Definition: xhci.h:332
uint8_t type
Endpoint type.
Definition: xhci.h:779
#define PCI_CLASS_ID(base, sub, progif)
Construct PCI class ID.
Definition: pci.h:180
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define XHCI_STATUS_IN
Input status direction.
Definition: xhci.h:440
const char * name
Definition: ath9k_hw.c:1984
A status stage transfer request block.
Definition: xhci.h:421
wmb()
struct xhci_trb_context context
Input context TRB.
Definition: xhci.h:703
uint64_t command
Command TRB pointer.
Definition: xhci.h:614
A transfer request block.
Definition: xhci.h:315
uint8_t readb(volatile uint8_t *io_addr)
Read byte from memory-mapped device.
uint32_t dbval
Doorbell register value.
Definition: xhci.h:865
static struct io_buffer * xhci_dequeue(struct xhci_trb_ring *ring)
Dequeue a transfer request block.
Definition: xhci.c:1328
struct dma_device dma
DMA device.
Definition: pci.h:192
struct xhci_trb_common common
Common fields.
Definition: xhci.h:687
static int xhci_root_clear_tt(struct usb_hub *hub, struct usb_port *port, struct usb_endpoint *ep)
Clear transaction translator buffer.
Definition: xhci.c:3207
static void xhci_evaluate_context_input(struct xhci_device *xhci, struct xhci_slot *slot __unused, struct xhci_endpoint *endpoint, void *input)
Populate evaluate context input context.
Definition: xhci.c:2216
static struct profiler xhci_message_profiler __profiler
Message transfer profiler.
Definition: xhci.c:47
unsigned int ports
Number of ports.
Definition: xhci.h:1091
#define XHCI_CONFIG_MAX_SLOTS_EN(slots)
Maximum device slots enabled.
Definition: xhci.h:197
static int xhci_stop(struct xhci_device *xhci)
Stop xHCI device.
Definition: xhci.c:1104
struct xhci_dcbaa dcbaa
Device context base address array.
Definition: xhci.h:1107
static int xhci_device_open(struct usb_device *usb)
Open device.
Definition: xhci.c:2725
A PCI driver.
Definition: pci.h:227
#define EBUSY
Device or resource busy.
Definition: errno.h:338
struct arbelprm_completion_queue_entry normal
Definition: arbel.h:11
#define XHCI_TRB_ENABLE_SLOT
An enable slot transfer request block.
Definition: xhci.h:482
static unsigned int unsigned int reg
Definition: myson.h:162
static physaddr_t xhci_ring_consumed(struct xhci_trb_ring *ring)
Calculate physical address of most recently consumed TRB.
Definition: xhci.h:936
#define XHCI_OP_USBSTS
USB status register.
Definition: xhci.h:164
#define XHCI_USBLEGSUP_BIOS_OWNED
USB legacy support BIOS ownership flag.
Definition: xhci.h:89
#define XHCI_PORTSC_PSIV(portsc)
Port speed ID value.
Definition: xhci.h:234
A USB hub.
Definition: usb.h:826
static int xhci_address_device(struct xhci_device *xhci, struct xhci_slot *slot)
Address device.
Definition: xhci.c:2061
#define XHCI_CTX_END
End of contexts.
Definition: xhci.h:954
#define XHCI_ICI(ctx)
Input context index.
Definition: xhci.h:960
static void xhci_hub_close(struct usb_hub *hub __unused)
Close hub.
Definition: xhci.c:3006
#define XHCI_XECP_NEXT(xecp)
Next xHCI extended capability pointer.
Definition: xhci.h:80
static unsigned int xhci_extended_capability(struct xhci_device *xhci, unsigned int id, unsigned int offset)
Find extended capability.
Definition: xhci.c:323
struct xhci_device * xhci
xHCI device
Definition: xhci.h:1132
#define XHCI_TRB_SET_TR_DEQUEUE_POINTER
A set transfer ring dequeue pointer transfer request block.
Definition: xhci.h:583
unsigned int cons
Consumer counter.
Definition: xhci.h:844
uint32_t next
Next descriptor address.
Definition: myson.h:18
static size_t xhci_device_context_offset(struct xhci_device *xhci, unsigned int ctx)
Calculate device context offset.
Definition: xhci.c:412
void * run
Runtime registers.
Definition: xhci.h:1082
An xHCI device.
Definition: xhci.h:1067
A set transfer ring dequeue pointer transfer request block.
Definition: xhci.h:567
struct usb_bus * alloc_usb_bus(struct device *dev, unsigned int ports, size_t mtu, struct usb_host_operations *op)
Allocate USB bus.
Definition: usb.c:2076
Error codes.
Low speed (1.5Mbps)
Definition: usb.h:48
#define XHCI_PORT_RESET_MAX_WAIT_MS
Maximum time to wait for a port reset to complete.
Definition: xhci.h:1016
#define USB_ENDPOINT_ATTR_TYPE_MASK
Endpoint attribute transfer type mask.
Definition: usb.h:266
size_t len
Length of transfer request blocks.
Definition: xhci.h:856
#define XHCI_SUPPORTED_SLOT
Supported protocol slot.
Definition: xhci.h:125
void * regs
Registers.
Definition: xhci.h:1069
An input control context.
Definition: xhci.h:721
#define XHCI_HCSPARAMS1_PORTS(params)
Number of ports.
Definition: xhci.h:49
static size_t xhci_align(size_t len)
Calculate buffer alignment.
Definition: xhci.c:393
unsigned long driver_data
Arbitrary driver data.
Definition: pci.h:160
uint8_t slot
Slot type.
Definition: xhci.h:476
struct xhci_trb_ring command
Command ring.
Definition: xhci.h:1113
static unsigned int xhci_ring_fill(struct xhci_trb_ring *ring)
Calculate space used in TRB ring.
Definition: xhci.h:898
A command completion event transfer request block.
Definition: xhci.h:612
#define XHCI_TRB_DISABLE_SLOT
A disable slot transfer request block.
Definition: xhci.h:501
uint8_t state
Endpoint state.
Definition: xhci.h:771
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:229
static void usb_set_hostdata(struct usb_device *usb, void *priv)
Set USB device host controller private data.
Definition: usb.h:769
uint8_t type
Type.
Definition: ena.h:16
#define XHCI_PCH_XUSB2PR
Intel PCH USB2 port routing register.
Definition: xhci.h:1030
struct xhci_trb_transfer transfer
Transfer event.
Definition: xhci.h:711
A port status change transfer request block.
Definition: xhci.h:664
uint16_t trb_len
Average TRB length.
Definition: xhci.h:787
union xhci_trb * pending
Current command (if any)
Definition: xhci.h:1117
#define XHCI_SUPPORTED_PSI_VALUE(psi)
Supported protocol PSI value.
Definition: xhci.h:134
#define XHCI_EP_TYPE(type)
Endpoint type.
Definition: xhci.h:812
physaddr_t dma_phys(struct dma_mapping *map, physaddr_t addr)
Get DMA address from physical address.
struct xhci_endpoint * endpoint[XHCI_CTX_END]
Endpoints, indexed by context ID.
Definition: xhci.h:1154
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
unsigned long user_to_phys(userptr_t userptr, off_t offset)
Convert user pointer to physical address.
static int xhci_set_tr_dequeue_pointer(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Set transfer ring dequeue pointer.
Definition: xhci.c:2340
#define XHCI_XECP_ID_SUPPORTED
Supported protocol extended capability.
Definition: xhci.h:101
#define DBGC(...)
Definition: compiler.h:505
uint8_t type
Type.
Definition: xhci.h:323
uint8_t flags
Flags.
Definition: xhci.h:356
static int xhci_context(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, unsigned int type, void(*populate)(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input))
Issue context-based command and wait for completion.
Definition: xhci.c:1969
struct dma_mapping map
DMA mapping.
Definition: xhci.h:858
struct pci_driver xhci_driver __pci_driver
XHCI PCI driver.
Definition: xhci.c:3434
#define XHCI_EP_TYPE_CONTROL
Control endpoint type.
Definition: xhci.h:815
uint8_t code
Completion code.
Definition: xhci.h:670
struct xhci_trb_disable_slot disable
Disable slot TRB.
Definition: xhci.h:701
char name[40]
Name.
Definition: device.h:75
#define XHCI_HCSPARAMS1_INTRS(params)
Number of interrupters.
Definition: xhci.h:46
Definition: bnxt_hsi.h:68
#define ENOENT
No such file or directory.
Definition: errno.h:514
static __always_inline void dma_set_mask_64bit(struct dma_device *dma)
Set 64-bit addressable space mask.
Definition: dma.h:474
An enable slot transfer request block.
Definition: xhci.h:466
struct usb_device * usb
Currently attached device (if in use)
Definition: usb.h:820
#define XHCI_SETUP_OUT
Setup stage output data direction.
Definition: xhci.h:393
static void xhci_deconfigure_endpoint_input(struct xhci_device *xhci __unused, struct xhci_slot *slot __unused, struct xhci_endpoint *endpoint, void *input)
Populate deconfigure endpoint input context.
Definition: xhci.c:2162
#define XHCI_OP_PORTHLPMC(port)
Port hardware link power management control register.
Definition: xhci.h:293
struct usb_bus * bus
USB bus.
Definition: xhci.h:1123
struct xhci_trb_enable_slot enable
Enable slot TRB.
Definition: xhci.h:699
uint8_t slot
Slot ID.
Definition: xhci.h:497
static void xhci_ring_free(struct xhci_trb_ring *ring)
Free transfer request block ring.
Definition: xhci.c:1259
struct dma_mapping buffer_map
Buffer DMA mapping.
Definition: xhci.h:1059
static int xhci_configure_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Configure endpoint.
Definition: xhci.c:2134
#define cpu_to_le64(value)
Definition: byteswap.h:108
unsigned int count
Number of page-sized scratchpad buffers.
Definition: xhci.h:1055
static int xhci_command_alloc(struct xhci_device *xhci)
Allocate command ring.
Definition: xhci.c:1425
A data stage transfer request block.
Definition: xhci.h:396
uint8_t code
Completion code.
Definition: xhci.h:597
Intel PCH quirk.
Definition: xhci.h:1019
void * db
Doorbell register.
Definition: xhci.h:863
static void xhci_init(struct xhci_device *xhci, void *regs)
Initialise device.
Definition: xhci.c:264
#define XHCI_TRB_RESET_ENDPOINT
A reset endpoint transfer request block.
Definition: xhci.h:545
Full speed (12Mbps)
Definition: xhci.h:145
uint8_t slot
Slot ID.
Definition: xhci.h:626
#define XHCI_TRANSFER_TRBS_LOG2
Number of TRBs in a transfer ring.
Definition: xhci.h:978
#define XHCI_DATA_IN
Input data direction.
Definition: xhci.h:415
uint8_t flags
Flags.
Definition: xhci.h:321
#define XHCI_TRB_PORT_STATUS
A port status change transfer request block.
Definition: xhci.h:661
static __always_inline int iob_map(struct io_buffer *iobuf, struct dma_device *dma, size_t len, int flags)
Map I/O buffer for DMA.
Definition: iobuf.h:226
#define XHCI_PORTSC_CSC
Connect status change.
Definition: xhci.h:249
An xHCI endpoint.
Definition: xhci.h:1158
uint8_t type
Type.
Definition: xhci.h:537
uint8_t direction
Transfer direction.
Definition: xhci.h:381
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
A data structure for storing profiling information.
Definition: profile.h:26
Success.
Definition: xhci.h:635
static int xhci_endpoint_stream(struct usb_endpoint *ep, struct io_buffer *iobuf, int zlp)
Enqueue stream transfer.
Definition: xhci.c:2642
Super speed (5Gbps)
Definition: usb.h:54
#define XHCI_RESET_MAX_WAIT_MS
Maximum time to wait for reset to complete.
Definition: xhci.h:996
static void profile_stop(struct profiler *profiler)
Stop profiling.
Definition: profile.h:171
#define XHCI_COMMAND_ABORT_DELAY_MS
Time to delay after aborting a command.
Definition: xhci.h:1010
#define XHCI_OP_PORTLI(port)
Port link info register.
Definition: xhci.h:290
#define XHCI_PORTSC_PRESERVE
Port status and control bits which should be preserved.
Definition: xhci.h:284
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:603
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:149
const char * name
Definition: init.h:42
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
uint64_t dequeue
Dequeue pointer.
Definition: xhci.h:569
struct device dev
Generic device.
Definition: pci.h:190
#define XHCI_TRB_STATUS
A status stage transfer request block.
Definition: xhci.h:437
static struct pci_device_id xhci_ids[]
XHCI PCI device IDs.
Definition: xhci.c:3427
unsigned int slots
Number of device slots.
Definition: xhci.h:1087
uint32_t add
Add context flags.
Definition: xhci.h:725
uint64_t dequeue
Transfer ring dequeue pointer.
Definition: xhci.h:785
struct xhci_scratchpad scratch
Scratchpad buffer.
Definition: xhci.h:1110
#define XHCI_USBLEGSUP_OS
USB legacy support OS owned semaphore.
Definition: xhci.h:92
unsigned int address
Device address, if assigned.
Definition: usb.h:718
static unsigned int xhci_ring_remaining(struct xhci_trb_ring *ring)
Calculate space remaining in TRB ring.
Definition: xhci.h:913
#define PCI_CLASS_SERIAL_USB_XHCI
xHCI USB controller
Definition: pci.h:136
uint64_t * array
Scratchpad array.
Definition: xhci.h:1061
static int xhci_endpoint_mtu(struct usb_endpoint *ep)
Update MTU.
Definition: xhci.c:2523
void writeb(uint8_t data, volatile uint8_t *io_addr)
Write byte to memory-mapped device.
unsigned int speed
Device speed.
Definition: usb.h:714
uint8_t address
USB address.
Definition: xhci.h:755
#define ECANCELED
Operation canceled.
Definition: errno.h:343
#define XHCI_TRB_ADDRESS_DEVICE
An address device transfer request block.
Definition: xhci.h:520
#define ENOTSUP
Operation not supported.
Definition: errno.h:589
uint8_t ports
Number of downstream ports.
Definition: xhci.h:747
#define STARTUP_LATE
Late startup.
Definition: init.h:64
void usb_port_changed(struct usb_port *port)
Report port status change.
Definition: usb.c:1839
#define PCI_CLASS_SERIAL
Definition: Pci22.h:272
#define ECODE(code)
Definition: xhci.c:233
Dynamic memory allocation.
static int xhci_port_psiv(struct xhci_device *xhci, unsigned int port, unsigned int speed)
Find protocol speed ID value.
Definition: xhci.c:848
#define XHCI_DATA_OUT
Output data direction.
Definition: xhci.h:418
static void xhci_transfer(struct xhci_device *xhci, struct xhci_trb_transfer *trb)
Handle transfer event.
Definition: xhci.c:1565
static void usb_bus_set_hostdata(struct usb_bus *bus, void *priv)
Set USB bus host controller private data.
Definition: usb.h:1030
unsigned int type
Endpoint type.
Definition: xhci.h:1168
struct xhci_trb_host_controller host
Host controller event.
Definition: xhci.h:717
#define XHCI_EP_DCS
Endpoint dequeue cycle state.
Definition: xhci.h:824
uint64_t input
Input context.
Definition: xhci.h:506
#define XHCI_EP_TYPE_IN
Input endpoint type.
Definition: xhci.h:818
static const char * xhci_speed_name(uint32_t psi)
Transcribe port speed (for debugging)
Definition: xhci.c:642
uint8_t status
Status.
Definition: ena.h:16
static int xhci_root_enable(struct usb_hub *hub, struct usb_port *port)
Enable port.
Definition: xhci.c:3082
#define XHCI_PORTSC_PP
Port power.
Definition: xhci.h:228
A startup/shutdown function.
Definition: init.h:41
static void xhci_bus_poll(struct usb_bus *bus)
Poll USB bus.
Definition: xhci.c:2959
#define XHCI_PORTSC_CCS
Current connect status.
Definition: xhci.h:207
void memset_user(userptr_t userptr, off_t offset, int c, size_t len)
Fill user buffer with a constant byte.
static void xhci_ring_reset(struct xhci_trb_ring *ring)
Reset transfer request block ring.
Definition: xhci.c:1243
A USB port.
Definition: usb.h:798
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:341
#define rmb()
Definition: io.h:484
#define ENOMEM
Not enough space.
Definition: errno.h:534
A USB endpoint.
Definition: usb.h:389
#define XHCI_COMMAND_MAX_WAIT_MS
Maximum time to wait for a command to complete.
Definition: xhci.h:1004
struct usb_device * usb
Underlying USB device, if any.
Definition: usb.h:832
struct dma_mapping array_map
Array DMA mapping.
Definition: xhci.h:1063
static int xhci_enable_slot(struct xhci_device *xhci, unsigned int type)
Enable slot.
Definition: xhci.c:1903
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:558
#define XHCI_OP_CRCR
Command ring control register.
Definition: xhci.h:179
union xhci_trb * trb
Transfer request blocks.
Definition: xhci.h:854
void * memcpy(void *dest, const void *src, size_t len) __nonnull
#define XHCI_TRB_COMPLETE
A command completion event transfer request block.
Definition: xhci.h:630
uint8_t slot
Slot ID.
Definition: xhci.h:605
const char * name
Name.
Definition: profile.h:28
static int xhci_probe(struct pci_device *pci)
Probe PCI device.
Definition: xhci.c:3319
struct xhci_pch pch
Intel PCH quirk.
Definition: xhci.h:1126
u8 port
Port number.
Definition: CIB_PRM.h:31
#define XHCI_CAP_RTSOFF
Runtime register space offset.
Definition: xhci.h:74
uint8_t slot
Slot ID.
Definition: xhci.h:541
A reset endpoint transfer request block.
Definition: xhci.h:529
void dma_free(struct dma_mapping *map, void *addr, size_t len)
Unmap and free DMA-coherent buffer.
static int xhci_port_slot_type(struct xhci_device *xhci, unsigned int port)
Find port slot type.
Definition: xhci.c:760
void unregister_usb_bus(struct usb_bus *bus)
Unregister USB bus.
Definition: usb.c:2147
struct usb_port * port
USB port.
Definition: usb.h:712
union xhci_trb * trb
Transfer request blocks.
Definition: xhci.h:877
uint8_t slot
Slot ID.
Definition: xhci.h:560
uint32_t drop
Drop context flags.
Definition: xhci.h:723
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
#define XHCI_EP_TYPE_PERIODIC
Periodic endpoint type.
Definition: xhci.h:821
USB 3.0.
Definition: usb.h:24
static void xhci_device_close(struct usb_device *usb)
Close device.
Definition: xhci.c:2807
int pci_read_config_dword(struct pci_device *pci, unsigned int where, uint32_t *value)
Read 32-bit dword from PCI configuration space.
#define XHCI_TRB_DATA
A data stage transfer request block.
Definition: xhci.h:412
struct xhci_trb_stop_endpoint stop
Stop endpoint TRB.
Definition: xhci.h:707
#define XHCI_MTU
Maximum transfer size.
Definition: xhci.h:28
Not connected.
Definition: usb.h:46
struct xhci_trb_template template
Template.
Definition: xhci.h:685
static void xhci_dump_port(struct xhci_device *xhci, unsigned int port)
Dump port registers.
Definition: xhci.c:483
unsigned int ctx
Context index.
Definition: xhci.h:1166
uint32_t len
Length.
Definition: xhci.h:375
#define DBGC_HDA(...)
Definition: compiler.h:506
struct usb_device * usb
USB device.
Definition: xhci.h:1134
static int xhci_device_address(struct usb_device *usb)
Assign device address.
Definition: xhci.c:2844
uint32_t xusb2pr
USB2 port routing register original value.
Definition: xhci.h:1021
static __always_inline void iob_unmap(struct io_buffer *iobuf)
Unmap I/O buffer for DMA.
Definition: iobuf.h:275
#define XHCI_PORTSC_CHANGE
Port status change mask.
Definition: xhci.h:270
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
uint8_t port
Root hub port number.
Definition: xhci.h:745
#define XHCI_SLOT_INFO(entries, hub, speed, route)
Construct slot context device info.
Definition: xhci.h:765
#define XHCI_TRB_TRANSFER
A transfer event transfer request block.
Definition: xhci.h:609
#define XHCI_USBLEGSUP_CTLSTS
USB legacy support control/status.
Definition: xhci.h:98
A setup stage transfer request block.
Definition: xhci.h:371
uint16_t mtu
Maximum packet size.
Definition: xhci.h:783
A transfer event transfer request block.
Definition: xhci.h:589
unsigned int burst
Maximum burst size.
Definition: usb.h:399
static void xhci_bus_close(struct usb_bus *bus)
Close USB bus.
Definition: xhci.c:2937
#define XHCI_PCH
Intel PCH quirk flag.
Definition: xhci.h:1027
#define XHCI_TRB_NOP_CMD
A no-op command transfer request block.
Definition: xhci.h:586
#define XHCI_TRB_SETUP
A setup stage transfer request block.
Definition: xhci.h:387
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
static __always_inline physaddr_t iob_dma(struct io_buffer *iobuf)
Get I/O buffer DMA address.
Definition: iobuf.h:264
static int xhci_root_open(struct usb_hub *hub)
Open root hub.
Definition: xhci.c:3024
#define DMA_TX
Device will read data from host memory.
Definition: dma.h:132
u32 link
Link to next descriptor.
Definition: ar9003_mac.h:68
unsigned long pci_bar_start(struct pci_device *pci, unsigned int reg)
Find the start of a PCI BAR.
Definition: pci.c:96
uint8_t type
Type.
Definition: xhci.h:379
uint32_t usb3pssen
USB3 port SuperSpeed enable register original value.
Definition: xhci.h:1023
#define USB_DIR_IN
Data transfer is from device to host.
Definition: usb.h:83
static void profile_start(struct profiler *profiler)
Start profiling.
Definition: profile.h:158
char name[32]
Name.
Definition: usb.h:710
static void xhci_legacy_claim(struct xhci_device *xhci)
Claim ownership from BIOS.
Definition: xhci.c:564
unsigned int ports
Number of ports.
Definition: usb.h:836
static void xhci_event_free(struct xhci_device *xhci)
Free event ring.
Definition: xhci.c:1539
Profiling.
#define USB_SPEED(mantissa, exponent)
Define a USB speed.
Definition: usb.h:35
uint32_t info
Device info.
Definition: xhci.h:741
struct xhci_slot * slot
xHCI slot
Definition: xhci.h:1162
#define XHCI_SUPPORTED_PORTS_PSIC(ports)
Supported protocol PSI count.
Definition: xhci.h:122
struct xhci_endpoint_context * context
Endpoint context.
Definition: xhci.h:1172
static void * dest
Definition: strings.h:176
#define cpu_to_le32(value)
Definition: byteswap.h:107
uint8_t id
Request identifier.
Definition: ena.h:12
A USB device.
Definition: usb.h:708
uint32_t revision
Entry point revision.
Definition: ib_mad.h:20
pseudo_bit_t value[0x00020]
Definition: arbel.h:13
uint8_t flags
Flags.
Definition: xhci.h:377
#define XHCI_PORTSC_LWS
Port link state write strobe.
Definition: xhci.h:243
#define XHCI_DCI(ctx)
Device context index.
Definition: xhci.h:957
static int xhci_deconfigure_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Deconfigure endpoint.
Definition: xhci.c:2188
An xHCI device slot.
Definition: xhci.h:1130
#define XHCI_OP_DCBAAP
Device context base address array pointer.
Definition: xhci.h:191
A context transfer request block.
Definition: xhci.h:504
#define XHCI_TRB_STOP_ENDPOINT
A stop endpoint transfer request block.
Definition: xhci.h:564
#define XHCI_TRB_HOST_CONTROLLER
A port status change transfer request block.
Definition: xhci.h:680
#define iob_unput(iobuf, len)
Definition: iobuf.h:135
#define XHCI_SUPPORTED_PSI(index)
Supported protocol PSI.
Definition: xhci.h:131
static int xhci_bus_open(struct usb_bus *bus)
Open USB bus.
Definition: xhci.c:2886
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static int xhci_enqueue_multi(struct xhci_trb_ring *ring, struct io_buffer *iobuf, const union xhci_trb *trbs, unsigned int count)
Enqueue multiple transfer request blocks.
Definition: xhci.c:1360
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
static void xhci_shutdown(int booting)
Prepare for exit.
Definition: xhci.c:3448
static int xhci_root_disable(struct usb_hub *hub, struct usb_port *port)
Disable port.
Definition: xhci.c:3117
uint32_t status
Status.
Definition: xhci.h:309
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:624
#define XHCI_XECP_ID(xecp)
xHCI extended capability ID
Definition: xhci.h:77
uint16_t cons
Consumer index.
Definition: ena.h:22
#define PCI_CLASS_SERIAL_USB
Definition: Pci22.h:278
A transfer request block.
Definition: xhci.h:683
static void xhci_dcbaa_free(struct xhci_device *xhci)
Free device context base address array.
Definition: xhci.c:952
unsigned long pci_bar_size(struct pci_device *pci, unsigned int reg)
Find the size of a PCI BAR.
Definition: pciextra.c:90
unsigned int usb_route_string(struct usb_device *usb)
Get USB route string.
Definition: usb.c:2266
const char * name
Name.
Definition: xhci.h:1073
PCI bus.
A PCI device.
Definition: pci.h:188
uint8_t slot
Slot ID.
Definition: xhci.h:579
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
static void xhci_root_close(struct usb_hub *hub __unused)
Close root hub.
Definition: xhci.c:3070
unsigned int id
Slot ID.
Definition: xhci.h:1136
#define XHCI_USBLEGSUP_BIOS
USB legacy support BIOS owned semaphore.
Definition: xhci.h:86
#define XHCI_USBLEGSUP_OS_OWNED
USB legacy support OS ownership flag.
Definition: xhci.h:95
#define XHCI_OP_PORTPMSC(port)
Port power management status and control register.
Definition: xhci.h:287
uint64_t * context
Context base addresses.
Definition: xhci.h:1047
#define ENODEV
No such device.
Definition: errno.h:509
#define XHCI_HCCPARAMS1_XECP(params)
xHCI extended capabilities pointer
Definition: xhci.h:68
static int xhci_scratchpad_alloc(struct xhci_device *xhci)
Allocate scratchpad buffers.
Definition: xhci.c:981
#define XHCI_BAR
xHCI PCI BAR
Definition: xhci.h:31
#define XHCI_HCCPARAMS1_ADDR64(params)
64-bit addressing capability
Definition: xhci.h:62
u32 addr
Definition: sky2.h:8
#define XHCI_CAP_CAPLENGTH
Capability register length.
Definition: xhci.h:34
static void xhci_dump(struct xhci_device *xhci)
Dump host controller registers.
Definition: xhci.c:442
unsigned char uint8_t
Definition: stdint.h:10
uint8_t nop
Definition: tcp.h:13
uint8_t slot
Slot.
Definition: edd.h:16
static int xhci_endpoint_open(struct usb_endpoint *ep)
Open endpoint.
Definition: xhci.c:2389
static void xhci_remove(struct pci_device *pci)
Remove PCI device.
Definition: xhci.c:3412
unsigned int intrs
Number of interrupters.
Definition: xhci.h:1089
static int xhci_writeq(struct xhci_device *xhci, physaddr_t value, void *reg)
Write potentially 64-bit register.
Definition: xhci.c:359
static void * usb_get_hostdata(struct usb_device *usb)
Get USB device host controller private data.
Definition: usb.h:780
unsigned int prod
Producer counter.
Definition: xhci.h:842
uint8_t type
Type.
Definition: xhci.h:493
#define XHCI_EP0_TRB_LEN
Control endpoint average TRB length.
Definition: xhci.h:827
struct xhci_trb_complete complete
Command completion event.
Definition: xhci.h:713
struct xhci_trb_set_tr_dequeue_pointer dequeue
Set transfer ring dequeue pointer TRB.
Definition: xhci.h:709
#define XHCI_SUPPORTED_PORTS_COUNT(ports)
Supported protocol port count.
Definition: xhci.h:119
static __always_inline int struct dma_mapping * map
Definition: dma.h:181
A PCI device ID list entry.
Definition: pci.h:152
static int xhci_endpoint_message(struct usb_endpoint *ep, struct io_buffer *iobuf)
Enqueue message transfer.
Definition: xhci.c:2543
#define le16_to_cpu(value)
Definition: byteswap.h:112
#define XHCI_HCSPARAMS1_SLOTS(params)
Number of device slots.
Definition: xhci.h:43
unsigned int uint32_t
Definition: stdint.h:12
A USB setup data packet.
Definition: usb.h:68
const char * usb_endpoint_name(struct usb_endpoint *ep)
Get USB endpoint name (for debugging)
Definition: usb.c:220
Scratchpad buffer.
Definition: xhci.h:1053
void * cap
Capability registers.
Definition: xhci.h:1078
struct usb_port * usb_root_hub_port(struct usb_device *usb)
Get USB root hub port.
Definition: usb.c:2286
unsigned int xecp
xHCI extended capabilities offset
Definition: xhci.h:1098
static void xhci_abort(struct xhci_device *xhci)
Abort command.
Definition: xhci.c:1779
#define XHCI_OP_PAGESIZE
Page size register.
Definition: xhci.h:170
#define XHCI_RUN_ERSTBA(intr)
Event ring segment table base address register.
Definition: xhci.h:299
#define XHCI_SUPPORTED_REVISION
Supported protocol revision.
Definition: xhci.h:104
A normal transfer request block.
Definition: xhci.h:350
struct i386_regs regs
Definition: registers.h:15
#define XHCI_TRB_IDT
Transfer request block immediate data flag.
Definition: xhci.h:341
#define XHCI_XECP_ID_LEGACY
USB legacy support extended capability.
Definition: xhci.h:83
int addr64
64-bit addressing capability
Definition: xhci.h:1094
size_t mtu
Maximum transfer size.
Definition: usb.h:397
#define XHCI_USBCMD_HCRST
Host controller reset.
Definition: xhci.h:161
A port status change transfer request block.
Definition: xhci.h:643
uint16_t request
Request.
Definition: usb.h:70
uint8_t port
Port ID.
Definition: xhci.h:647
struct xhci_trb_normal normal
Normal TRB.
Definition: xhci.h:689
struct startup_fn xhci_startup __startup_fn(STARTUP_LATE)
Startup/shutdown function.
#define DMA_RX
Device will write data to host memory.
Definition: dma.h:135
static void xhci_configure_endpoint_input(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input)
Populate configure endpoint input context.
Definition: xhci.c:2094
void * dma_alloc(struct dma_device *dma, struct dma_mapping *map, size_t len, size_t align)
Allocate and map DMA-coherent buffer.
static int xhci_reset(struct xhci_device *xhci)
Reset xHCI device.
Definition: xhci.c:1136
struct usb_port * usb_transaction_translator(struct usb_device *usb)
Get USB transaction translator.
Definition: usb.c:2302
userptr_t buffer
Scratchpad buffer area.
Definition: xhci.h:1057
unsigned long physaddr_t
Definition: stdint.h:20
#define __unused
Declare a variable or data structure as unused.
Definition: compiler.h:573
static void xhci_run(struct xhci_device *xhci)
Start xHCI device.
Definition: xhci.c:1082
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:351
Endpoint is running.
Definition: xhci.h:799
#define XHCI_DBVAL(target, stream)
Calculate doorbell register value.
Definition: xhci.h:889
static struct io_buffer * xhci_dequeue_multi(struct xhci_trb_ring *ring)
Dequeue multiple transfer request blocks.
Definition: xhci.c:1389
#define XHCI_PAGESIZE(pagesize)
Page size.
Definition: xhci.h:173
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
uint8_t type
Type.
Definition: xhci.h:556
#define XHCI_PORTSC_PLS_RXDETECT
RxDetect port link state.
Definition: xhci.h:222
struct xhci_trb_reset_endpoint reset
Reset endpoint TRB.
Definition: xhci.h:705
#define XHCI_TRB_CONFIGURE_ENDPOINT
A configure endpoint transfer request block.
Definition: xhci.h:523
#define XHCI_SUPPORTED_PORTS_OFFSET(ports)
Supported protocol port offset.
Definition: xhci.h:116
Command ring stopped.
Definition: xhci.h:639
static unsigned int xhci_supported_protocol(struct xhci_device *xhci, unsigned int port)
Find supported protocol extended capability for a port.
Definition: xhci.c:665
#define XHCI_OP_CONFIG
Configure register.
Definition: xhci.h:194
USB eXtensible Host Controller Interface (xHCI) driver.
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
void dma_ufree(struct dma_mapping *map, userptr_t addr, size_t len)
Unmap and free DMA-coherent buffer from external (user) memory.
#define XHCI_PCH_USB3PSSEN
Intel PCH SuperSpeed enable register.
Definition: xhci.h:1036
#define XHCI_OP_PORTSC(port)
Port status and control register.
Definition: xhci.h:204
static void * usb_bus_get_hostdata(struct usb_bus *bus)
Get USB bus host controller private data.
Definition: usb.h:1041
static int xhci_disable_slot(struct xhci_device *xhci, unsigned int slot)
Disable slot.
Definition: xhci.c:1937
uint8_t tt_id
TT hub slot ID.
Definition: xhci.h:749
static void xhci_pch_fix(struct xhci_device *xhci, struct pci_device *pci)
Fix Intel PCH-specific quirks.
Definition: xhci.c:3265
struct usb_endpoint * ep
USB endpoint.
Definition: xhci.h:1164
uint32_t control
Control.
Definition: xhci.h:311
uint32_t len
Length.
Definition: ena.h:14
#define XHCI_CTX(address)
Calculate context index from USB endpoint address.
Definition: xhci.h:946
#define XHCI_STOP_MAX_WAIT_MS
Maximum time to wait for host controller to stop.
Definition: xhci.h:990
#define XHCI_TRB_NORMAL
A normal transfer request block.
Definition: xhci.h:364
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
#define DBGC2(...)
Definition: compiler.h:522
size_t pagesize
Page size.
Definition: xhci.h:1101
Universal Serial Bus (USB)
#define XHCI_PORT_POWER_DELAY_MS
Time to delay after enabling power to a port.
Definition: xhci.h:231
int register_usb_bus(struct usb_bus *bus)
Register USB bus.
Definition: usb.c:2112
static void xhci_event_poll(struct xhci_device *xhci)
Poll event ring.
Definition: xhci.c:1714
#define XHCI_CRCR_RCS
Command ring cycle state.
Definition: xhci.h:182
A stop endpoint transfer request block.
Definition: xhci.h:548
unsigned int mask
Ring counter mask.
Definition: xhci.h:848
static void xhci_scratchpad_free(struct xhci_device *xhci)
Free scratchpad buffers.
Definition: xhci.c:1048
#define XHCI_TRB_TYPE_MASK
Transfer request block type mask.
Definition: xhci.h:347
static void usb_complete(struct usb_endpoint *ep, struct io_buffer *iobuf)
Complete transfer (without error)
Definition: usb.h:1066
uint8_t interval
Polling interval.
Definition: xhci.h:775
uint8_t tt_port
TT port number.
Definition: xhci.h:751
unsigned int address
Port address.
Definition: usb.h:802
void * data
Start of data.
Definition: iobuf.h:48
static void xhci_host_controller(struct xhci_device *xhci, struct xhci_trb_host_controller *trb)
Handle host controller event.
Definition: xhci.c:1699
struct usb_hub * hub
USB hub.
Definition: usb.h:800
struct dma_mapping map
DMA mapping.
Definition: xhci.h:1049
struct io_buffer ** iobuf
I/O buffers.
Definition: xhci.h:851
#define XHCI_TRB_EVALUATE_CONTEXT
An evaluate context transfer request block.
Definition: xhci.h:526
uint16_t count
Number of entries.
Definition: ena.h:22
static void xhci_complete(struct xhci_device *xhci, struct xhci_trb_complete *trb)
Handle command completion event.
Definition: xhci.c:1638
#define XHCI_CAP_DBOFF
Doorbell offset.
Definition: xhci.h:71
High speed (480Mbps)
Definition: usb.h:52
struct usb_endpoint * ep[32]
Endpoint list.
Definition: usb.h:730
static int xhci_port_speed(struct xhci_device *xhci, unsigned int port, unsigned int psiv)
Find port speed.
Definition: xhci.c:784
struct xhci_trb_ring ring
Transfer ring.
Definition: xhci.h:1174
unsigned int interval
Interval (in microframes)
Definition: usb.h:401
struct pci_device_id * id
Driver device ID.
Definition: pci.h:223
static int xhci_stop_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Stop endpoint.
Definition: xhci.c:2307
static size_t xhci_input_context_offset(struct xhci_device *xhci, unsigned int ctx)
Calculate input context offset.
Definition: xhci.c:424
#define XHCI_HCCPARAMS1_CSZ_SHIFT(params)
Context size shift.
Definition: xhci.h:65
#define XHCI_CRCR_CA
Command abort.
Definition: xhci.h:185
struct usb_endpoint_host_operations endpoint
Endpoint operations.
Definition: usb.h:1012
int pci_write_config_dword(struct pci_device *pci, unsigned int where, uint32_t value)
Write 32-bit dword to PCI configuration space.
#define XHCI_PCH_USB3PRM
Intel PCH USB3 port routing mask register.
Definition: xhci.h:1039
#define cpu_to_le16(value)
Definition: byteswap.h:106
static void xhci_pch_undo(struct xhci_device *xhci, struct pci_device *pci)
Undo Intel PCH-specific quirk fixes.
Definition: xhci.c:3303
#define XHCI_USBCMD_RUN
Run/stop.
Definition: xhci.h:158
void iounmap(volatile const void *io_addr)
Unmap I/O address.
#define XHCI_TRB_CH
Transfer request block chain flag.
Definition: xhci.h:335
#define XHCI_CAP_HCSPARAMS1
Structural parameters 1.
Definition: xhci.h:40
USB host controller operations.
Definition: usb.h:1010
#define USB_ENDPOINT_ATTR_CONTROL
Control endpoint transfer type.
Definition: usb.h:272
struct xhci_trb_port_status port
Port status changed event.
Definition: xhci.h:715
struct dma_device * dma
DMA device.
Definition: xhci.h:1071
static int xhci_endpoint_reset(struct usb_endpoint *ep)
Reset endpoint.
Definition: xhci.c:2495
static int xhci_enqueue(struct xhci_trb_ring *ring, struct io_buffer *iobuf, const union xhci_trb *trb)
Enqueue a transfer request block.
Definition: xhci.c:1285
static int xhci_evaluate_context(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Evaluate context.
Definition: xhci.c:2247
static void xhci_doorbell(struct xhci_trb_ring *ring)
Ring doorbell register.
Definition: xhci.c:1406
static unsigned int xhci_port_protocol(struct xhci_device *xhci, unsigned int port)
Find port protocol.
Definition: xhci.c:699
uint16_t residual
Residual transfer length.
Definition: xhci.h:593
Super speed.
Definition: xhci.h:151
#define XHCI_PORTSC_PR
Port reset.
Definition: xhci.h:213
unsigned int legacy
USB legacy support capability (if present and enabled)
Definition: xhci.h:1104
__be32 raw[7]
Definition: CIB_PRM.h:28
struct usb_hub * hub
Root hub.
Definition: usb.h:974
#define XHCI_CRCR_CRR
Command ring running.
Definition: xhci.h:188
#define XHCI_CONFIG_MAX_SLOTS_EN_MASK
Maximum device slots enabled mask.
Definition: xhci.h:200
#define XHCI_PORTSC_PED
Port enabled.
Definition: xhci.h:210
struct xhci_trb_link * link
Link TRB (if applicable)
Definition: xhci.h:860
#define XHCI_CAP_HCCPARAMS1
Capability parameters.
Definition: xhci.h:59
int snprintf(char *buf, size_t size, const char *fmt,...)
Write a formatted string to a buffer.
Definition: vsprintf.c:382
Low speed (1.5Mbps)
Definition: xhci.h:147
unsigned int shift
Ring size (log2)
Definition: xhci.h:846
A transfer request block command/transfer ring.
Definition: xhci.h:840
userptr_t dma_umalloc(struct dma_device *dma, struct dma_mapping *map, size_t len, size_t align)
Allocate and map DMA-coherent buffer from external (user) memory.
static void usb_hub_set_drvdata(struct usb_hub *hub, void *priv)
Set USB hub driver private data.
Definition: usb.h:922
struct usb_device * usb
USB device.
Definition: usb.h:391
static void xhci_address_device_input(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint, void *input)
Populate address device input context.
Definition: xhci.c:2019
uint64_t transfer
Transfer TRB pointer.
Definition: xhci.h:591
static struct usb_port * usb_port(struct usb_hub *hub, unsigned int address)
Get USB port.
Definition: usb.h:945
unsigned int csz_shift
Context size shift.
Definition: xhci.h:1096
#define XHCI_EVENT_TRBS_LOG2
Number of TRBs in the event ring.
Definition: xhci.h:972
struct usb_setup_packet packet
Setup packet.
Definition: xhci.h:373
#define XHCI_SUPPORTED_PSI_MANTISSA(psi)
Supported protocol PSI mantissa.
Definition: xhci.h:137
#define XHCI_MIN_ALIGN
Minimum alignment required for data structures.
Definition: xhci.h:25
uint16_t protocol
Protocol ID.
Definition: stp.h:18
void * db
Doorbell registers.
Definition: xhci.h:1084
#define XHCI_TRB_LINK
A link transfer request block.
Definition: xhci.h:460
unsigned int interval
Endpoint interval.
Definition: xhci.h:1170
#define XHCI_SETUP_IN
Setup stage input data direction.
Definition: xhci.h:390
#define XHCI_PORTSC_PLS_DISABLED
Disabled port link state.
Definition: xhci.h:219
static int xhci_reset_endpoint(struct xhci_device *xhci, struct xhci_slot *slot, struct xhci_endpoint *endpoint)
Reset endpoint.
Definition: xhci.c:2275
#define DBG_EXTRA
Definition: compiler.h:319
int(* open)(struct usb_endpoint *ep)
Open endpoint.
Definition: usb.h:435
#define XHCI_SUPPORTED_REVISION_VER(revision)
Supported protocol minor revision.
Definition: xhci.h:107
#define XHCI_OP_USBCMD
USB command register.
Definition: xhci.h:155
struct arbelprm_port_state_change_st data
Message.
Definition: arbel.h:12
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:539
#define XHCI_RUN_ERSTSZ(intr)
Event ring segment table size register.
Definition: xhci.h:296
#define XHCI_HCSPARAMS2_SCRATCHPADS(params)
Number of page-sized scratchpad buffers.
Definition: xhci.h:55
#define XHCI_STATUS_OUT
Output status direction.
Definition: xhci.h:443
uint8_t code
Completion code.
Definition: xhci.h:618
static __always_inline physaddr_t dma(struct dma_mapping *map, void *addr)
Get DMA address from virtual address.
Definition: dma.h:436
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
#define XHCI_PCH_XUSB2PRM
Intel PCH USB2 port routing mask register.
Definition: xhci.h:1033
#define XHCI_USBSTS_HCH
Host controller halted.
Definition: xhci.h:167
struct usb_bus * bus
USB bus.
Definition: usb.h:830
#define fls(x)
Find last (i.e.
Definition: strings.h:166
#define le64_to_cpu(value)
Definition: byteswap.h:114
struct xhci_trb_link link
Link TRB.
Definition: xhci.h:697
static void xhci_legacy_init(struct xhci_device *xhci)
Initialise USB legacy support.
Definition: xhci.c:533
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
#define DBG_LOG
Definition: compiler.h:317
struct xhci_slot ** slot
Device slots, indexed by slot ID.
Definition: xhci.h:1120
#define XHCI_TRB_C
Transfer request block cycle bit flag.
Definition: xhci.h:329
#define XHCI_CAP_HCSPARAMS2
Structural parameters 2.
Definition: xhci.h:52
High speed (480Mbps)
Definition: xhci.h:149
A DMA mapping.
Definition: dma.h:32
static int xhci_event_alloc(struct xhci_device *xhci)
Allocate event ring.
Definition: xhci.c:1474
static int xhci_command(struct xhci_device *xhci, union xhci_trb *trb)
Issue command and wait for completion.
Definition: xhci.c:1811
unsigned int quirks
Quirks.
Definition: xhci.h:1075
#define NULL
NULL pointer (VOID *)
Definition: Base.h:362
uint64_t parameter
Parameter.
Definition: xhci.h:307
#define XHCI_SUPPORTED_PSI_EXPONENT(psi)
Supported protocol PSI exponent.
Definition: xhci.h:140
struct xhci_device * xhci
xHCI device
Definition: xhci.h:1160
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:669
String functions.
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:286
struct xhci_event_ring event
Event ring.
Definition: xhci.h:1115
static unsigned int xhci_endpoint_count(size_t len, int zlp)
Calculate number of TRBs.
Definition: xhci.c:2621
A USB bus.
Definition: usb.h:951
#define XHCI_LINK_STATE_DELAY_MS
Time to delay after writing the port link state.
Definition: xhci.h:246
unsigned int attributes
Attributes.
Definition: usb.h:395
static int xhci_legacy_prevent_release
Prevent the release of ownership back to BIOS.
Definition: xhci.c:526
#define XHCI_OP_DNCTRL
Device notifcation control register.
Definition: xhci.h:176
uint8_t bus
Bus.
Definition: edd.h:14
Full speed (12Mbps)
Definition: usb.h:50
void writeq(uint64_t data, volatile uint64_t *io_addr)
Write 64-bit qword to memory-mapped device.
void * op
Operational registers.
Definition: xhci.h:1080
unsigned int address
Endpoint address.
Definition: usb.h:393
#define XHCI_RUN_ERDP(intr)
Event ring dequeue pointer register.
Definition: xhci.h:302
uint8_t type
Type.
Definition: xhci.h:474
void free_usb_bus(struct usb_bus *bus)
Free USB bus.
Definition: usb.c:2171
static int xhci_nop(struct xhci_device *xhci)
Issue NOP and wait for completion.
Definition: xhci.c:1875
#define XHCI_CTX_EP0
Endpoint zero context index.
Definition: xhci.h:951
A slot context.
Definition: xhci.h:739
#define XHCI_PORTSC_PLS_MASK
Port link state mask.
Definition: xhci.h:225
uint8_t endpoint
Endpoint ID.
Definition: xhci.h:577
#define XHCI_TRB_IOC
Transfer request block interrupt on completion flag.
Definition: xhci.h:338
static int xhci_hub_open(struct usb_hub *hub)
Open hub.
Definition: xhci.c:2979
uint8_t type
Type.
Definition: xhci.h:512
static void xhci_port_status(struct xhci_device *xhci, struct xhci_trb_port_status *trb)
Handle port status event.
Definition: xhci.c:1675
static int xhci_ring_alloc(struct xhci_device *xhci, struct xhci_trb_ring *ring, unsigned int shift, unsigned int slot, unsigned int target, unsigned int stream)
Allocate transfer request block ring.
Definition: xhci.c:1186
void usb_complete_err(struct usb_endpoint *ep, struct io_buffer *iobuf, int rc)
Complete transfer (possibly with error)
Definition: usb.c:586
#define XHCI_ENDPOINT_STATE_MASK
Endpoint state mask.
Definition: xhci.h:809
static void * usb_hub_get_drvdata(struct usb_hub *hub)
Get USB hub driver private data.
Definition: usb.h:933
#define XHCI_CMD_TRBS_LOG2
Number of TRBs (excluding Link TRB) in the command ring.
Definition: xhci.h:966
static struct usb_host_operations xhci_operations
USB host controller operations.
Definition: xhci.c:3226
static void * usb_endpoint_get_hostdata(struct usb_endpoint *ep)
Get USB endpoint host controller private data.
Definition: usb.h:572
String functions.
static void xhci_legacy_release(struct xhci_device *xhci)
Release ownership back to BIOS.
Definition: xhci.c:611
uint8_t slot
Slot ID.
Definition: xhci.h:516
static int xhci_dcbaa_alloc(struct xhci_device *xhci)
Allocate device context base address array.
Definition: xhci.c:910
#define XHCI_SUPPORTED_NAME
Supported protocol name.
Definition: xhci.h:110
void * memset(void *dest, int character, size_t len) __nonnull
#define XHCI_CTX_SLOT
Slot context index.
Definition: xhci.h:943
static void xhci_endpoint_close(struct usb_endpoint *ep)
Close endpoint.
Definition: xhci.c:2465
A persistent I/O buffer.
Definition: iobuf.h:33