iPXE
xhci.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or (at your option) any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdlib.h>
00027 #include <stdio.h>
00028 #include <unistd.h>
00029 #include <string.h>
00030 #include <strings.h>
00031 #include <errno.h>
00032 #include <byteswap.h>
00033 #include <ipxe/malloc.h>
00034 #include <ipxe/umalloc.h>
00035 #include <ipxe/pci.h>
00036 #include <ipxe/usb.h>
00037 #include <ipxe/init.h>
00038 #include <ipxe/profile.h>
00039 #include "xhci.h"
00040 
00041 /** @file
00042  *
00043  * USB eXtensible Host Controller Interface (xHCI) driver
00044  *
00045  */
00046 
00047 /** Message transfer profiler */
00048 static struct profiler xhci_message_profiler __profiler =
00049         { .name = "xhci.message" };
00050 
00051 /** Stream transfer profiler */
00052 static struct profiler xhci_stream_profiler __profiler =
00053         { .name = "xhci.stream" };
00054 
00055 /** Event ring profiler */
00056 static struct profiler xhci_event_profiler __profiler =
00057         { .name = "xhci.event" };
00058 
00059 /** Transfer event profiler */
00060 static struct profiler xhci_transfer_profiler __profiler =
00061         { .name = "xhci.transfer" };
00062 
00063 /* Disambiguate the various error causes */
00064 #define EIO_DATA                                                        \
00065         __einfo_error ( EINFO_EIO_DATA )
00066 #define EINFO_EIO_DATA                                                  \
00067         __einfo_uniqify ( EINFO_EIO, ( 2 - 0 ),                         \
00068                           "Data buffer error" )
00069 #define EIO_BABBLE                                                      \
00070         __einfo_error ( EINFO_EIO_BABBLE )
00071 #define EINFO_EIO_BABBLE                                                \
00072         __einfo_uniqify ( EINFO_EIO, ( 3 - 0 ),                         \
00073                           "Babble detected" )
00074 #define EIO_USB                                                         \
00075         __einfo_error ( EINFO_EIO_USB )
00076 #define EINFO_EIO_USB                                                   \
00077         __einfo_uniqify ( EINFO_EIO, ( 4 - 0 ),                         \
00078                           "USB transaction error" )
00079 #define EIO_TRB                                                         \
00080         __einfo_error ( EINFO_EIO_TRB )
00081 #define EINFO_EIO_TRB                                                   \
00082         __einfo_uniqify ( EINFO_EIO, ( 5 - 0 ),                         \
00083                           "TRB error" )
00084 #define EIO_STALL                                                       \
00085         __einfo_error ( EINFO_EIO_STALL )
00086 #define EINFO_EIO_STALL                                                 \
00087         __einfo_uniqify ( EINFO_EIO, ( 6 - 0 ),                         \
00088                           "Stall error" )
00089 #define EIO_RESOURCE                                                    \
00090         __einfo_error ( EINFO_EIO_RESOURCE )
00091 #define EINFO_EIO_RESOURCE                                              \
00092         __einfo_uniqify ( EINFO_EIO, ( 7 - 0 ),                         \
00093                           "Resource error" )
00094 #define EIO_BANDWIDTH                                                   \
00095         __einfo_error ( EINFO_EIO_BANDWIDTH )
00096 #define EINFO_EIO_BANDWIDTH                                             \
00097         __einfo_uniqify ( EINFO_EIO, ( 8 - 0 ),                         \
00098                           "Bandwidth error" )
00099 #define EIO_NO_SLOTS                                                    \
00100         __einfo_error ( EINFO_EIO_NO_SLOTS )
00101 #define EINFO_EIO_NO_SLOTS                                              \
00102         __einfo_uniqify ( EINFO_EIO, ( 9 - 0 ),                         \
00103                           "No slots available" )
00104 #define EIO_STREAM_TYPE                                                 \
00105         __einfo_error ( EINFO_EIO_STREAM_TYPE )
00106 #define EINFO_EIO_STREAM_TYPE                                           \
00107         __einfo_uniqify ( EINFO_EIO, ( 10 - 0 ),                        \
00108                           "Invalid stream type" )
00109 #define EIO_SLOT                                                        \
00110         __einfo_error ( EINFO_EIO_SLOT )
00111 #define EINFO_EIO_SLOT                                                  \
00112         __einfo_uniqify ( EINFO_EIO, ( 11 - 0 ),                        \
00113                           "Slot not enabled" )
00114 #define EIO_ENDPOINT                                                    \
00115         __einfo_error ( EINFO_EIO_ENDPOINT )
00116 #define EINFO_EIO_ENDPOINT                                              \
00117         __einfo_uniqify ( EINFO_EIO, ( 12 - 0 ),                        \
00118                           "Endpoint not enabled" )
00119 #define EIO_SHORT                                                       \
00120         __einfo_error ( EINFO_EIO_SHORT )
00121 #define EINFO_EIO_SHORT                                                 \
00122         __einfo_uniqify ( EINFO_EIO, ( 13 - 0 ),                        \
00123                           "Short packet" )
00124 #define EIO_UNDERRUN                                                    \
00125         __einfo_error ( EINFO_EIO_UNDERRUN )
00126 #define EINFO_EIO_UNDERRUN                                              \
00127         __einfo_uniqify ( EINFO_EIO, ( 14 - 0 ),                        \
00128                           "Ring underrun" )
00129 #define EIO_OVERRUN                                                     \
00130         __einfo_error ( EINFO_EIO_OVERRUN )
00131 #define EINFO_EIO_OVERRUN                                               \
00132         __einfo_uniqify ( EINFO_EIO, ( 15 - 0 ),                        \
00133                           "Ring overrun" )
00134 #define EIO_VF_RING_FULL                                                \
00135         __einfo_error ( EINFO_EIO_VF_RING_FULL )
00136 #define EINFO_EIO_VF_RING_FULL                                          \
00137         __einfo_uniqify ( EINFO_EIO, ( 16 - 0 ),                        \
00138                           "Virtual function event ring full" )
00139 #define EIO_PARAMETER                                                   \
00140         __einfo_error ( EINFO_EIO_PARAMETER )
00141 #define EINFO_EIO_PARAMETER                                             \
00142         __einfo_uniqify ( EINFO_EIO, ( 17 - 0 ),                        \
00143                           "Parameter error" )
00144 #define EIO_BANDWIDTH_OVERRUN                                           \
00145         __einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
00146 #define EINFO_EIO_BANDWIDTH_OVERRUN                                     \
00147         __einfo_uniqify ( EINFO_EIO, ( 18 - 0 ),                        \
00148                           "Bandwidth overrun" )
00149 #define EIO_CONTEXT                                                     \
00150         __einfo_error ( EINFO_EIO_CONTEXT )
00151 #define EINFO_EIO_CONTEXT                                               \
00152         __einfo_uniqify ( EINFO_EIO, ( 19 - 0 ),                        \
00153                           "Context state error" )
00154 #define EIO_NO_PING                                                     \
00155         __einfo_error ( EINFO_EIO_NO_PING )
00156 #define EINFO_EIO_NO_PING                                               \
00157         __einfo_uniqify ( EINFO_EIO, ( 20 - 0 ),                        \
00158                           "No ping response" )
00159 #define EIO_RING_FULL                                                   \
00160         __einfo_error ( EINFO_EIO_RING_FULL )
00161 #define EINFO_EIO_RING_FULL                                             \
00162         __einfo_uniqify ( EINFO_EIO, ( 21 - 0 ),                        \
00163                           "Event ring full" )
00164 #define EIO_INCOMPATIBLE                                                \
00165         __einfo_error ( EINFO_EIO_INCOMPATIBLE )
00166 #define EINFO_EIO_INCOMPATIBLE                                          \
00167         __einfo_uniqify ( EINFO_EIO, ( 22 - 0 ),                        \
00168                           "Incompatible device" )
00169 #define EIO_MISSED                                                      \
00170         __einfo_error ( EINFO_EIO_MISSED )
00171 #define EINFO_EIO_MISSED                                                \
00172         __einfo_uniqify ( EINFO_EIO, ( 23 - 0 ),                        \
00173                           "Missed service error" )
00174 #define EIO_CMD_STOPPED                                                 \
00175         __einfo_error ( EINFO_EIO_CMD_STOPPED )
00176 #define EINFO_EIO_CMD_STOPPED                                           \
00177         __einfo_uniqify ( EINFO_EIO, ( 24 - 0 ),                        \
00178                           "Command ring stopped" )
00179 #define EIO_CMD_ABORTED                                                 \
00180         __einfo_error ( EINFO_EIO_CMD_ABORTED )
00181 #define EINFO_EIO_CMD_ABORTED                                           \
00182         __einfo_uniqify ( EINFO_EIO, ( 25 - 0 ),                        \
00183                           "Command aborted" )
00184 #define EIO_STOP                                                        \
00185         __einfo_error ( EINFO_EIO_STOP )
00186 #define EINFO_EIO_STOP                                                  \
00187         __einfo_uniqify ( EINFO_EIO, ( 26 - 0 ),                        \
00188                           "Stopped" )
00189 #define EIO_STOP_LEN                                                    \
00190         __einfo_error ( EINFO_EIO_STOP_LEN )
00191 #define EINFO_EIO_STOP_LEN                                              \
00192         __einfo_uniqify ( EINFO_EIO, ( 27 - 0 ),                        \
00193                           "Stopped - length invalid" )
00194 #define EIO_STOP_SHORT                                                  \
00195         __einfo_error ( EINFO_EIO_STOP_SHORT )
00196 #define EINFO_EIO_STOP_SHORT                                            \
00197         __einfo_uniqify ( EINFO_EIO, ( 28 - 0 ),                        \
00198                           "Stopped - short packet" )
00199 #define EIO_LATENCY                                                     \
00200         __einfo_error ( EINFO_EIO_LATENCY )
00201 #define EINFO_EIO_LATENCY                                               \
00202         __einfo_uniqify ( EINFO_EIO, ( 29 - 0 ),                        \
00203                           "Maximum exit latency too large" )
00204 #define EIO_ISOCH                                                       \
00205         __einfo_error ( EINFO_EIO_ISOCH )
00206 #define EINFO_EIO_ISOCH                                                 \
00207         __einfo_uniqify ( EINFO_EIO, ( 31 - 0 ),                        \
00208                           "Isochronous buffer overrun" )
00209 #define EPROTO_LOST                                                     \
00210         __einfo_error ( EINFO_EPROTO_LOST )
00211 #define EINFO_EPROTO_LOST                                               \
00212         __einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ),                    \
00213                           "Event lost" )
00214 #define EPROTO_UNDEFINED                                                \
00215         __einfo_error ( EINFO_EPROTO_UNDEFINED )
00216 #define EINFO_EPROTO_UNDEFINED                                          \
00217         __einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ),                    \
00218                           "Undefined error" )
00219 #define EPROTO_STREAM_ID                                                \
00220         __einfo_error ( EINFO_EPROTO_STREAM_ID )
00221 #define EINFO_EPROTO_STREAM_ID                                          \
00222         __einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ),                    \
00223                           "Invalid stream ID" )
00224 #define EPROTO_SECONDARY                                                \
00225         __einfo_error ( EINFO_EPROTO_SECONDARY )
00226 #define EINFO_EPROTO_SECONDARY                                          \
00227         __einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ),                    \
00228                           "Secondary bandwidth error" )
00229 #define EPROTO_SPLIT                                                    \
00230         __einfo_error ( EINFO_EPROTO_SPLIT )
00231 #define EINFO_EPROTO_SPLIT                                              \
00232         __einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ),                    \
00233                           "Split transaction error" )
00234 #define ECODE(code)                                                     \
00235         ( ( (code) < 32 ) ?                                             \
00236           EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE,     \
00237                   EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE,            \
00238                   EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE,         \
00239                   EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN,      \
00240                   EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER,         \
00241                   EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING,      \
00242                   EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED,          \
00243                   EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP,           \
00244                   EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY,            \
00245                   EIO_ISOCH ) :                                         \
00246           ( (code) < 64 ) ?                                             \
00247           EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST,           \
00248                   EPROTO_UNDEFINED, EPROTO_STREAM_ID,                   \
00249                   EPROTO_SECONDARY, EPROTO_SPLIT ) :                    \
00250           EFAULT )
00251 
00252 /******************************************************************************
00253  *
00254  * Register access
00255  *
00256  ******************************************************************************
00257  */
00258 
00259 /**
00260  * Initialise device
00261  *
00262  * @v xhci              xHCI device
00263  * @v regs              MMIO registers
00264  */
00265 static void xhci_init ( struct xhci_device *xhci, void *regs ) {
00266         uint32_t hcsparams1;
00267         uint32_t hcsparams2;
00268         uint32_t hccparams1;
00269         uint32_t pagesize;
00270         size_t caplength;
00271         size_t rtsoff;
00272         size_t dboff;
00273 
00274         /* Locate capability, operational, runtime, and doorbell registers */
00275         xhci->cap = regs;
00276         caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
00277         rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
00278         dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
00279         xhci->op = ( xhci->cap + caplength );
00280         xhci->run = ( xhci->cap + rtsoff );
00281         xhci->db = ( xhci->cap + dboff );
00282         DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n",
00283                 xhci->name, virt_to_phys ( xhci->cap ),
00284                 virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ),
00285                 virt_to_phys ( xhci->db ) );
00286 
00287         /* Read structural parameters 1 */
00288         hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
00289         xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
00290         xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
00291         xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
00292         DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n",
00293                xhci->name, xhci->slots, xhci->intrs, xhci->ports );
00294 
00295         /* Read structural parameters 2 */
00296         hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
00297         xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
00298         DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
00299                 xhci->name, xhci->scratchpads );
00300 
00301         /* Read capability parameters 1 */
00302         hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
00303         xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
00304         xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
00305         xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
00306 
00307         /* Read page size */
00308         pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
00309         xhci->pagesize = XHCI_PAGESIZE ( pagesize );
00310         assert ( xhci->pagesize != 0 );
00311         assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
00312         DBGC2 ( xhci, "XHCI %s page size %zd bytes\n",
00313                 xhci->name, xhci->pagesize );
00314 }
00315 
00316 /**
00317  * Find extended capability
00318  *
00319  * @v xhci              xHCI device
00320  * @v id                Capability ID
00321  * @v offset            Offset to previous extended capability instance, or zero
00322  * @ret offset          Offset to extended capability, or zero if not found
00323  */
00324 static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
00325                                                unsigned int id,
00326                                                unsigned int offset ) {
00327         uint32_t xecp;
00328         unsigned int next;
00329 
00330         /* Locate the extended capability */
00331         while ( 1 ) {
00332 
00333                 /* Locate first or next capability as applicable */
00334                 if ( offset ) {
00335                         xecp = readl ( xhci->cap + offset );
00336                         next = XHCI_XECP_NEXT ( xecp );
00337                 } else {
00338                         next = xhci->xecp;
00339                 }
00340                 if ( ! next )
00341                         return 0;
00342                 offset += next;
00343 
00344                 /* Check if this is the requested capability */
00345                 xecp = readl ( xhci->cap + offset );
00346                 if ( XHCI_XECP_ID ( xecp ) == id )
00347                         return offset;
00348         }
00349 }
00350 
00351 /**
00352  * Write potentially 64-bit register
00353  *
00354  * @v xhci              xHCI device
00355  * @v value             Value
00356  * @v reg               Register address
00357  * @ret rc              Return status code
00358  */
00359 static inline __attribute__ (( always_inline )) int
00360 xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
00361 
00362         /* If this is a 32-bit build, then this can never fail
00363          * (allowing the compiler to optimise out the error path).
00364          */
00365         if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
00366                 writel ( value, reg );
00367                 writel ( 0, ( reg + sizeof ( uint32_t ) ) );
00368                 return 0;
00369         }
00370 
00371         /* If the device does not support 64-bit addresses and this
00372          * address is outside the 32-bit address space, then fail.
00373          */
00374         if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
00375                 DBGC ( xhci, "XHCI %s cannot access address %lx\n",
00376                        xhci->name, value );
00377                 return -ENOTSUP;
00378         }
00379 
00380         /* If this is a 64-bit build, then writeq() is available */
00381         writeq ( value, reg );
00382         return 0;
00383 }
00384 
00385 /**
00386  * Calculate buffer alignment
00387  *
00388  * @v len               Length
00389  * @ret align           Buffer alignment
00390  *
00391  * Determine alignment required for a buffer which must be aligned to
00392  * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
00393  */
00394 static inline size_t xhci_align ( size_t len ) {
00395         size_t align;
00396 
00397         /* Align to own length (rounded up to a power of two) */
00398         align = ( 1 << fls ( len - 1 ) );
00399 
00400         /* Round up to XHCI_MIN_ALIGN if needed */
00401         if ( align < XHCI_MIN_ALIGN )
00402                 align = XHCI_MIN_ALIGN;
00403 
00404         return align;
00405 }
00406 
00407 /**
00408  * Calculate device context offset
00409  *
00410  * @v xhci              xHCI device
00411  * @v ctx               Context index
00412  */
00413 static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
00414                                                   unsigned int ctx ) {
00415 
00416         return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
00417 }
00418 
00419 /**
00420  * Calculate input context offset
00421  *
00422  * @v xhci              xHCI device
00423  * @v ctx               Context index
00424  */
00425 static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
00426                                                  unsigned int ctx ) {
00427 
00428         return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
00429 }
00430 
00431 /******************************************************************************
00432  *
00433  * Diagnostics
00434  *
00435  ******************************************************************************
00436  */
00437 
00438 /**
00439  * Dump host controller registers
00440  *
00441  * @v xhci              xHCI device
00442  */
00443 static inline void xhci_dump ( struct xhci_device *xhci ) {
00444         uint32_t usbcmd;
00445         uint32_t usbsts;
00446         uint32_t pagesize;
00447         uint32_t dnctrl;
00448         uint32_t config;
00449 
00450         /* Do nothing unless debugging is enabled */
00451         if ( ! DBG_LOG )
00452                 return;
00453 
00454         /* Dump USBCMD */
00455         usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
00456         DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd,
00457                ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
00458                ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
00459 
00460         /* Dump USBSTS */
00461         usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
00462         DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts,
00463                ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
00464 
00465         /* Dump PAGESIZE */
00466         pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
00467         DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize );
00468 
00469         /* Dump DNCTRL */
00470         dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
00471         DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl );
00472 
00473         /* Dump CONFIG */
00474         config = readl ( xhci->op + XHCI_OP_CONFIG );
00475         DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config );
00476 }
00477 
00478 /**
00479  * Dump port registers
00480  *
00481  * @v xhci              xHCI device
00482  * @v port              Port number
00483  */
00484 static inline void xhci_dump_port ( struct xhci_device *xhci,
00485                                     unsigned int port ) {
00486         uint32_t portsc;
00487         uint32_t portpmsc;
00488         uint32_t portli;
00489         uint32_t porthlpmc;
00490 
00491         /* Do nothing unless debugging is enabled */
00492         if ( ! DBG_LOG )
00493                 return;
00494 
00495         /* Dump PORTSC */
00496         portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
00497         DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n",
00498                xhci->name, port, portsc,
00499                ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
00500                ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
00501                ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
00502                ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
00503                XHCI_PORTSC_PSIV ( portsc ) );
00504 
00505         /* Dump PORTPMSC */
00506         portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
00507         DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc );
00508 
00509         /* Dump PORTLI */
00510         portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
00511         DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli );
00512 
00513         /* Dump PORTHLPMC */
00514         porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
00515         DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n",
00516                xhci->name, port, porthlpmc );
00517 }
00518 
00519 /******************************************************************************
00520  *
00521  * USB legacy support
00522  *
00523  ******************************************************************************
00524  */
00525 
00526 /** Prevent the release of ownership back to BIOS */
00527 static int xhci_legacy_prevent_release;
00528 
00529 /**
00530  * Initialise USB legacy support
00531  *
00532  * @v xhci              xHCI device
00533  */
00534 static void xhci_legacy_init ( struct xhci_device *xhci ) {
00535         unsigned int legacy;
00536         uint8_t bios;
00537 
00538         /* Locate USB legacy support capability (if present) */
00539         legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
00540         if ( ! legacy ) {
00541                 /* Not an error; capability may not be present */
00542                 DBGC ( xhci, "XHCI %s has no USB legacy support capability\n",
00543                        xhci->name );
00544                 return;
00545         }
00546 
00547         /* Check if legacy USB support is enabled */
00548         bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
00549         if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
00550                 /* Not an error; already owned by OS */
00551                 DBGC ( xhci, "XHCI %s USB legacy support already disabled\n",
00552                        xhci->name );
00553                 return;
00554         }
00555 
00556         /* Record presence of USB legacy support capability */
00557         xhci->legacy = legacy;
00558 }
00559 
00560 /**
00561  * Claim ownership from BIOS
00562  *
00563  * @v xhci              xHCI device
00564  */
00565 static void xhci_legacy_claim ( struct xhci_device *xhci ) {
00566         uint32_t ctlsts;
00567         uint8_t bios;
00568         unsigned int i;
00569 
00570         /* Do nothing unless legacy support capability is present */
00571         if ( ! xhci->legacy )
00572                 return;
00573 
00574         /* Claim ownership */
00575         writeb ( XHCI_USBLEGSUP_OS_OWNED,
00576                  xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
00577 
00578         /* Wait for BIOS to release ownership */
00579         for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
00580 
00581                 /* Check if BIOS has released ownership */
00582                 bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
00583                 if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
00584                         DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n",
00585                                xhci->name );
00586                         ctlsts = readl ( xhci->cap + xhci->legacy +
00587                                          XHCI_USBLEGSUP_CTLSTS );
00588                         if ( ctlsts ) {
00589                                 DBGC ( xhci, "XHCI %s warning: BIOS retained "
00590                                        "SMIs: %08x\n", xhci->name, ctlsts );
00591                         }
00592                         return;
00593                 }
00594 
00595                 /* Delay */
00596                 mdelay ( 1 );
00597         }
00598 
00599         /* BIOS did not release ownership.  Claim it forcibly by
00600          * disabling all SMIs.
00601          */
00602         DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly "
00603                "disabling SMIs\n", xhci->name );
00604         writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS );
00605 }
00606 
00607 /**
00608  * Release ownership back to BIOS
00609  *
00610  * @v xhci              xHCI device
00611  */
00612 static void xhci_legacy_release ( struct xhci_device *xhci ) {
00613 
00614         /* Do nothing unless legacy support capability is present */
00615         if ( ! xhci->legacy )
00616                 return;
00617 
00618         /* Do nothing if releasing ownership is prevented */
00619         if ( xhci_legacy_prevent_release ) {
00620                 DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n",
00621                        xhci->name );
00622                 return;
00623         }
00624 
00625         /* Release ownership */
00626         writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
00627         DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name );
00628 }
00629 
00630 /******************************************************************************
00631  *
00632  * Supported protocols
00633  *
00634  ******************************************************************************
00635  */
00636 
00637 /**
00638  * Transcribe port speed (for debugging)
00639  *
00640  * @v psi               Protocol speed ID
00641  * @ret speed           Transcribed speed
00642  */
00643 static inline const char * xhci_speed_name ( uint32_t psi ) {
00644         static const char *exponents[4] = { "", "k", "M", "G" };
00645         static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
00646         unsigned int mantissa;
00647         unsigned int exponent;
00648 
00649         /* Extract mantissa and exponent */
00650         mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
00651         exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
00652 
00653         /* Transcribe speed */
00654         snprintf ( buf, sizeof ( buf ), "%d%sbps",
00655                    mantissa, exponents[exponent] );
00656         return buf;
00657 }
00658 
00659 /**
00660  * Find supported protocol extended capability for a port
00661  *
00662  * @v xhci              xHCI device
00663  * @v port              Port number
00664  * @ret supported       Offset to extended capability, or zero if not found
00665  */
00666 static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
00667                                               unsigned int port ) {
00668         unsigned int supported = 0;
00669         unsigned int offset;
00670         unsigned int count;
00671         uint32_t ports;
00672 
00673         /* Iterate over all supported protocol structures */
00674         while ( ( supported = xhci_extended_capability ( xhci,
00675                                                          XHCI_XECP_ID_SUPPORTED,
00676                                                          supported ) ) ) {
00677 
00678                 /* Determine port range */
00679                 ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
00680                 offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
00681                 count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
00682 
00683                 /* Check if port lies within this range */
00684                 if ( ( port - offset ) < count )
00685                         return supported;
00686         }
00687 
00688         DBGC ( xhci, "XHCI %s-%d has no supported protocol\n",
00689                xhci->name, port );
00690         return 0;
00691 }
00692 
00693 /**
00694  * Find port protocol
00695  *
00696  * @v xhci              xHCI device
00697  * @v port              Port number
00698  * @ret protocol        USB protocol, or zero if not found
00699  */
00700 static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
00701                                          unsigned int port ) {
00702         unsigned int supported = xhci_supported_protocol ( xhci, port );
00703         union {
00704                 uint32_t raw;
00705                 char text[5];
00706         } name;
00707         unsigned int protocol;
00708         unsigned int type;
00709         unsigned int psic;
00710         unsigned int psiv;
00711         unsigned int i;
00712         uint32_t revision;
00713         uint32_t ports;
00714         uint32_t slot;
00715         uint32_t psi;
00716 
00717         /* Fail if there is no supported protocol */
00718         if ( ! supported )
00719                 return 0;
00720 
00721         /* Determine protocol version */
00722         revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
00723         protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
00724 
00725         /* Describe port protocol */
00726         if ( DBG_EXTRA ) {
00727                 name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
00728                                                  XHCI_SUPPORTED_NAME ) );
00729                 name.text[4] = '\0';
00730                 slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
00731                 type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
00732                 DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d",
00733                         xhci->name, port, name.text, protocol, type );
00734                 ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
00735                 psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
00736                 if ( psic ) {
00737                         DBGC2 ( xhci, " speeds" );
00738                         for ( i = 0 ; i < psic ; i++ ) {
00739                                 psi = readl ( xhci->cap + supported +
00740                                               XHCI_SUPPORTED_PSI ( i ) );
00741                                 psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
00742                                 DBGC2 ( xhci, " %d:%s", psiv,
00743                                         xhci_speed_name ( psi ) );
00744                         }
00745                 }
00746                 if ( xhci->quirks & XHCI_BAD_PSIV )
00747                         DBGC2 ( xhci, " (ignored)" );
00748                 DBGC2 ( xhci, "\n" );
00749         }
00750 
00751         return protocol;
00752 }
00753 
00754 /**
00755  * Find port slot type
00756  *
00757  * @v xhci              xHCI device
00758  * @v port              Port number
00759  * @ret type            Slot type, or negative error
00760  */
00761 static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
00762         unsigned int supported = xhci_supported_protocol ( xhci, port );
00763         unsigned int type;
00764         uint32_t slot;
00765 
00766         /* Fail if there is no supported protocol */
00767         if ( ! supported )
00768                 return -ENOTSUP;
00769 
00770         /* Get slot type */
00771         slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
00772         type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
00773 
00774         return type;
00775 }
00776 
00777 /**
00778  * Find port speed
00779  *
00780  * @v xhci              xHCI device
00781  * @v port              Port number
00782  * @v psiv              Protocol speed ID value
00783  * @ret speed           Port speed, or negative error
00784  */
00785 static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
00786                              unsigned int psiv ) {
00787         unsigned int supported = xhci_supported_protocol ( xhci, port );
00788         unsigned int psic;
00789         unsigned int mantissa;
00790         unsigned int exponent;
00791         unsigned int speed;
00792         unsigned int i;
00793         uint32_t ports;
00794         uint32_t psi;
00795 
00796         /* Fail if there is no supported protocol */
00797         if ( ! supported )
00798                 return -ENOTSUP;
00799 
00800         /* Get protocol speed ID count */
00801         ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
00802         psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
00803 
00804         /* Use protocol speed ID table unless device is known to be faulty */
00805         if ( ! ( xhci->quirks & XHCI_BAD_PSIV ) ) {
00806 
00807                 /* Iterate over PSI dwords looking for a match */
00808                 for ( i = 0 ; i < psic ; i++ ) {
00809                         psi = readl ( xhci->cap + supported +
00810                                       XHCI_SUPPORTED_PSI ( i ) );
00811                         if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
00812                                 mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
00813                                 exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
00814                                 speed = USB_SPEED ( mantissa, exponent );
00815                                 return speed;
00816                         }
00817                 }
00818 
00819                 /* Record device as faulty if no match is found */
00820                 if ( psic != 0 ) {
00821                         DBGC ( xhci, "XHCI %s-%d spurious PSI value %d: "
00822                                "assuming PSI table is invalid\n",
00823                                xhci->name, port, psiv );
00824                         xhci->quirks |= XHCI_BAD_PSIV;
00825                 }
00826         }
00827 
00828         /* Use the default mappings */
00829         switch ( psiv ) {
00830         case XHCI_SPEED_LOW :   return USB_SPEED_LOW;
00831         case XHCI_SPEED_FULL :  return USB_SPEED_FULL;
00832         case XHCI_SPEED_HIGH :  return USB_SPEED_HIGH;
00833         case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
00834         default:
00835                 DBGC ( xhci, "XHCI %s-%d unrecognised PSI value %d\n",
00836                        xhci->name, port, psiv );
00837                 return -ENOTSUP;
00838         }
00839 }
00840 
00841 /**
00842  * Find protocol speed ID value
00843  *
00844  * @v xhci              xHCI device
00845  * @v port              Port number
00846  * @v speed             USB speed
00847  * @ret psiv            Protocol speed ID value, or negative error
00848  */
00849 static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
00850                             unsigned int speed ) {
00851         unsigned int supported = xhci_supported_protocol ( xhci, port );
00852         unsigned int psic;
00853         unsigned int mantissa;
00854         unsigned int exponent;
00855         unsigned int psiv;
00856         unsigned int i;
00857         uint32_t ports;
00858         uint32_t psi;
00859 
00860         /* Fail if there is no supported protocol */
00861         if ( ! supported )
00862                 return -ENOTSUP;
00863 
00864         /* Get protocol speed ID count */
00865         ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
00866         psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
00867 
00868         /* Use the default mappings if applicable */
00869         if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
00870                 switch ( speed ) {
00871                 case USB_SPEED_LOW :    return XHCI_SPEED_LOW;
00872                 case USB_SPEED_FULL :   return XHCI_SPEED_FULL;
00873                 case USB_SPEED_HIGH :   return XHCI_SPEED_HIGH;
00874                 case USB_SPEED_SUPER :  return XHCI_SPEED_SUPER;
00875                 default:
00876                         DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n",
00877                                xhci->name, port, speed );
00878                         return -ENOTSUP;
00879                 }
00880         }
00881 
00882         /* Iterate over PSI dwords looking for a match */
00883         for ( i = 0 ; i < psic ; i++ ) {
00884                 psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
00885                 mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
00886                 exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
00887                 if ( speed == USB_SPEED ( mantissa, exponent ) ) {
00888                         psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
00889                         return psiv;
00890                 }
00891         }
00892 
00893         DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n",
00894                xhci->name, port, speed );
00895         return -ENOENT;
00896 }
00897 
00898 /******************************************************************************
00899  *
00900  * Device context base address array
00901  *
00902  ******************************************************************************
00903  */
00904 
00905 /**
00906  * Allocate device context base address array
00907  *
00908  * @v xhci              xHCI device
00909  * @ret rc              Return status code
00910  */
00911 static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
00912         size_t len;
00913         physaddr_t dcbaap;
00914         int rc;
00915 
00916         /* Allocate and initialise structure.  Must be at least
00917          * 64-byte aligned and must not cross a page boundary, so
00918          * align on its own size (rounded up to a power of two and
00919          * with a minimum of 64 bytes).
00920          */
00921         len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
00922         xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
00923         if ( ! xhci->dcbaa ) {
00924                 DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
00925                 rc = -ENOMEM;
00926                 goto err_alloc;
00927         }
00928         memset ( xhci->dcbaa, 0, len );
00929 
00930         /* Program DCBAA pointer */
00931         dcbaap = virt_to_phys ( xhci->dcbaa );
00932         if ( ( rc = xhci_writeq ( xhci, dcbaap,
00933                                   xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
00934                 goto err_writeq;
00935 
00936         DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n",
00937                 xhci->name, dcbaap, ( dcbaap + len ) );
00938         return 0;
00939 
00940  err_writeq:
00941         free_dma ( xhci->dcbaa, len );
00942  err_alloc:
00943         return rc;
00944 }
00945 
00946 /**
00947  * Free device context base address array
00948  *
00949  * @v xhci              xHCI device
00950  */
00951 static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
00952         size_t len;
00953         unsigned int i;
00954 
00955         /* Sanity check */
00956         for ( i = 0 ; i <= xhci->slots ; i++ )
00957                 assert ( xhci->dcbaa[i] == 0 );
00958 
00959         /* Clear DCBAA pointer */
00960         xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
00961 
00962         /* Free DCBAA */
00963         len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
00964         free_dma ( xhci->dcbaa, len );
00965 }
00966 
00967 /******************************************************************************
00968  *
00969  * Scratchpad buffers
00970  *
00971  ******************************************************************************
00972  */
00973 
00974 /**
00975  * Allocate scratchpad buffers
00976  *
00977  * @v xhci              xHCI device
00978  * @ret rc              Return status code
00979  */
00980 static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
00981         size_t array_len;
00982         size_t len;
00983         physaddr_t phys;
00984         unsigned int i;
00985         int rc;
00986 
00987         /* Do nothing if no scratchpad buffers are used */
00988         if ( ! xhci->scratchpads )
00989                 return 0;
00990 
00991         /* Allocate scratchpads */
00992         len = ( xhci->scratchpads * xhci->pagesize );
00993         xhci->scratchpad = umalloc ( len );
00994         if ( ! xhci->scratchpad ) {
00995                 DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
00996                        xhci->name );
00997                 rc = -ENOMEM;
00998                 goto err_alloc;
00999         }
01000         memset_user ( xhci->scratchpad, 0, 0, len );
01001 
01002         /* Allocate scratchpad array */
01003         array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
01004         xhci->scratchpad_array =
01005                 malloc_dma ( array_len, xhci_align ( array_len ) );
01006         if ( ! xhci->scratchpad_array ) {
01007                 DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
01008                        "array\n", xhci->name );
01009                 rc = -ENOMEM;
01010                 goto err_alloc_array;
01011         }
01012 
01013         /* Populate scratchpad array */
01014         for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
01015                 phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
01016                 xhci->scratchpad_array[i] = phys;
01017         }
01018 
01019         /* Set scratchpad array pointer */
01020         assert ( xhci->dcbaa != NULL );
01021         xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
01022 
01023         DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
01024                 xhci->name, user_to_phys ( xhci->scratchpad, 0 ),
01025                 user_to_phys ( xhci->scratchpad, len ),
01026                 virt_to_phys ( xhci->scratchpad_array ),
01027                 ( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
01028         return 0;
01029 
01030         free_dma ( xhci->scratchpad_array, array_len );
01031  err_alloc_array:
01032         ufree ( xhci->scratchpad );
01033  err_alloc:
01034         return rc;
01035 }
01036 
01037 /**
01038  * Free scratchpad buffers
01039  *
01040  * @v xhci              xHCI device
01041  */
01042 static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
01043         size_t array_len;
01044 
01045         /* Do nothing if no scratchpad buffers are used */
01046         if ( ! xhci->scratchpads )
01047                 return;
01048 
01049         /* Clear scratchpad array pointer */
01050         assert ( xhci->dcbaa != NULL );
01051         xhci->dcbaa[0] = 0;
01052 
01053         /* Free scratchpad array */
01054         array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
01055         free_dma ( xhci->scratchpad_array, array_len );
01056 
01057         /* Free scratchpads */
01058         ufree ( xhci->scratchpad );
01059 }
01060 
01061 /******************************************************************************
01062  *
01063  * Run / stop / reset
01064  *
01065  ******************************************************************************
01066  */
01067 
01068 /**
01069  * Start xHCI device
01070  *
01071  * @v xhci              xHCI device
01072  */
01073 static void xhci_run ( struct xhci_device *xhci ) {
01074         uint32_t config;
01075         uint32_t usbcmd;
01076 
01077         /* Configure number of device slots */
01078         config = readl ( xhci->op + XHCI_OP_CONFIG );
01079         config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
01080         config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
01081         writel ( config, xhci->op + XHCI_OP_CONFIG );
01082 
01083         /* Set run/stop bit */
01084         usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
01085         usbcmd |= XHCI_USBCMD_RUN;
01086         writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
01087 }
01088 
01089 /**
01090  * Stop xHCI device
01091  *
01092  * @v xhci              xHCI device
01093  * @ret rc              Return status code
01094  */
01095 static int xhci_stop ( struct xhci_device *xhci ) {
01096         uint32_t usbcmd;
01097         uint32_t usbsts;
01098         unsigned int i;
01099 
01100         /* Clear run/stop bit */
01101         usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
01102         usbcmd &= ~XHCI_USBCMD_RUN;
01103         writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
01104 
01105         /* Wait for device to stop */
01106         for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
01107 
01108                 /* Check if device is stopped */
01109                 usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
01110                 if ( usbsts & XHCI_USBSTS_HCH )
01111                         return 0;
01112 
01113                 /* Delay */
01114                 mdelay ( 1 );
01115         }
01116 
01117         DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name );
01118         return -ETIMEDOUT;
01119 }
01120 
01121 /**
01122  * Reset xHCI device
01123  *
01124  * @v xhci              xHCI device
01125  * @ret rc              Return status code
01126  */
01127 static int xhci_reset ( struct xhci_device *xhci ) {
01128         uint32_t usbcmd;
01129         unsigned int i;
01130         int rc;
01131 
01132         /* The xHCI specification states that resetting a running
01133          * device may result in undefined behaviour, so try stopping
01134          * it first.
01135          */
01136         if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
01137                 /* Ignore errors and attempt to reset the device anyway */
01138         }
01139 
01140         /* Reset device */
01141         writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
01142 
01143         /* Wait for reset to complete */
01144         for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
01145 
01146                 /* Check if reset is complete */
01147                 usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
01148                 if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
01149                         return 0;
01150 
01151                 /* Delay */
01152                 mdelay ( 1 );
01153         }
01154 
01155         DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name );
01156         return -ETIMEDOUT;
01157 }
01158 
01159 /******************************************************************************
01160  *
01161  * Transfer request blocks
01162  *
01163  ******************************************************************************
01164  */
01165 
01166 /**
01167  * Allocate transfer request block ring
01168  *
01169  * @v xhci              xHCI device
01170  * @v ring              TRB ring
01171  * @v shift             Ring size (log2)
01172  * @v slot              Device slot
01173  * @v target            Doorbell target
01174  * @v stream            Doorbell stream ID
01175  * @ret rc              Return status code
01176  */
01177 static int xhci_ring_alloc ( struct xhci_device *xhci,
01178                              struct xhci_trb_ring *ring,
01179                              unsigned int shift, unsigned int slot,
01180                              unsigned int target, unsigned int stream ) {
01181         struct xhci_trb_link *link;
01182         unsigned int count;
01183         int rc;
01184 
01185         /* Sanity check */
01186         assert ( shift > 0 );
01187 
01188         /* Initialise structure */
01189         memset ( ring, 0, sizeof ( *ring ) );
01190         ring->shift = shift;
01191         count = ( 1U << shift );
01192         ring->mask = ( count - 1 );
01193         ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
01194         ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
01195         ring->dbval = XHCI_DBVAL ( target, stream );
01196 
01197         /* Allocate I/O buffers */
01198         ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
01199         if ( ! ring->iobuf ) {
01200                 rc = -ENOMEM;
01201                 goto err_alloc_iobuf;
01202         }
01203 
01204         /* Allocate TRBs */
01205         ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
01206         if ( ! ring->trb ) {
01207                 rc = -ENOMEM;
01208                 goto err_alloc_trb;
01209         }
01210         memset ( ring->trb, 0, ring->len );
01211 
01212         /* Initialise Link TRB */
01213         link = &ring->trb[count].link;
01214         link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
01215         link->flags = XHCI_TRB_TC;
01216         link->type = XHCI_TRB_LINK;
01217         ring->link = link;
01218 
01219         return 0;
01220 
01221         free_dma ( ring->trb, ring->len );
01222  err_alloc_trb:
01223         free ( ring->iobuf );
01224  err_alloc_iobuf:
01225         return rc;
01226 }
01227 
01228 /**
01229  * Reset transfer request block ring
01230  *
01231  * @v ring              TRB ring
01232  */
01233 static void xhci_ring_reset ( struct xhci_trb_ring *ring ) {
01234         unsigned int count = ( 1U << ring->shift );
01235 
01236         /* Reset producer and consumer counters */
01237         ring->prod = 0;
01238         ring->cons = 0;
01239 
01240         /* Reset TRBs (except Link TRB) */
01241         memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) );
01242 }
01243 
01244 /**
01245  * Free transfer request block ring
01246  *
01247  * @v ring              TRB ring
01248  */
01249 static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
01250         unsigned int count = ( 1U << ring->shift );
01251         unsigned int i;
01252 
01253         /* Sanity checks */
01254         assert ( ring->cons == ring->prod );
01255         for ( i = 0 ; i < count ; i++ )
01256                 assert ( ring->iobuf[i] == NULL );
01257 
01258         /* Free TRBs */
01259         free_dma ( ring->trb, ring->len );
01260 
01261         /* Free I/O buffers */
01262         free ( ring->iobuf );
01263 }
01264 
01265 /**
01266  * Enqueue a transfer request block
01267  *
01268  * @v ring              TRB ring
01269  * @v iobuf             I/O buffer (if any)
01270  * @v trb               Transfer request block (with empty Cycle flag)
01271  * @ret rc              Return status code
01272  *
01273  * This operation does not implicitly ring the doorbell register.
01274  */
01275 static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
01276                           const union xhci_trb *trb ) {
01277         union xhci_trb *dest;
01278         unsigned int prod;
01279         unsigned int mask;
01280         unsigned int index;
01281         unsigned int cycle;
01282 
01283         /* Sanity check */
01284         assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
01285 
01286         /* Fail if ring is full */
01287         if ( ! xhci_ring_remaining ( ring ) )
01288                 return -ENOBUFS;
01289 
01290         /* Update producer counter (and link TRB, if applicable) */
01291         prod = ring->prod++;
01292         mask = ring->mask;
01293         cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
01294         index = ( prod & mask );
01295         if ( index == 0 )
01296                 ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
01297 
01298         /* Record I/O buffer */
01299         ring->iobuf[index] = iobuf;
01300 
01301         /* Enqueue TRB */
01302         dest = &ring->trb[index];
01303         dest->template.parameter = trb->template.parameter;
01304         dest->template.status = trb->template.status;
01305         wmb();
01306         dest->template.control = ( trb->template.control |
01307                                    cpu_to_le32 ( cycle ) );
01308 
01309         return 0;
01310 }
01311 
01312 /**
01313  * Dequeue a transfer request block
01314  *
01315  * @v ring              TRB ring
01316  * @ret iobuf           I/O buffer
01317  */
01318 static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
01319         struct io_buffer *iobuf;
01320         unsigned int cons;
01321         unsigned int mask;
01322         unsigned int index;
01323 
01324         /* Sanity check */
01325         assert ( xhci_ring_fill ( ring ) != 0 );
01326 
01327         /* Update consumer counter */
01328         cons = ring->cons++;
01329         mask = ring->mask;
01330         index = ( cons & mask );
01331 
01332         /* Retrieve I/O buffer */
01333         iobuf = ring->iobuf[index];
01334         ring->iobuf[index] = NULL;
01335 
01336         return iobuf;
01337 }
01338 
01339 /**
01340  * Enqueue multiple transfer request blocks
01341  *
01342  * @v ring              TRB ring
01343  * @v iobuf             I/O buffer
01344  * @v trbs              Transfer request blocks (with empty Cycle flag)
01345  * @v count             Number of transfer request blocks
01346  * @ret rc              Return status code
01347  *
01348  * This operation does not implicitly ring the doorbell register.
01349  */
01350 static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
01351                                 struct io_buffer *iobuf,
01352                                 const union xhci_trb *trbs,
01353                                 unsigned int count ) {
01354         const union xhci_trb *trb = trbs;
01355         int rc;
01356 
01357         /* Sanity check */
01358         assert ( iobuf != NULL );
01359 
01360         /* Fail if ring does not have sufficient space */
01361         if ( xhci_ring_remaining ( ring ) < count )
01362                 return -ENOBUFS;
01363 
01364         /* Enqueue each TRB, recording the I/O buffer with the final TRB */
01365         while ( count-- ) {
01366                 rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
01367                 assert ( rc == 0 ); /* Should never be able to fail */
01368         }
01369 
01370         return 0;
01371 }
01372 
01373 /**
01374  * Dequeue multiple transfer request blocks
01375  *
01376  * @v ring              TRB ring
01377  * @ret iobuf           I/O buffer
01378  */
01379 static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
01380         struct io_buffer *iobuf;
01381 
01382         /* Dequeue TRBs until we reach the final TRB for an I/O buffer */
01383         do {
01384                 iobuf = xhci_dequeue ( ring );
01385         } while ( iobuf == NULL );
01386 
01387         return iobuf;
01388 }
01389 
01390 /**
01391  * Ring doorbell register
01392  *
01393  * @v ring              TRB ring
01394  */
01395 static inline __attribute__ (( always_inline )) void
01396 xhci_doorbell ( struct xhci_trb_ring *ring ) {
01397 
01398         wmb();
01399         writel ( ring->dbval, ring->db );
01400 }
01401 
01402 /******************************************************************************
01403  *
01404  * Command and event rings
01405  *
01406  ******************************************************************************
01407  */
01408 
01409 /**
01410  * Allocate command ring
01411  *
01412  * @v xhci              xHCI device
01413  * @ret rc              Return status code
01414  */
01415 static int xhci_command_alloc ( struct xhci_device *xhci ) {
01416         physaddr_t crp;
01417         int rc;
01418 
01419         /* Allocate TRB ring */
01420         if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
01421                                       0, 0, 0 ) ) != 0 )
01422                 goto err_ring_alloc;
01423 
01424         /* Program command ring control register */
01425         crp = virt_to_phys ( xhci->command.trb );
01426         if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
01427                                   xhci->op + XHCI_OP_CRCR ) ) != 0 )
01428                 goto err_writeq;
01429 
01430         DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n",
01431                 xhci->name, crp, ( crp + xhci->command.len ) );
01432         return 0;
01433 
01434  err_writeq:
01435         xhci_ring_free ( &xhci->command );
01436  err_ring_alloc:
01437         return rc;
01438 }
01439 
01440 /**
01441  * Free command ring
01442  *
01443  * @v xhci              xHCI device
01444  */
01445 static void xhci_command_free ( struct xhci_device *xhci ) {
01446 
01447         /* Sanity check */
01448         assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
01449 
01450         /* Clear command ring control register */
01451         xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
01452 
01453         /* Free TRB ring */
01454         xhci_ring_free ( &xhci->command );
01455 }
01456 
01457 /**
01458  * Allocate event ring
01459  *
01460  * @v xhci              xHCI device
01461  * @ret rc              Return status code
01462  */
01463 static int xhci_event_alloc ( struct xhci_device *xhci ) {
01464         struct xhci_event_ring *event = &xhci->event;
01465         unsigned int count;
01466         size_t len;
01467         int rc;
01468 
01469         /* Allocate event ring */
01470         count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
01471         len = ( count * sizeof ( event->trb[0] ) );
01472         event->trb = malloc_dma ( len, xhci_align ( len ) );
01473         if ( ! event->trb ) {
01474                 rc = -ENOMEM;
01475                 goto err_alloc_trb;
01476         }
01477         memset ( event->trb, 0, len );
01478 
01479         /* Allocate event ring segment table */
01480         event->segment = malloc_dma ( sizeof ( event->segment[0] ),
01481                                       xhci_align ( sizeof (event->segment[0])));
01482         if ( ! event->segment ) {
01483                 rc = -ENOMEM;
01484                 goto err_alloc_segment;
01485         }
01486         memset ( event->segment, 0, sizeof ( event->segment[0] ) );
01487         event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
01488         event->segment[0].count = cpu_to_le32 ( count );
01489 
01490         /* Program event ring registers */
01491         writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
01492         if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
01493                                   xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
01494                 goto err_writeq_erdp;
01495         if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
01496                                   xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
01497                 goto err_writeq_erstba;
01498 
01499         DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
01500                 xhci->name, virt_to_phys ( event->trb ),
01501                 ( virt_to_phys ( event->trb ) + len ),
01502                 virt_to_phys ( event->segment ),
01503                 ( virt_to_phys ( event->segment ) +
01504                   sizeof (event->segment[0] ) ) );
01505         return 0;
01506 
01507         xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
01508  err_writeq_erstba:
01509         xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
01510  err_writeq_erdp:
01511         free_dma ( event->trb, len );
01512  err_alloc_segment:
01513         free_dma ( event->segment, sizeof ( event->segment[0] ) );
01514  err_alloc_trb:
01515         return rc;
01516 }
01517 
01518 /**
01519  * Free event ring
01520  *
01521  * @v xhci              xHCI device
01522  */
01523 static void xhci_event_free ( struct xhci_device *xhci ) {
01524         struct xhci_event_ring *event = &xhci->event;
01525         unsigned int count;
01526         size_t len;
01527 
01528         /* Clear event ring registers */
01529         writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
01530         xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
01531         xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
01532 
01533         /* Free event ring segment table */
01534         free_dma ( event->segment, sizeof ( event->segment[0] ) );
01535 
01536         /* Free event ring */
01537         count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
01538         len = ( count * sizeof ( event->trb[0] ) );
01539         free_dma ( event->trb, len );
01540 }
01541 
01542 /**
01543  * Handle transfer event
01544  *
01545  * @v xhci              xHCI device
01546  * @v trb               Transfer event TRB
01547  */
01548 static void xhci_transfer ( struct xhci_device *xhci,
01549                             struct xhci_trb_transfer *trb ) {
01550         struct xhci_slot *slot;
01551         struct xhci_endpoint *endpoint;
01552         struct io_buffer *iobuf;
01553         int rc;
01554 
01555         /* Profile transfer events */
01556         profile_start ( &xhci_transfer_profiler );
01557 
01558         /* Identify slot */
01559         if ( ( trb->slot > xhci->slots ) ||
01560              ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) {
01561                 DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n",
01562                        xhci->name, trb->slot );
01563                 DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
01564                 return;
01565         }
01566 
01567         /* Identify endpoint */
01568         if ( ( trb->endpoint >= XHCI_CTX_END ) ||
01569              ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) {
01570                 DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid "
01571                        "%d:\n", xhci->name, slot->id, trb->endpoint );
01572                 DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
01573                 return;
01574         }
01575 
01576         /* Dequeue TRB(s) */
01577         iobuf = xhci_dequeue_multi ( &endpoint->ring );
01578         assert ( iobuf != NULL );
01579 
01580         /* Check for errors */
01581         if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
01582                  ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
01583 
01584                 /* Construct error */
01585                 rc = -ECODE ( trb->code );
01586                 DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n",
01587                        xhci->name, slot->id, endpoint->ctx, trb->code,
01588                        strerror ( rc ) );
01589                 DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
01590 
01591                 /* Sanity check */
01592                 assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
01593                          != XHCI_ENDPOINT_RUNNING );
01594 
01595                 /* Report failure to USB core */
01596                 usb_complete_err ( endpoint->ep, iobuf, rc );
01597                 return;
01598         }
01599 
01600         /* Record actual transfer size */
01601         iob_unput ( iobuf, le16_to_cpu ( trb->residual ) );
01602 
01603         /* Sanity check (for successful completions only) */
01604         assert ( xhci_ring_consumed ( &endpoint->ring ) ==
01605                  le64_to_cpu ( trb->transfer ) );
01606 
01607         /* Report completion to USB core */
01608         usb_complete ( endpoint->ep, iobuf );
01609         profile_stop ( &xhci_transfer_profiler );
01610 }
01611 
01612 /**
01613  * Handle command completion event
01614  *
01615  * @v xhci              xHCI device
01616  * @v trb               Command completion event
01617  */
01618 static void xhci_complete ( struct xhci_device *xhci,
01619                             struct xhci_trb_complete *trb ) {
01620         int rc;
01621 
01622         /* Ignore "command ring stopped" notifications */
01623         if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) {
01624                 DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name );
01625                 return;
01626         }
01627 
01628         /* Ignore unexpected completions */
01629         if ( ! xhci->pending ) {
01630                 rc = -ECODE ( trb->code );
01631                 DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n",
01632                        xhci->name, trb->code, strerror ( rc ) );
01633                 DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
01634                 return;
01635         }
01636 
01637         /* Dequeue command TRB */
01638         xhci_dequeue ( &xhci->command );
01639 
01640         /* Sanity check */
01641         assert ( xhci_ring_consumed ( &xhci->command ) ==
01642                  le64_to_cpu ( trb->command ) );
01643 
01644         /* Record completion */
01645         memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) );
01646         xhci->pending = NULL;
01647 }
01648 
01649 /**
01650  * Handle port status event
01651  *
01652  * @v xhci              xHCI device
01653  * @v trb               Port status event
01654  */
01655 static void xhci_port_status ( struct xhci_device *xhci,
01656                                struct xhci_trb_port_status *trb ) {
01657         struct usb_port *port = usb_port ( xhci->bus->hub, trb->port );
01658         uint32_t portsc;
01659 
01660         /* Sanity check */
01661         assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) );
01662 
01663         /* Record disconnections and clear changes */
01664         portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) );
01665         port->disconnected |= ( portsc & XHCI_PORTSC_CSC );
01666         portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
01667         writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) );
01668 
01669         /* Report port status change */
01670         usb_port_changed ( port );
01671 }
01672 
01673 /**
01674  * Handle host controller event
01675  *
01676  * @v xhci              xHCI device
01677  * @v trb               Host controller event
01678  */
01679 static void xhci_host_controller ( struct xhci_device *xhci,
01680                                    struct xhci_trb_host_controller *trb ) {
01681         int rc;
01682 
01683         /* Construct error */
01684         rc = -ECODE ( trb->code );
01685         DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n",
01686                xhci->name, trb->code, strerror ( rc ) );
01687 }
01688 
01689 /**
01690  * Poll event ring
01691  *
01692  * @v xhci              xHCI device
01693  */
01694 static void xhci_event_poll ( struct xhci_device *xhci ) {
01695         struct xhci_event_ring *event = &xhci->event;
01696         union xhci_trb *trb;
01697         unsigned int shift = XHCI_EVENT_TRBS_LOG2;
01698         unsigned int count = ( 1 << shift );
01699         unsigned int mask = ( count - 1 );
01700         unsigned int consumed;
01701         unsigned int type;
01702 
01703         /* Poll for events */
01704         profile_start ( &xhci_event_profiler );
01705         for ( consumed = 0 ; ; consumed++ ) {
01706 
01707                 /* Stop if we reach an empty TRB */
01708                 rmb();
01709                 trb = &event->trb[ event->cons & mask ];
01710                 if ( ! ( ( trb->common.flags ^
01711                            ( event->cons >> shift ) ) & XHCI_TRB_C ) )
01712                         break;
01713 
01714                 /* Consume this TRB */
01715                 event->cons++;
01716 
01717                 /* Handle TRB */
01718                 type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
01719                 switch ( type ) {
01720 
01721                 case XHCI_TRB_TRANSFER :
01722                         xhci_transfer ( xhci, &trb->transfer );
01723                         break;
01724 
01725                 case XHCI_TRB_COMPLETE :
01726                         xhci_complete ( xhci, &trb->complete );
01727                         break;
01728 
01729                 case XHCI_TRB_PORT_STATUS:
01730                         xhci_port_status ( xhci, &trb->port );
01731                         break;
01732 
01733                 case XHCI_TRB_HOST_CONTROLLER:
01734                         xhci_host_controller ( xhci, &trb->host );
01735                         break;
01736 
01737                 default:
01738                         DBGC ( xhci, "XHCI %s unrecognised event %#x\n:",
01739                                xhci->name, ( event->cons - 1 ) );
01740                         DBGC_HDA ( xhci, virt_to_phys ( trb ),
01741                                    trb, sizeof ( *trb ) );
01742                         break;
01743                 }
01744         }
01745 
01746         /* Update dequeue pointer if applicable */
01747         if ( consumed ) {
01748                 xhci_writeq ( xhci, virt_to_phys ( trb ),
01749                               xhci->run + XHCI_RUN_ERDP ( 0 ) );
01750                 profile_stop ( &xhci_event_profiler );
01751         }
01752 }
01753 
01754 /**
01755  * Abort command
01756  *
01757  * @v xhci              xHCI device
01758  */
01759 static void xhci_abort ( struct xhci_device *xhci ) {
01760         physaddr_t crp;
01761 
01762         /* Abort the command */
01763         DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
01764         xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR );
01765 
01766         /* Allow time for command to abort */
01767         mdelay ( XHCI_COMMAND_ABORT_DELAY_MS );
01768 
01769         /* Sanity check */
01770         assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
01771 
01772         /* Consume (and ignore) any final command status */
01773         xhci_event_poll ( xhci );
01774 
01775         /* Reset the command ring control register */
01776         xhci_ring_reset ( &xhci->command );
01777         crp = virt_to_phys ( xhci->command.trb );
01778         xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
01779 }
01780 
01781 /**
01782  * Issue command and wait for completion
01783  *
01784  * @v xhci              xHCI device
01785  * @v trb               Transfer request block (with empty Cycle flag)
01786  * @ret rc              Return status code
01787  *
01788  * On a successful completion, the TRB will be overwritten with the
01789  * completion.
01790  */
01791 static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
01792         struct xhci_trb_complete *complete = &trb->complete;
01793         unsigned int i;
01794         int rc;
01795 
01796         /* Record the pending command */
01797         xhci->pending = trb;
01798 
01799         /* Enqueue the command */
01800         if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
01801                 goto err_enqueue;
01802 
01803         /* Ring the command doorbell */
01804         xhci_doorbell ( &xhci->command );
01805 
01806         /* Wait for the command to complete */
01807         for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
01808 
01809                 /* Poll event ring */
01810                 xhci_event_poll ( xhci );
01811 
01812                 /* Check for completion */
01813                 if ( ! xhci->pending ) {
01814                         if ( complete->code != XHCI_CMPLT_SUCCESS ) {
01815                                 rc = -ECODE ( complete->code );
01816                                 DBGC ( xhci, "XHCI %s command failed (code "
01817                                        "%d): %s\n", xhci->name, complete->code,
01818                                        strerror ( rc ) );
01819                                 DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
01820                                 return rc;
01821                         }
01822                         return 0;
01823                 }
01824 
01825                 /* Delay */
01826                 mdelay ( 1 );
01827         }
01828 
01829         /* Timeout */
01830         DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name );
01831         rc = -ETIMEDOUT;
01832 
01833         /* Abort command */
01834         xhci_abort ( xhci );
01835 
01836  err_enqueue:
01837         xhci->pending = NULL;
01838         return rc;
01839 }
01840 
01841 /**
01842  * Issue NOP and wait for completion
01843  *
01844  * @v xhci              xHCI device
01845  * @ret rc              Return status code
01846  */
01847 static inline int xhci_nop ( struct xhci_device *xhci ) {
01848         union xhci_trb trb;
01849         struct xhci_trb_common *nop = &trb.common;
01850         int rc;
01851 
01852         /* Construct command */
01853         memset ( nop, 0, sizeof ( *nop ) );
01854         nop->flags = XHCI_TRB_IOC;
01855         nop->type = XHCI_TRB_NOP_CMD;
01856 
01857         /* Issue command and wait for completion */
01858         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
01859                 return rc;
01860 
01861         return 0;
01862 }
01863 
01864 /**
01865  * Enable slot
01866  *
01867  * @v xhci              xHCI device
01868  * @v type              Slot type
01869  * @ret slot            Device slot ID, or negative error
01870  */
01871 static inline int xhci_enable_slot ( struct xhci_device *xhci,
01872                                      unsigned int type ) {
01873         union xhci_trb trb;
01874         struct xhci_trb_enable_slot *enable = &trb.enable;
01875         struct xhci_trb_complete *enabled = &trb.complete;
01876         unsigned int slot;
01877         int rc;
01878 
01879         /* Construct command */
01880         memset ( enable, 0, sizeof ( *enable ) );
01881         enable->slot = type;
01882         enable->type = XHCI_TRB_ENABLE_SLOT;
01883 
01884         /* Issue command and wait for completion */
01885         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
01886                 DBGC ( xhci, "XHCI %s could not enable new slot: %s\n",
01887                        xhci->name, strerror ( rc ) );
01888                 return rc;
01889         }
01890 
01891         /* Extract slot number */
01892         slot = enabled->slot;
01893 
01894         DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot );
01895         return slot;
01896 }
01897 
01898 /**
01899  * Disable slot
01900  *
01901  * @v xhci              xHCI device
01902  * @v slot              Device slot
01903  * @ret rc              Return status code
01904  */
01905 static inline int xhci_disable_slot ( struct xhci_device *xhci,
01906                                       unsigned int slot ) {
01907         union xhci_trb trb;
01908         struct xhci_trb_disable_slot *disable = &trb.disable;
01909         int rc;
01910 
01911         /* Construct command */
01912         memset ( disable, 0, sizeof ( *disable ) );
01913         disable->type = XHCI_TRB_DISABLE_SLOT;
01914         disable->slot = slot;
01915 
01916         /* Issue command and wait for completion */
01917         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
01918                 DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n",
01919                        xhci->name, slot, strerror ( rc ) );
01920                 return rc;
01921         }
01922 
01923         DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot );
01924         return 0;
01925 }
01926 
01927 /**
01928  * Issue context-based command and wait for completion
01929  *
01930  * @v xhci              xHCI device
01931  * @v slot              Device slot
01932  * @v endpoint          Endpoint
01933  * @v type              TRB type
01934  * @v populate          Input context populater
01935  * @ret rc              Return status code
01936  */
01937 static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
01938                           struct xhci_endpoint *endpoint, unsigned int type,
01939                           void ( * populate ) ( struct xhci_device *xhci,
01940                                                 struct xhci_slot *slot,
01941                                                 struct xhci_endpoint *endpoint,
01942                                                 void *input ) ) {
01943         union xhci_trb trb;
01944         struct xhci_trb_context *context = &trb.context;
01945         size_t len;
01946         void *input;
01947         int rc;
01948 
01949         /* Allocate an input context */
01950         len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
01951         input = malloc_dma ( len, xhci_align ( len ) );
01952         if ( ! input ) {
01953                 rc = -ENOMEM;
01954                 goto err_alloc;
01955         }
01956         memset ( input, 0, len );
01957 
01958         /* Populate input context */
01959         populate ( xhci, slot, endpoint, input );
01960 
01961         /* Construct command */
01962         memset ( context, 0, sizeof ( *context ) );
01963         context->type = type;
01964         context->input = cpu_to_le64 ( virt_to_phys ( input ) );
01965         context->slot = slot->id;
01966 
01967         /* Issue command and wait for completion */
01968         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
01969                 goto err_command;
01970 
01971  err_command:
01972         free_dma ( input, len );
01973  err_alloc:
01974         return rc;
01975 }
01976 
01977 /**
01978  * Populate address device input context
01979  *
01980  * @v xhci              xHCI device
01981  * @v slot              Device slot
01982  * @v endpoint          Endpoint
01983  * @v input             Input context
01984  */
01985 static void xhci_address_device_input ( struct xhci_device *xhci,
01986                                         struct xhci_slot *slot,
01987                                         struct xhci_endpoint *endpoint,
01988                                         void *input ) {
01989         struct xhci_control_context *control_ctx;
01990         struct xhci_slot_context *slot_ctx;
01991         struct xhci_endpoint_context *ep_ctx;
01992 
01993         /* Sanity checks */
01994         assert ( endpoint->ctx == XHCI_CTX_EP0 );
01995 
01996         /* Populate control context */
01997         control_ctx = input;
01998         control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
01999                                          ( 1 << XHCI_CTX_EP0 ) );
02000 
02001         /* Populate slot context */
02002         slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
02003         slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
02004                                                         slot->route ) );
02005         slot_ctx->port = slot->port;
02006         slot_ctx->tt_id = slot->tt_id;
02007         slot_ctx->tt_port = slot->tt_port;
02008 
02009         /* Populate control endpoint context */
02010         ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
02011         ep_ctx->type = XHCI_EP_TYPE_CONTROL;
02012         ep_ctx->burst = endpoint->ep->burst;
02013         ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
02014         ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
02015                                         XHCI_EP_DCS );
02016         ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
02017 }
02018 
02019 /**
02020  * Address device
02021  *
02022  * @v xhci              xHCI device
02023  * @v slot              Device slot
02024  * @ret rc              Return status code
02025  */
02026 static inline int xhci_address_device ( struct xhci_device *xhci,
02027                                         struct xhci_slot *slot ) {
02028         struct usb_device *usb = slot->usb;
02029         struct xhci_slot_context *slot_ctx;
02030         int rc;
02031 
02032         /* Assign device address */
02033         if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
02034                                    XHCI_TRB_ADDRESS_DEVICE,
02035                                    xhci_address_device_input ) ) != 0 )
02036                 return rc;
02037 
02038         /* Get assigned address */
02039         slot_ctx = ( slot->context +
02040                      xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
02041         usb->address = slot_ctx->address;
02042         DBGC2 ( xhci, "XHCI %s assigned address %d to %s\n",
02043                 xhci->name, usb->address, usb->name );
02044 
02045         return 0;
02046 }
02047 
02048 /**
02049  * Populate configure endpoint input context
02050  *
02051  * @v xhci              xHCI device
02052  * @v slot              Device slot
02053  * @v endpoint          Endpoint
02054  * @v input             Input context
02055  */
02056 static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
02057                                             struct xhci_slot *slot,
02058                                             struct xhci_endpoint *endpoint,
02059                                             void *input ) {
02060         struct xhci_control_context *control_ctx;
02061         struct xhci_slot_context *slot_ctx;
02062         struct xhci_endpoint_context *ep_ctx;
02063 
02064         /* Populate control context */
02065         control_ctx = input;
02066         control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
02067                                          ( 1 << endpoint->ctx ) );
02068 
02069         /* Populate slot context */
02070         slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
02071         slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
02072                                                         ( slot->ports ? 1 : 0 ),
02073                                                         slot->psiv, 0 ) );
02074         slot_ctx->ports = slot->ports;
02075 
02076         /* Populate endpoint context */
02077         ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
02078         ep_ctx->interval = endpoint->interval;
02079         ep_ctx->type = endpoint->type;
02080         ep_ctx->burst = endpoint->ep->burst;
02081         ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
02082         ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
02083                                         XHCI_EP_DCS );
02084         ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
02085 }
02086 
02087 /**
02088  * Configure endpoint
02089  *
02090  * @v xhci              xHCI device
02091  * @v slot              Device slot
02092  * @v endpoint          Endpoint
02093  * @ret rc              Return status code
02094  */
02095 static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
02096                                             struct xhci_slot *slot,
02097                                             struct xhci_endpoint *endpoint ) {
02098         int rc;
02099 
02100         /* Configure endpoint */
02101         if ( ( rc = xhci_context ( xhci, slot, endpoint,
02102                                    XHCI_TRB_CONFIGURE_ENDPOINT,
02103                                    xhci_configure_endpoint_input ) ) != 0 )
02104                 return rc;
02105 
02106         DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
02107                 xhci->name, slot->id, endpoint->ctx );
02108         return 0;
02109 }
02110 
02111 /**
02112  * Populate deconfigure endpoint input context
02113  *
02114  * @v xhci              xHCI device
02115  * @v slot              Device slot
02116  * @v endpoint          Endpoint
02117  * @v input             Input context
02118  */
02119 static void
02120 xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
02121                                   struct xhci_slot *slot __unused,
02122                                   struct xhci_endpoint *endpoint,
02123                                   void *input ) {
02124         struct xhci_control_context *control_ctx;
02125         struct xhci_slot_context *slot_ctx;
02126 
02127         /* Populate control context */
02128         control_ctx = input;
02129         control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
02130         control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
02131 
02132         /* Populate slot context */
02133         slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
02134         slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
02135                                                         0, 0, 0 ) );
02136 }
02137 
02138 /**
02139  * Deconfigure endpoint
02140  *
02141  * @v xhci              xHCI device
02142  * @v slot              Device slot
02143  * @v endpoint          Endpoint
02144  * @ret rc              Return status code
02145  */
02146 static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
02147                                               struct xhci_slot *slot,
02148                                               struct xhci_endpoint *endpoint ) {
02149         int rc;
02150 
02151         /* Deconfigure endpoint */
02152         if ( ( rc = xhci_context ( xhci, slot, endpoint,
02153                                    XHCI_TRB_CONFIGURE_ENDPOINT,
02154                                    xhci_deconfigure_endpoint_input ) ) != 0 )
02155                 return rc;
02156 
02157         DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
02158                 xhci->name, slot->id, endpoint->ctx );
02159         return 0;
02160 }
02161 
02162 /**
02163  * Populate evaluate context input context
02164  *
02165  * @v xhci              xHCI device
02166  * @v slot              Device slot
02167  * @v endpoint          Endpoint
02168  * @v input             Input context
02169  */
02170 static void xhci_evaluate_context_input ( struct xhci_device *xhci,
02171                                           struct xhci_slot *slot __unused,
02172                                           struct xhci_endpoint *endpoint,
02173                                           void *input ) {
02174         struct xhci_control_context *control_ctx;
02175         struct xhci_slot_context *slot_ctx;
02176         struct xhci_endpoint_context *ep_ctx;
02177 
02178         /* Populate control context */
02179         control_ctx = input;
02180         control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
02181                                          ( 1 << endpoint->ctx ) );
02182 
02183         /* Populate slot context */
02184         slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
02185         slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
02186                                                         0, 0, 0 ) );
02187 
02188         /* Populate endpoint context */
02189         ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
02190         ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
02191 }
02192 
02193 /**
02194  * Evaluate context
02195  *
02196  * @v xhci              xHCI device
02197  * @v slot              Device slot
02198  * @v endpoint          Endpoint
02199  * @ret rc              Return status code
02200  */
02201 static inline int xhci_evaluate_context ( struct xhci_device *xhci,
02202                                           struct xhci_slot *slot,
02203                                           struct xhci_endpoint *endpoint ) {
02204         int rc;
02205 
02206         /* Configure endpoint */
02207         if ( ( rc = xhci_context ( xhci, slot, endpoint,
02208                                    XHCI_TRB_EVALUATE_CONTEXT,
02209                                    xhci_evaluate_context_input ) ) != 0 )
02210                 return rc;
02211 
02212         DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
02213                 xhci->name, slot->id, endpoint->ctx );
02214         return 0;
02215 }
02216 
02217 /**
02218  * Reset endpoint
02219  *
02220  * @v xhci              xHCI device
02221  * @v slot              Device slot
02222  * @v endpoint          Endpoint
02223  * @ret rc              Return status code
02224  */
02225 static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
02226                                         struct xhci_slot *slot,
02227                                         struct xhci_endpoint *endpoint ) {
02228         union xhci_trb trb;
02229         struct xhci_trb_reset_endpoint *reset = &trb.reset;
02230         int rc;
02231 
02232         /* Construct command */
02233         memset ( reset, 0, sizeof ( *reset ) );
02234         reset->slot = slot->id;
02235         reset->endpoint = endpoint->ctx;
02236         reset->type = XHCI_TRB_RESET_ENDPOINT;
02237 
02238         /* Issue command and wait for completion */
02239         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
02240                 DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint "
02241                        "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
02242                        endpoint->context->state, strerror ( rc ) );
02243                 return rc;
02244         }
02245 
02246         return 0;
02247 }
02248 
02249 /**
02250  * Stop endpoint
02251  *
02252  * @v xhci              xHCI device
02253  * @v slot              Device slot
02254  * @v endpoint          Endpoint
02255  * @ret rc              Return status code
02256  */
02257 static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
02258                                        struct xhci_slot *slot,
02259                                        struct xhci_endpoint *endpoint ) {
02260         union xhci_trb trb;
02261         struct xhci_trb_stop_endpoint *stop = &trb.stop;
02262         int rc;
02263 
02264         /* Construct command */
02265         memset ( stop, 0, sizeof ( *stop ) );
02266         stop->slot = slot->id;
02267         stop->endpoint = endpoint->ctx;
02268         stop->type = XHCI_TRB_STOP_ENDPOINT;
02269 
02270         /* Issue command and wait for completion */
02271         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
02272                 DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint "
02273                        "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
02274                        endpoint->context->state, strerror ( rc ) );
02275                 return rc;
02276         }
02277 
02278         return 0;
02279 }
02280 
02281 /**
02282  * Set transfer ring dequeue pointer
02283  *
02284  * @v xhci              xHCI device
02285  * @v slot              Device slot
02286  * @v endpoint          Endpoint
02287  * @ret rc              Return status code
02288  */
02289 static inline int
02290 xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
02291                               struct xhci_slot *slot,
02292                               struct xhci_endpoint *endpoint ) {
02293         union xhci_trb trb;
02294         struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
02295         struct xhci_trb_ring *ring = &endpoint->ring;
02296         unsigned int cons;
02297         unsigned int mask;
02298         unsigned int index;
02299         unsigned int dcs;
02300         int rc;
02301 
02302         /* Construct command */
02303         memset ( dequeue, 0, sizeof ( *dequeue ) );
02304         cons = ring->cons;
02305         mask = ring->mask;
02306         dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
02307         index = ( cons & mask );
02308         dequeue->dequeue =
02309                 cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
02310         dequeue->slot = slot->id;
02311         dequeue->endpoint = endpoint->ctx;
02312         dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
02313 
02314         /* Issue command and wait for completion */
02315         if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
02316                 DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue "
02317                        "pointer in state %d: %s\n", xhci->name, slot->id,
02318                        endpoint->ctx, endpoint->context->state, strerror ( rc));
02319                 return rc;
02320         }
02321 
02322         return 0;
02323 }
02324 
02325 /******************************************************************************
02326  *
02327  * Endpoint operations
02328  *
02329  ******************************************************************************
02330  */
02331 
02332 /**
02333  * Open endpoint
02334  *
02335  * @v ep                USB endpoint
02336  * @ret rc              Return status code
02337  */
02338 static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
02339         struct usb_device *usb = ep->usb;
02340         struct xhci_slot *slot = usb_get_hostdata ( usb );
02341         struct xhci_device *xhci = slot->xhci;
02342         struct xhci_endpoint *endpoint;
02343         unsigned int ctx;
02344         unsigned int type;
02345         unsigned int interval;
02346         int rc;
02347 
02348         /* Calculate context index */
02349         ctx = XHCI_CTX ( ep->address );
02350         assert ( slot->endpoint[ctx] == NULL );
02351 
02352         /* Calculate endpoint type */
02353         type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
02354         if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
02355                 type = XHCI_EP_TYPE_CONTROL;
02356         if ( ep->address & USB_DIR_IN )
02357                 type |= XHCI_EP_TYPE_IN;
02358 
02359         /* Calculate interval */
02360         if ( type & XHCI_EP_TYPE_PERIODIC ) {
02361                 interval = ( fls ( ep->interval ) - 1 );
02362         } else {
02363                 interval = ep->interval;
02364         }
02365 
02366         /* Allocate and initialise structure */
02367         endpoint = zalloc ( sizeof ( *endpoint ) );
02368         if ( ! endpoint ) {
02369                 rc = -ENOMEM;
02370                 goto err_alloc;
02371         }
02372         usb_endpoint_set_hostdata ( ep, endpoint );
02373         slot->endpoint[ctx] = endpoint;
02374         endpoint->xhci = xhci;
02375         endpoint->slot = slot;
02376         endpoint->ep = ep;
02377         endpoint->ctx = ctx;
02378         endpoint->type = type;
02379         endpoint->interval = interval;
02380         endpoint->context = ( ( ( void * ) slot->context ) +
02381                               xhci_device_context_offset ( xhci, ctx ) );
02382 
02383         /* Allocate transfer ring */
02384         if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
02385                                       XHCI_TRANSFER_TRBS_LOG2,
02386                                       slot->id, ctx, 0 ) ) != 0 )
02387                 goto err_ring_alloc;
02388 
02389         /* Configure endpoint, if applicable */
02390         if ( ( ctx != XHCI_CTX_EP0 ) &&
02391              ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
02392                 goto err_configure_endpoint;
02393 
02394         DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n",
02395                 xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
02396                 ( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
02397         return 0;
02398 
02399         xhci_deconfigure_endpoint ( xhci, slot, endpoint );
02400  err_configure_endpoint:
02401         xhci_ring_free ( &endpoint->ring );
02402  err_ring_alloc:
02403         slot->endpoint[ctx] = NULL;
02404         free ( endpoint );
02405  err_alloc:
02406         return rc;
02407 }
02408 
02409 /**
02410  * Close endpoint
02411  *
02412  * @v ep                USB endpoint
02413  */
02414 static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
02415         struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
02416         struct xhci_slot *slot = endpoint->slot;
02417         struct xhci_device *xhci = slot->xhci;
02418         struct io_buffer *iobuf;
02419         unsigned int ctx = endpoint->ctx;
02420 
02421         /* Deconfigure endpoint, if applicable */
02422         if ( ctx != XHCI_CTX_EP0 )
02423                 xhci_deconfigure_endpoint ( xhci, slot, endpoint );
02424 
02425         /* Cancel any incomplete transfers */
02426         while ( xhci_ring_fill ( &endpoint->ring ) ) {
02427                 iobuf = xhci_dequeue_multi ( &endpoint->ring );
02428                 usb_complete_err ( ep, iobuf, -ECANCELED );
02429         }
02430 
02431         /* Free endpoint */
02432         xhci_ring_free ( &endpoint->ring );
02433         slot->endpoint[ctx] = NULL;
02434         free ( endpoint );
02435 }
02436 
02437 /**
02438  * Reset endpoint
02439  *
02440  * @v ep                USB endpoint
02441  * @ret rc              Return status code
02442  */
02443 static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
02444         struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
02445         struct xhci_slot *slot = endpoint->slot;
02446         struct xhci_device *xhci = slot->xhci;
02447         int rc;
02448 
02449         /* Reset endpoint context */
02450         if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
02451                 return rc;
02452 
02453         /* Set transfer ring dequeue pointer */
02454         if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
02455                 return rc;
02456 
02457         /* Ring doorbell to resume processing */
02458         xhci_doorbell ( &endpoint->ring );
02459 
02460         DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n",
02461                xhci->name, slot->id, endpoint->ctx );
02462         return 0;
02463 }
02464 
02465 /**
02466  * Update MTU
02467  *
02468  * @v ep                USB endpoint
02469  * @ret rc              Return status code
02470  */
02471 static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
02472         struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
02473         struct xhci_slot *slot = endpoint->slot;
02474         struct xhci_device *xhci = slot->xhci;
02475         int rc;
02476 
02477         /* Evalulate context */
02478         if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
02479                 return rc;
02480 
02481         return 0;
02482 }
02483 
02484 /**
02485  * Enqueue message transfer
02486  *
02487  * @v ep                USB endpoint
02488  * @v iobuf             I/O buffer
02489  * @ret rc              Return status code
02490  */
02491 static int xhci_endpoint_message ( struct usb_endpoint *ep,
02492                                    struct io_buffer *iobuf ) {
02493         struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
02494         struct usb_setup_packet *packet;
02495         unsigned int input;
02496         size_t len;
02497         union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
02498                              1 /* status */ ];
02499         union xhci_trb *trb = trbs;
02500         struct xhci_trb_setup *setup;
02501         struct xhci_trb_data *data;
02502         struct xhci_trb_status *status;
02503         int rc;
02504 
02505         /* Profile message transfers */
02506         profile_start ( &xhci_message_profiler );
02507 
02508         /* Construct setup stage TRB */
02509         memset ( trbs, 0, sizeof ( trbs ) );
02510         assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
02511         packet = iobuf->data;
02512         iob_pull ( iobuf, sizeof ( *packet ) );
02513         setup = &(trb++)->setup;
02514         memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
02515         setup->len = cpu_to_le32 ( sizeof ( *packet ) );
02516         setup->flags = XHCI_TRB_IDT;
02517         setup->type = XHCI_TRB_SETUP;
02518         len = iob_len ( iobuf );
02519         input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
02520         if ( len )
02521                 setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
02522 
02523         /* Construct data stage TRB, if applicable */
02524         if ( len ) {
02525                 data = &(trb++)->data;
02526                 data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
02527                 data->len = cpu_to_le32 ( len );
02528                 data->type = XHCI_TRB_DATA;
02529                 data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
02530         }
02531 
02532         /* Construct status stage TRB */
02533         status = &(trb++)->status;
02534         status->flags = XHCI_TRB_IOC;
02535         status->type = XHCI_TRB_STATUS;
02536         status->direction =
02537                 ( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
02538 
02539         /* Enqueue TRBs */
02540         if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
02541                                          ( trb - trbs ) ) ) != 0 )
02542                 return rc;
02543 
02544         /* Ring the doorbell */
02545         xhci_doorbell ( &endpoint->ring );
02546 
02547         profile_stop ( &xhci_message_profiler );
02548         return 0;
02549 }
02550 
02551 /**
02552  * Calculate number of TRBs
02553  *
02554  * @v len               Length of data
02555  * @v zlp               Append a zero-length packet
02556  * @ret count           Number of transfer descriptors
02557  */
02558 static unsigned int xhci_endpoint_count ( size_t len, int zlp ) {
02559         unsigned int count;
02560 
02561         /* Split into 64kB TRBs */
02562         count = ( ( len + XHCI_MTU - 1 ) / XHCI_MTU );
02563 
02564         /* Append a zero-length TRB if applicable */
02565         if ( zlp || ( count == 0 ) )
02566                 count++;
02567 
02568         return count;
02569 }
02570 
02571 /**
02572  * Enqueue stream transfer
02573  *
02574  * @v ep                USB endpoint
02575  * @v iobuf             I/O buffer
02576  * @v zlp               Append a zero-length packet
02577  * @ret rc              Return status code
02578  */
02579 static int xhci_endpoint_stream ( struct usb_endpoint *ep,
02580                                   struct io_buffer *iobuf, int zlp ) {
02581         struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
02582         void *data = iobuf->data;
02583         size_t len = iob_len ( iobuf );
02584         unsigned int count = xhci_endpoint_count ( len, zlp );
02585         union xhci_trb trbs[count];
02586         union xhci_trb *trb = trbs;
02587         struct xhci_trb_normal *normal;
02588         unsigned int i;
02589         size_t trb_len;
02590         int rc;
02591 
02592         /* Profile stream transfers */
02593         profile_start ( &xhci_stream_profiler );
02594 
02595         /* Construct normal TRBs */
02596         memset ( &trbs, 0, sizeof ( trbs ) );
02597         for ( i = 0 ; i < count ; i ++ ) {
02598 
02599                 /* Calculate TRB length */
02600                 trb_len = XHCI_MTU;
02601                 if ( trb_len > len )
02602                         trb_len = len;
02603 
02604                 /* Construct normal TRB */
02605                 normal = &trb->normal;
02606                 normal->data = cpu_to_le64 ( virt_to_phys ( data ) );
02607                 normal->len = cpu_to_le32 ( trb_len );
02608                 normal->type = XHCI_TRB_NORMAL;
02609                 normal->flags = XHCI_TRB_CH;
02610 
02611                 /* Move to next TRB */
02612                 data += trb_len;
02613                 len -= trb_len;
02614                 trb++;
02615         }
02616 
02617         /* Mark zero-length packet (if present) as a separate transfer */
02618         if ( zlp && ( count > 1 ) )
02619                 trb[-2].normal.flags = 0;
02620 
02621         /* Generate completion for final TRB */
02622         trb[-1].normal.flags = XHCI_TRB_IOC;
02623 
02624         /* Enqueue TRBs */
02625         if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
02626                                          count ) ) != 0 )
02627                 return rc;
02628 
02629         /* Ring the doorbell */
02630         xhci_doorbell ( &endpoint->ring );
02631 
02632         profile_stop ( &xhci_stream_profiler );
02633         return 0;
02634 }
02635 
02636 /******************************************************************************
02637  *
02638  * Device operations
02639  *
02640  ******************************************************************************
02641  */
02642 
02643 /**
02644  * Open device
02645  *
02646  * @v usb               USB device
02647  * @ret rc              Return status code
02648  */
02649 static int xhci_device_open ( struct usb_device *usb ) {
02650         struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
02651         struct usb_port *tt = usb_transaction_translator ( usb );
02652         struct xhci_slot *slot;
02653         struct xhci_slot *tt_slot;
02654         size_t len;
02655         int type;
02656         int id;
02657         int rc;
02658 
02659         /* Determine applicable slot type */
02660         type = xhci_port_slot_type ( xhci, usb->port->address );
02661         if ( type < 0 ) {
02662                 rc = type;
02663                 DBGC ( xhci, "XHCI %s-%d has no slot type\n",
02664                        xhci->name, usb->port->address );
02665                 goto err_type;
02666         }
02667 
02668         /* Allocate a device slot number */
02669         id = xhci_enable_slot ( xhci, type );
02670         if ( id < 0 ) {
02671                 rc = id;
02672                 goto err_enable_slot;
02673         }
02674         assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) );
02675         assert ( xhci->slot[id] == NULL );
02676 
02677         /* Allocate and initialise structure */
02678         slot = zalloc ( sizeof ( *slot ) );
02679         if ( ! slot ) {
02680                 rc = -ENOMEM;
02681                 goto err_alloc;
02682         }
02683         usb_set_hostdata ( usb, slot );
02684         xhci->slot[id] = slot;
02685         slot->xhci = xhci;
02686         slot->usb = usb;
02687         slot->id = id;
02688         if ( tt ) {
02689                 tt_slot = usb_get_hostdata ( tt->hub->usb );
02690                 slot->tt_id = tt_slot->id;
02691                 slot->tt_port = tt->address;
02692         }
02693 
02694         /* Allocate a device context */
02695         len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
02696         slot->context = malloc_dma ( len, xhci_align ( len ) );
02697         if ( ! slot->context ) {
02698                 rc = -ENOMEM;
02699                 goto err_alloc_context;
02700         }
02701         memset ( slot->context, 0, len );
02702 
02703         /* Set device context base address */
02704         assert ( xhci->dcbaa[id] == 0 );
02705         xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
02706 
02707         DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
02708                 xhci->name, slot->id, virt_to_phys ( slot->context ),
02709                 ( virt_to_phys ( slot->context ) + len ), usb->name );
02710         return 0;
02711 
02712         xhci->dcbaa[id] = 0;
02713         free_dma ( slot->context, len );
02714  err_alloc_context:
02715         xhci->slot[id] = NULL;
02716         free ( slot );
02717  err_alloc:
02718         xhci_disable_slot ( xhci, id );
02719  err_enable_slot:
02720  err_type:
02721         return rc;
02722 }
02723 
02724 /**
02725  * Close device
02726  *
02727  * @v usb               USB device
02728  */
02729 static void xhci_device_close ( struct usb_device *usb ) {
02730         struct xhci_slot *slot = usb_get_hostdata ( usb );
02731         struct xhci_device *xhci = slot->xhci;
02732         size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
02733         unsigned int id = slot->id;
02734         int rc;
02735 
02736         /* Disable slot */
02737         if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) {
02738                 /* Slot is still enabled.  Leak the slot context,
02739                  * since the controller may still write to this
02740                  * memory, and leave the DCBAA entry intact.
02741                  *
02742                  * If the controller later reports that this same slot
02743                  * has been re-enabled, then some assertions will be
02744                  * triggered.
02745                  */
02746                 DBGC ( xhci, "XHCI %s slot %d leaking context memory\n",
02747                        xhci->name, slot->id );
02748                 slot->context = NULL;
02749         }
02750 
02751         /* Free slot */
02752         if ( slot->context ) {
02753                 free_dma ( slot->context, len );
02754                 xhci->dcbaa[id] = 0;
02755         }
02756         xhci->slot[id] = NULL;
02757         free ( slot );
02758 }
02759 
02760 /**
02761  * Assign device address
02762  *
02763  * @v usb               USB device
02764  * @ret rc              Return status code
02765  */
02766 static int xhci_device_address ( struct usb_device *usb ) {
02767         struct xhci_slot *slot = usb_get_hostdata ( usb );
02768         struct xhci_device *xhci = slot->xhci;
02769         struct usb_port *root_port;
02770         int psiv;
02771         int rc;
02772 
02773         /* Calculate route string */
02774         slot->route = usb_route_string ( usb );
02775 
02776         /* Calculate root hub port number */
02777         root_port = usb_root_hub_port ( usb );
02778         slot->port = root_port->address;
02779 
02780         /* Calculate protocol speed ID */
02781         psiv = xhci_port_psiv ( xhci, slot->port, usb->speed );
02782         if ( psiv < 0 ) {
02783                 rc = psiv;
02784                 return rc;
02785         }
02786         slot->psiv = psiv;
02787 
02788         /* Address device */
02789         if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
02790                 return rc;
02791 
02792         return 0;
02793 }
02794 
02795 /******************************************************************************
02796  *
02797  * Bus operations
02798  *
02799  ******************************************************************************
02800  */
02801 
02802 /**
02803  * Open USB bus
02804  *
02805  * @v bus               USB bus
02806  * @ret rc              Return status code
02807  */
02808 static int xhci_bus_open ( struct usb_bus *bus ) {
02809         struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
02810         int rc;
02811 
02812         /* Allocate device slot array */
02813         xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) );
02814         if ( ! xhci->slot ) {
02815                 rc = -ENOMEM;
02816                 goto err_slot_alloc;
02817         }
02818 
02819         /* Allocate device context base address array */
02820         if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
02821                 goto err_dcbaa_alloc;
02822 
02823         /* Allocate scratchpad buffers */
02824         if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
02825                 goto err_scratchpad_alloc;
02826 
02827         /* Allocate command ring */
02828         if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
02829                 goto err_command_alloc;
02830 
02831         /* Allocate event ring */
02832         if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
02833                 goto err_event_alloc;
02834 
02835         /* Start controller */
02836         xhci_run ( xhci );
02837 
02838         return 0;
02839 
02840         xhci_stop ( xhci );
02841         xhci_event_free ( xhci );
02842  err_event_alloc:
02843         xhci_command_free ( xhci );
02844  err_command_alloc:
02845         xhci_scratchpad_free ( xhci );
02846  err_scratchpad_alloc:
02847         xhci_dcbaa_free ( xhci );
02848  err_dcbaa_alloc:
02849         free ( xhci->slot );
02850  err_slot_alloc:
02851         return rc;
02852 }
02853 
02854 /**
02855  * Close USB bus
02856  *
02857  * @v bus               USB bus
02858  */
02859 static void xhci_bus_close ( struct usb_bus *bus ) {
02860         struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
02861         unsigned int i;
02862 
02863         /* Sanity checks */
02864         assert ( xhci->slot != NULL );
02865         for ( i = 0 ; i <= xhci->slots ; i++ )
02866                 assert ( xhci->slot[i] == NULL );
02867 
02868         xhci_stop ( xhci );
02869         xhci_event_free ( xhci );
02870         xhci_command_free ( xhci );
02871         xhci_scratchpad_free ( xhci );
02872         xhci_dcbaa_free ( xhci );
02873         free ( xhci->slot );
02874 }
02875 
02876 /**
02877  * Poll USB bus
02878  *
02879  * @v bus               USB bus
02880  */
02881 static void xhci_bus_poll ( struct usb_bus *bus ) {
02882         struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
02883 
02884         /* Poll event ring */
02885         xhci_event_poll ( xhci );
02886 }
02887 
02888 /******************************************************************************
02889  *
02890  * Hub operations
02891  *
02892  ******************************************************************************
02893  */
02894 
02895 /**
02896  * Open hub
02897  *
02898  * @v hub               USB hub
02899  * @ret rc              Return status code
02900  */
02901 static int xhci_hub_open ( struct usb_hub *hub ) {
02902         struct xhci_slot *slot;
02903 
02904         /* Do nothing if this is the root hub */
02905         if ( ! hub->usb )
02906                 return 0;
02907 
02908         /* Get device slot */
02909         slot = usb_get_hostdata ( hub->usb );
02910 
02911         /* Update device slot hub parameters.  We don't inform the
02912          * hardware of this information until the hub's interrupt
02913          * endpoint is opened, since the only mechanism for so doing
02914          * provided by the xHCI specification is a Configure Endpoint
02915          * command, and we can't issue that command until we have a
02916          * non-EP0 endpoint to configure.
02917          */
02918         slot->ports = hub->ports;
02919 
02920         return 0;
02921 }
02922 
02923 /**
02924  * Close hub
02925  *
02926  * @v hub               USB hub
02927  */
02928 static void xhci_hub_close ( struct usb_hub *hub __unused ) {
02929 
02930         /* Nothing to do */
02931 }
02932 
02933 /******************************************************************************
02934  *
02935  * Root hub operations
02936  *
02937  ******************************************************************************
02938  */
02939 
02940 /**
02941  * Open root hub
02942  *
02943  * @v hub               USB hub
02944  * @ret rc              Return status code
02945  */
02946 static int xhci_root_open ( struct usb_hub *hub ) {
02947         struct usb_bus *bus = hub->bus;
02948         struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
02949         struct usb_port *port;
02950         uint32_t portsc;
02951         unsigned int i;
02952 
02953         /* Enable power to all ports */
02954         for ( i = 1 ; i <= xhci->ports ; i++ ) {
02955                 portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
02956                 portsc &= XHCI_PORTSC_PRESERVE;
02957                 portsc |= XHCI_PORTSC_PP;
02958                 writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
02959         }
02960 
02961         /* xHCI spec requires us to potentially wait 20ms after
02962          * enabling power to a port.
02963          */
02964         mdelay ( XHCI_PORT_POWER_DELAY_MS );
02965 
02966         /* USB3 ports may power up as Disabled */
02967         for ( i = 1 ; i <= xhci->ports ; i++ ) {
02968                 portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
02969                 port = usb_port ( hub, i );
02970                 if ( ( port->protocol >= USB_PROTO_3_0 ) &&
02971                      ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
02972                        XHCI_PORTSC_PLS_DISABLED ) ) {
02973                         /* Force link state to RxDetect */
02974                         portsc &= XHCI_PORTSC_PRESERVE;
02975                         portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
02976                         writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
02977                 }
02978         }
02979 
02980         /* Some xHCI cards seem to require an additional delay after
02981          * setting the link state to RxDetect.
02982          */
02983         mdelay ( XHCI_LINK_STATE_DELAY_MS );
02984 
02985         /* Record hub driver private data */
02986         usb_hub_set_drvdata ( hub, xhci );
02987 
02988         return 0;
02989 }
02990 
02991 /**
02992  * Close root hub
02993  *
02994  * @v hub               USB hub
02995  */
02996 static void xhci_root_close ( struct usb_hub *hub ) {
02997 
02998         /* Clear hub driver private data */
02999         usb_hub_set_drvdata ( hub, NULL );
03000 }
03001 
03002 /**
03003  * Enable port
03004  *
03005  * @v hub               USB hub
03006  * @v port              USB port
03007  * @ret rc              Return status code
03008  */
03009 static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
03010         struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
03011         uint32_t portsc;
03012         unsigned int i;
03013 
03014         /* Reset port */
03015         portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
03016         portsc &= XHCI_PORTSC_PRESERVE;
03017         portsc |= XHCI_PORTSC_PR;
03018         writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
03019 
03020         /* Wait for port to become enabled */
03021         for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
03022 
03023                 /* Check port status */
03024                 portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
03025                 if ( portsc & XHCI_PORTSC_PED )
03026                         return 0;
03027 
03028                 /* Delay */
03029                 mdelay ( 1 );
03030         }
03031 
03032         DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n",
03033                xhci->name, port->address );
03034         return -ETIMEDOUT;
03035 }
03036 
03037 /**
03038  * Disable port
03039  *
03040  * @v hub               USB hub
03041  * @v port              USB port
03042  * @ret rc              Return status code
03043  */
03044 static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
03045         struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
03046         uint32_t portsc;
03047 
03048         /* Disable port */
03049         portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
03050         portsc &= XHCI_PORTSC_PRESERVE;
03051         portsc |= XHCI_PORTSC_PED;
03052         writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
03053 
03054         return 0;
03055 }
03056 
03057 /**
03058  * Update root hub port speed
03059  *
03060  * @v hub               USB hub
03061  * @v port              USB port
03062  * @ret rc              Return status code
03063  */
03064 static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
03065         struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
03066         uint32_t portsc;
03067         unsigned int psiv;
03068         int ccs;
03069         int ped;
03070         int csc;
03071         int speed;
03072         int rc;
03073 
03074         /* Read port status */
03075         portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
03076         DBGC2 ( xhci, "XHCI %s-%d status is %08x\n",
03077                 xhci->name, port->address, portsc );
03078         ccs = ( portsc & XHCI_PORTSC_CCS );
03079         ped = ( portsc & XHCI_PORTSC_PED );
03080         csc = ( portsc & XHCI_PORTSC_CSC );
03081         psiv = XHCI_PORTSC_PSIV ( portsc );
03082 
03083         /* Record disconnections and clear changes */
03084         port->disconnected |= csc;
03085         portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
03086         writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
03087 
03088         /* Port speed is not valid unless port is connected */
03089         if ( ! ccs ) {
03090                 port->speed = USB_SPEED_NONE;
03091                 return 0;
03092         }
03093 
03094         /* For USB2 ports, the PSIV field is not valid until the port
03095          * completes reset and becomes enabled.
03096          */
03097         if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
03098                 port->speed = USB_SPEED_FULL;
03099                 return 0;
03100         }
03101 
03102         /* Get port speed and map to generic USB speed */
03103         speed = xhci_port_speed ( xhci, port->address, psiv );
03104         if ( speed < 0 ) {
03105                 rc = speed;
03106                 return rc;
03107         }
03108 
03109         port->speed = speed;
03110         return 0;
03111 }
03112 
03113 /**
03114  * Clear transaction translator buffer
03115  *
03116  * @v hub               USB hub
03117  * @v port              USB port
03118  * @v ep                USB endpoint
03119  * @ret rc              Return status code
03120  */
03121 static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
03122                                 struct usb_endpoint *ep ) {
03123         struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
03124 
03125         /* Should never be called; this is a root hub */
03126         DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name,
03127                port->address, ep->usb->name, usb_endpoint_name ( ep ) );
03128 
03129         return -ENOTSUP;
03130 }
03131 
03132 /******************************************************************************
03133  *
03134  * PCI interface
03135  *
03136  ******************************************************************************
03137  */
03138 
03139 /** USB host controller operations */
03140 static struct usb_host_operations xhci_operations = {
03141         .endpoint = {
03142                 .open = xhci_endpoint_open,
03143                 .close = xhci_endpoint_close,
03144                 .reset = xhci_endpoint_reset,
03145                 .mtu = xhci_endpoint_mtu,
03146                 .message = xhci_endpoint_message,
03147                 .stream = xhci_endpoint_stream,
03148         },
03149         .device = {
03150                 .open = xhci_device_open,
03151                 .close = xhci_device_close,
03152                 .address = xhci_device_address,
03153         },
03154         .bus = {
03155                 .open = xhci_bus_open,
03156                 .close = xhci_bus_close,
03157                 .poll = xhci_bus_poll,
03158         },
03159         .hub = {
03160                 .open = xhci_hub_open,
03161                 .close = xhci_hub_close,
03162         },
03163         .root = {
03164                 .open = xhci_root_open,
03165                 .close = xhci_root_close,
03166                 .enable = xhci_root_enable,
03167                 .disable = xhci_root_disable,
03168                 .speed = xhci_root_speed,
03169                 .clear_tt = xhci_root_clear_tt,
03170         },
03171 };
03172 
03173 /**
03174  * Fix Intel PCH-specific quirks
03175  *
03176  * @v xhci              xHCI device
03177  * @v pci               PCI device
03178  */
03179 static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) {
03180         struct xhci_pch *pch = &xhci->pch;
03181         uint32_t xusb2pr;
03182         uint32_t xusb2prm;
03183         uint32_t usb3pssen;
03184         uint32_t usb3prm;
03185 
03186         /* Enable SuperSpeed capability.  Do this before rerouting
03187          * USB2 ports, so that USB3 devices connect at SuperSpeed.
03188          */
03189         pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen );
03190         pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm );
03191         if ( usb3prm & ~usb3pssen ) {
03192                 DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n",
03193                        xhci->name, ( usb3prm & ~usb3pssen ) );
03194         }
03195         pch->usb3pssen = usb3pssen;
03196         usb3pssen |= usb3prm;
03197         pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen );
03198 
03199         /* Route USB2 ports from EHCI to xHCI */
03200         pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr );
03201         pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm );
03202         if ( xusb2prm & ~xusb2pr ) {
03203                 DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n",
03204                        xhci->name, ( xusb2prm & ~xusb2pr ) );
03205         }
03206         pch->xusb2pr = xusb2pr;
03207         xusb2pr |= xusb2prm;
03208         pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr );
03209 }
03210 
03211 /**
03212  * Undo Intel PCH-specific quirk fixes
03213  *
03214  * @v xhci              xHCI device
03215  * @v pci               PCI device
03216  */
03217 static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) {
03218         struct xhci_pch *pch = &xhci->pch;
03219 
03220         /* Restore USB2 port routing to original state */
03221         pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr );
03222 
03223         /* Restore SuperSpeed capability to original state */
03224         pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen );
03225 }
03226 
03227 /**
03228  * Probe PCI device
03229  *
03230  * @v pci               PCI device
03231  * @ret rc              Return status code
03232  */
03233 static int xhci_probe ( struct pci_device *pci ) {
03234         struct xhci_device *xhci;
03235         struct usb_port *port;
03236         unsigned long bar_start;
03237         size_t bar_size;
03238         unsigned int i;
03239         int rc;
03240 
03241         /* Allocate and initialise structure */
03242         xhci = zalloc ( sizeof ( *xhci ) );
03243         if ( ! xhci ) {
03244                 rc = -ENOMEM;
03245                 goto err_alloc;
03246         }
03247         xhci->name = pci->dev.name;
03248         xhci->quirks = pci->id->driver_data;
03249 
03250         /* Fix up PCI device */
03251         adjust_pci_device ( pci );
03252 
03253         /* Map registers */
03254         bar_start = pci_bar_start ( pci, XHCI_BAR );
03255         bar_size = pci_bar_size ( pci, XHCI_BAR );
03256         xhci->regs = ioremap ( bar_start, bar_size );
03257         if ( ! xhci->regs ) {
03258                 rc = -ENODEV;
03259                 goto err_ioremap;
03260         }
03261 
03262         /* Initialise xHCI device */
03263         xhci_init ( xhci, xhci->regs );
03264 
03265         /* Initialise USB legacy support and claim ownership */
03266         xhci_legacy_init ( xhci );
03267         xhci_legacy_claim ( xhci );
03268 
03269         /* Fix Intel PCH-specific quirks, if applicable */
03270         if ( xhci->quirks & XHCI_PCH )
03271                 xhci_pch_fix ( xhci, pci );
03272 
03273         /* Reset device */
03274         if ( ( rc = xhci_reset ( xhci ) ) != 0 )
03275                 goto err_reset;
03276 
03277         /* Allocate USB bus */
03278         xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports, XHCI_MTU,
03279                                     &xhci_operations );
03280         if ( ! xhci->bus ) {
03281                 rc = -ENOMEM;
03282                 goto err_alloc_bus;
03283         }
03284         usb_bus_set_hostdata ( xhci->bus, xhci );
03285         usb_hub_set_drvdata ( xhci->bus->hub, xhci );
03286 
03287         /* Set port protocols */
03288         for ( i = 1 ; i <= xhci->ports ; i++ ) {
03289                 port = usb_port ( xhci->bus->hub, i );
03290                 port->protocol = xhci_port_protocol ( xhci, i );
03291         }
03292 
03293         /* Register USB bus */
03294         if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
03295                 goto err_register;
03296 
03297         pci_set_drvdata ( pci, xhci );
03298         return 0;
03299 
03300         unregister_usb_bus ( xhci->bus );
03301  err_register:
03302         free_usb_bus ( xhci->bus );
03303  err_alloc_bus:
03304         xhci_reset ( xhci );
03305  err_reset:
03306         if ( xhci->quirks & XHCI_PCH )
03307                 xhci_pch_undo ( xhci, pci );
03308         xhci_legacy_release ( xhci );
03309         iounmap ( xhci->regs );
03310  err_ioremap:
03311         free ( xhci );
03312  err_alloc:
03313         return rc;
03314 }
03315 
03316 /**
03317  * Remove PCI device
03318  *
03319  * @v pci               PCI device
03320  */
03321 static void xhci_remove ( struct pci_device *pci ) {
03322         struct xhci_device *xhci = pci_get_drvdata ( pci );
03323         struct usb_bus *bus = xhci->bus;
03324 
03325         unregister_usb_bus ( bus );
03326         free_usb_bus ( bus );
03327         xhci_reset ( xhci );
03328         if ( xhci->quirks & XHCI_PCH )
03329                 xhci_pch_undo ( xhci, pci );
03330         xhci_legacy_release ( xhci );
03331         iounmap ( xhci->regs );
03332         free ( xhci );
03333 }
03334 
03335 /** XHCI PCI device IDs */
03336 static struct pci_device_id xhci_ids[] = {
03337         PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ),
03338         PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ),
03339         PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
03340 };
03341 
03342 /** XHCI PCI driver */
03343 struct pci_driver xhci_driver __pci_driver = {
03344         .ids = xhci_ids,
03345         .id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
03346         .class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
03347                                 PCI_CLASS_SERIAL_USB_XHCI ),
03348         .probe = xhci_probe,
03349         .remove = xhci_remove,
03350 };
03351 
03352 /**
03353  * Prepare for exit
03354  *
03355  * @v booting           System is shutting down for OS boot
03356  */
03357 static void xhci_shutdown ( int booting ) {
03358         /* If we are shutting down to boot an OS, then prevent the
03359          * release of ownership back to BIOS.
03360          */
03361         xhci_legacy_prevent_release = booting;
03362 }
03363 
03364 /** Startup/shutdown function */
03365 struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = {
03366         .shutdown = xhci_shutdown,
03367 };