iPXE
ehci.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or (at your option) any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdlib.h>
00027 #include <unistd.h>
00028 #include <string.h>
00029 #include <strings.h>
00030 #include <errno.h>
00031 #include <byteswap.h>
00032 #include <ipxe/malloc.h>
00033 #include <ipxe/pci.h>
00034 #include <ipxe/usb.h>
00035 #include <ipxe/init.h>
00036 #include "ehci.h"
00037 
00038 /** @file
00039  *
00040  * USB Enhanced Host Controller Interface (EHCI) driver
00041  *
00042  */
00043 
00044 /**
00045  * Construct error code from transfer descriptor status
00046  *
00047  * @v status            Transfer descriptor status
00048  * @ret rc              Error code
00049  *
00050  * Bits 2-5 of the status code provide some indication as to the root
00051  * cause of the error.  We incorporate these into the error code as
00052  * reported to usb_complete_err().
00053  */
00054 #define EIO_STATUS( status ) EUNIQ ( EINFO_EIO, ( ( (status) >> 2 ) & 0xf ) )
00055 
00056 /******************************************************************************
00057  *
00058  * Register access
00059  *
00060  ******************************************************************************
00061  */
00062 
00063 /**
00064  * Initialise device
00065  *
00066  * @v ehci              EHCI device
00067  * @v regs              MMIO registers
00068  */
00069 static void ehci_init ( struct ehci_device *ehci, void *regs ) {
00070         uint32_t hcsparams;
00071         uint32_t hccparams;
00072         size_t caplength;
00073 
00074         /* Locate capability and operational registers */
00075         ehci->cap = regs;
00076         caplength = readb ( ehci->cap + EHCI_CAP_CAPLENGTH );
00077         ehci->op = ( ehci->cap + caplength );
00078         DBGC2 ( ehci, "EHCI %s cap %08lx op %08lx\n", ehci->name,
00079                 virt_to_phys ( ehci->cap ), virt_to_phys ( ehci->op ) );
00080 
00081         /* Read structural parameters */
00082         hcsparams = readl ( ehci->cap + EHCI_CAP_HCSPARAMS );
00083         ehci->ports = EHCI_HCSPARAMS_PORTS ( hcsparams );
00084         DBGC ( ehci, "EHCI %s has %d ports\n", ehci->name, ehci->ports );
00085 
00086         /* Read capability parameters 1 */
00087         hccparams = readl ( ehci->cap + EHCI_CAP_HCCPARAMS );
00088         ehci->addr64 = EHCI_HCCPARAMS_ADDR64 ( hccparams );
00089         ehci->flsize = ( EHCI_HCCPARAMS_FLSIZE ( hccparams ) ?
00090                          EHCI_FLSIZE_SMALL : EHCI_FLSIZE_DEFAULT );
00091         ehci->eecp = EHCI_HCCPARAMS_EECP ( hccparams );
00092         DBGC2 ( ehci, "EHCI %s %d-bit flsize %d\n", ehci->name,
00093                 ( ehci->addr64 ? 64 : 32 ), ehci->flsize );
00094 }
00095 
00096 /**
00097  * Find extended capability
00098  *
00099  * @v ehci              EHCI device
00100  * @v pci               PCI device
00101  * @v id                Capability ID
00102  * @v offset            Offset to previous extended capability instance, or zero
00103  * @ret offset          Offset to extended capability, or zero if not found
00104  */
00105 static unsigned int ehci_extended_capability ( struct ehci_device *ehci,
00106                                                struct pci_device *pci,
00107                                                unsigned int id,
00108                                                unsigned int offset ) {
00109         uint32_t eecp;
00110 
00111         /* Locate the extended capability */
00112         while ( 1 ) {
00113 
00114                 /* Locate first or next capability as applicable */
00115                 if ( offset ) {
00116                         pci_read_config_dword ( pci, offset, &eecp );
00117                         offset = EHCI_EECP_NEXT ( eecp );
00118                 } else {
00119                         offset = ehci->eecp;
00120                 }
00121                 if ( ! offset )
00122                         return 0;
00123 
00124                 /* Check if this is the requested capability */
00125                 pci_read_config_dword ( pci, offset, &eecp );
00126                 if ( EHCI_EECP_ID ( eecp ) == id )
00127                         return offset;
00128         }
00129 }
00130 
00131 /**
00132  * Calculate buffer alignment
00133  *
00134  * @v len               Length
00135  * @ret align           Buffer alignment
00136  *
00137  * Determine alignment required for a buffer which must be aligned to
00138  * at least EHCI_MIN_ALIGN and which must not cross a page boundary.
00139  */
00140 static inline size_t ehci_align ( size_t len ) {
00141         size_t align;
00142 
00143         /* Align to own length (rounded up to a power of two) */
00144         align = ( 1 << fls ( len - 1 ) );
00145 
00146         /* Round up to EHCI_MIN_ALIGN if needed */
00147         if ( align < EHCI_MIN_ALIGN )
00148                 align = EHCI_MIN_ALIGN;
00149 
00150         return align;
00151 }
00152 
00153 /**
00154  * Check control data structure reachability
00155  *
00156  * @v ehci              EHCI device
00157  * @v ptr               Data structure pointer
00158  * @ret rc              Return status code
00159  */
00160 static int ehci_ctrl_reachable ( struct ehci_device *ehci, void *ptr ) {
00161         physaddr_t phys = virt_to_phys ( ptr );
00162         uint32_t segment;
00163 
00164         /* Always reachable in a 32-bit build */
00165         if ( sizeof ( physaddr_t ) <= sizeof ( uint32_t ) )
00166                 return 0;
00167 
00168         /* Reachable only if control segment matches in a 64-bit build */
00169         segment = ( ( ( uint64_t ) phys ) >> 32 );
00170         if ( segment == ehci->ctrldssegment )
00171                 return 0;
00172 
00173         return -ENOTSUP;
00174 }
00175 
00176 /******************************************************************************
00177  *
00178  * Diagnostics
00179  *
00180  ******************************************************************************
00181  */
00182 
00183 /**
00184  * Dump host controller registers
00185  *
00186  * @v ehci              EHCI device
00187  */
00188 static __unused void ehci_dump ( struct ehci_device *ehci ) {
00189         uint8_t caplength;
00190         uint16_t hciversion;
00191         uint32_t hcsparams;
00192         uint32_t hccparams;
00193         uint32_t usbcmd;
00194         uint32_t usbsts;
00195         uint32_t usbintr;
00196         uint32_t frindex;
00197         uint32_t ctrldssegment;
00198         uint32_t periodiclistbase;
00199         uint32_t asynclistaddr;
00200         uint32_t configflag;
00201 
00202         /* Do nothing unless debugging is enabled */
00203         if ( ! DBG_LOG )
00204                 return;
00205 
00206         /* Dump capability registers */
00207         caplength = readb ( ehci->cap + EHCI_CAP_CAPLENGTH );
00208         hciversion = readw ( ehci->cap + EHCI_CAP_HCIVERSION );
00209         hcsparams = readl ( ehci->cap + EHCI_CAP_HCSPARAMS );
00210         hccparams = readl ( ehci->cap + EHCI_CAP_HCCPARAMS );
00211         DBGC ( ehci, "EHCI %s caplen %02x hciversion %04x hcsparams %08x "
00212                "hccparams %08x\n", ehci->name, caplength, hciversion,
00213                hcsparams,  hccparams );
00214 
00215         /* Dump operational registers */
00216         usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
00217         usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
00218         usbintr = readl ( ehci->op + EHCI_OP_USBINTR );
00219         frindex = readl ( ehci->op + EHCI_OP_FRINDEX );
00220         ctrldssegment = readl ( ehci->op + EHCI_OP_CTRLDSSEGMENT );
00221         periodiclistbase = readl ( ehci->op + EHCI_OP_PERIODICLISTBASE );
00222         asynclistaddr = readl ( ehci->op + EHCI_OP_ASYNCLISTADDR );
00223         configflag = readl ( ehci->op + EHCI_OP_CONFIGFLAG );
00224         DBGC ( ehci, "EHCI %s usbcmd %08x usbsts %08x usbint %08x frindx "
00225                "%08x\n", ehci->name, usbcmd, usbsts, usbintr, frindex );
00226         DBGC ( ehci, "EHCI %s ctrlds %08x period %08x asyncl %08x cfgflg "
00227                "%08x\n", ehci->name, ctrldssegment, periodiclistbase,
00228                asynclistaddr, configflag );
00229 }
00230 
00231 /******************************************************************************
00232  *
00233  * USB legacy support
00234  *
00235  ******************************************************************************
00236  */
00237 
00238 /** Prevent the release of ownership back to BIOS */
00239 static int ehci_legacy_prevent_release;
00240 
00241 /**
00242  * Initialise USB legacy support
00243  *
00244  * @v ehci              EHCI device
00245  * @v pci               PCI device
00246  */
00247 static void ehci_legacy_init ( struct ehci_device *ehci,
00248                                struct pci_device *pci ) {
00249         unsigned int legacy;
00250         uint8_t bios;
00251 
00252         /* Locate USB legacy support capability (if present) */
00253         legacy = ehci_extended_capability ( ehci, pci, EHCI_EECP_ID_LEGACY, 0 );
00254         if ( ! legacy ) {
00255                 /* Not an error; capability may not be present */
00256                 DBGC ( ehci, "EHCI %s has no USB legacy support capability\n",
00257                        ehci->name );
00258                 return;
00259         }
00260 
00261         /* Check if legacy USB support is enabled */
00262         pci_read_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_BIOS ), &bios );
00263         if ( ! ( bios & EHCI_USBLEGSUP_BIOS_OWNED ) ) {
00264                 /* Not an error; already owned by OS */
00265                 DBGC ( ehci, "EHCI %s USB legacy support already disabled\n",
00266                        ehci->name );
00267                 return;
00268         }
00269 
00270         /* Record presence of USB legacy support capability */
00271         ehci->legacy = legacy;
00272 }
00273 
00274 /**
00275  * Claim ownership from BIOS
00276  *
00277  * @v ehci              EHCI device
00278  * @v pci               PCI device
00279  */
00280 static void ehci_legacy_claim ( struct ehci_device *ehci,
00281                                 struct pci_device *pci ) {
00282         unsigned int legacy = ehci->legacy;
00283         uint32_t ctlsts;
00284         uint8_t bios;
00285         unsigned int i;
00286 
00287         /* Do nothing unless legacy support capability is present */
00288         if ( ! legacy )
00289                 return;
00290 
00291         /* Dump original SMI usage */
00292         pci_read_config_dword ( pci, ( legacy + EHCI_USBLEGSUP_CTLSTS ),
00293                                 &ctlsts );
00294         if ( ctlsts ) {
00295                 DBGC ( ehci, "EHCI %s BIOS using SMIs: %08x\n",
00296                        ehci->name, ctlsts );
00297         }
00298 
00299         /* Claim ownership */
00300         pci_write_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_OS ),
00301                                 EHCI_USBLEGSUP_OS_OWNED );
00302 
00303         /* Wait for BIOS to release ownership */
00304         for ( i = 0 ; i < EHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
00305 
00306                 /* Check if BIOS has released ownership */
00307                 pci_read_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_BIOS ),
00308                                        &bios );
00309                 if ( ! ( bios & EHCI_USBLEGSUP_BIOS_OWNED ) ) {
00310                         DBGC ( ehci, "EHCI %s claimed ownership from BIOS\n",
00311                                ehci->name );
00312                         pci_read_config_dword ( pci, ( legacy +
00313                                                        EHCI_USBLEGSUP_CTLSTS ),
00314                                                 &ctlsts );
00315                         if ( ctlsts ) {
00316                                 DBGC ( ehci, "EHCI %s warning: BIOS retained "
00317                                        "SMIs: %08x\n", ehci->name, ctlsts );
00318                         }
00319                         return;
00320                 }
00321 
00322                 /* Delay */
00323                 mdelay ( 1 );
00324         }
00325 
00326         /* BIOS did not release ownership.  Claim it forcibly by
00327          * disabling all SMIs.
00328          */
00329         DBGC ( ehci, "EHCI %s could not claim ownership from BIOS: forcibly "
00330                "disabling SMIs\n", ehci->name );
00331         pci_write_config_dword ( pci, ( legacy + EHCI_USBLEGSUP_CTLSTS ), 0 );
00332 }
00333 
00334 /**
00335  * Release ownership back to BIOS
00336  *
00337  * @v ehci              EHCI device
00338  * @v pci               PCI device
00339  */
00340 static void ehci_legacy_release ( struct ehci_device *ehci,
00341                                   struct pci_device *pci ) {
00342         unsigned int legacy = ehci->legacy;
00343         uint32_t ctlsts;
00344 
00345         /* Do nothing unless legacy support capability is present */
00346         if ( ! legacy )
00347                 return;
00348 
00349         /* Do nothing if releasing ownership is prevented */
00350         if ( ehci_legacy_prevent_release ) {
00351                 DBGC ( ehci, "EHCI %s not releasing ownership to BIOS\n",
00352                        ehci->name );
00353                 return;
00354         }
00355 
00356         /* Release ownership */
00357         pci_write_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_OS ), 0 );
00358         DBGC ( ehci, "EHCI %s released ownership to BIOS\n", ehci->name );
00359 
00360         /* Dump restored SMI usage */
00361         pci_read_config_dword ( pci, ( legacy + EHCI_USBLEGSUP_CTLSTS ),
00362                                 &ctlsts );
00363         DBGC ( ehci, "EHCI %s BIOS reclaimed SMIs: %08x\n",
00364                ehci->name, ctlsts );
00365 }
00366 
00367 /******************************************************************************
00368  *
00369  * Companion controllers
00370  *
00371  ******************************************************************************
00372  */
00373 
00374 /**
00375  * Poll child companion controllers
00376  *
00377  * @v ehci              EHCI device
00378  */
00379 static void ehci_poll_companions ( struct ehci_device *ehci ) {
00380         struct usb_bus *bus;
00381         struct device_description *desc;
00382 
00383         /* Poll any USB buses belonging to child companion controllers */
00384         for_each_usb_bus ( bus ) {
00385 
00386                 /* Get underlying devices description */
00387                 desc = &bus->dev->desc;
00388 
00389                 /* Skip buses that are not PCI devices */
00390                 if ( desc->bus_type != BUS_TYPE_PCI )
00391                         continue;
00392 
00393                 /* Skip buses that are not part of the same PCI device */
00394                 if ( PCI_FIRST_FUNC ( desc->location ) !=
00395                      PCI_FIRST_FUNC ( ehci->bus->dev->desc.location ) )
00396                         continue;
00397 
00398                 /* Skip buses that are not UHCI or OHCI PCI devices */
00399                 if ( ( desc->class != PCI_CLASS ( PCI_CLASS_SERIAL,
00400                                                   PCI_CLASS_SERIAL_USB,
00401                                                   PCI_CLASS_SERIAL_USB_UHCI ))&&
00402                      ( desc->class != PCI_CLASS ( PCI_CLASS_SERIAL,
00403                                                   PCI_CLASS_SERIAL_USB,
00404                                                   PCI_CLASS_SERIAL_USB_OHCI ) ))
00405                         continue;
00406 
00407                 /* Poll child companion controller bus */
00408                 DBGC2 ( ehci, "EHCI %s polling companion %s\n",
00409                         ehci->name, bus->name );
00410                 usb_poll ( bus );
00411         }
00412 }
00413 
00414 /**
00415  * Locate EHCI companion controller
00416  *
00417  * @v pci               PCI device
00418  * @ret busdevfn        EHCI companion controller bus:dev.fn (if any)
00419  */
00420 unsigned int ehci_companion ( struct pci_device *pci ) {
00421         struct pci_device tmp;
00422         unsigned int busdevfn;
00423         int rc;
00424 
00425         /* Look for an EHCI function on the same PCI device */
00426         busdevfn = pci->busdevfn;
00427         while ( ++busdevfn <= PCI_LAST_FUNC ( pci->busdevfn ) ) {
00428                 pci_init ( &tmp, busdevfn );
00429                 if ( ( rc = pci_read_config ( &tmp ) ) != 0 )
00430                         continue;
00431                 if ( tmp.class == PCI_CLASS ( PCI_CLASS_SERIAL,
00432                                               PCI_CLASS_SERIAL_USB,
00433                                               PCI_CLASS_SERIAL_USB_EHCI ) )
00434                         return busdevfn;
00435         }
00436 
00437         return 0;
00438 }
00439 
00440 /******************************************************************************
00441  *
00442  * Run / stop / reset
00443  *
00444  ******************************************************************************
00445  */
00446 
00447 /**
00448  * Start EHCI device
00449  *
00450  * @v ehci              EHCI device
00451  */
00452 static void ehci_run ( struct ehci_device *ehci ) {
00453         uint32_t usbcmd;
00454 
00455         /* Set run/stop bit */
00456         usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
00457         usbcmd &= ~EHCI_USBCMD_FLSIZE_MASK;
00458         usbcmd |= ( EHCI_USBCMD_RUN | EHCI_USBCMD_FLSIZE ( ehci->flsize ) |
00459                     EHCI_USBCMD_PERIODIC | EHCI_USBCMD_ASYNC );
00460         writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
00461 }
00462 
00463 /**
00464  * Stop EHCI device
00465  *
00466  * @v ehci              EHCI device
00467  * @ret rc              Return status code
00468  */
00469 static int ehci_stop ( struct ehci_device *ehci ) {
00470         uint32_t usbcmd;
00471         uint32_t usbsts;
00472         unsigned int i;
00473 
00474         /* Clear run/stop bit */
00475         usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
00476         usbcmd &= ~( EHCI_USBCMD_RUN | EHCI_USBCMD_PERIODIC |
00477                      EHCI_USBCMD_ASYNC );
00478         writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
00479 
00480         /* Wait for device to stop */
00481         for ( i = 0 ; i < EHCI_STOP_MAX_WAIT_MS ; i++ ) {
00482 
00483                 /* Check if device is stopped */
00484                 usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
00485                 if ( usbsts & EHCI_USBSTS_HCH )
00486                         return 0;
00487 
00488                 /* Delay */
00489                 mdelay ( 1 );
00490         }
00491 
00492         DBGC ( ehci, "EHCI %s timed out waiting for stop\n", ehci->name );
00493         return -ETIMEDOUT;
00494 }
00495 
00496 /**
00497  * Reset EHCI device
00498  *
00499  * @v ehci              EHCI device
00500  * @ret rc              Return status code
00501  */
00502 static int ehci_reset ( struct ehci_device *ehci ) {
00503         uint32_t usbcmd;
00504         unsigned int i;
00505         int rc;
00506 
00507         /* The EHCI specification states that resetting a running
00508          * device may result in undefined behaviour, so try stopping
00509          * it first.
00510          */
00511         if ( ( rc = ehci_stop ( ehci ) ) != 0 ) {
00512                 /* Ignore errors and attempt to reset the device anyway */
00513         }
00514 
00515         /* Reset device */
00516         writel ( EHCI_USBCMD_HCRST, ehci->op + EHCI_OP_USBCMD );
00517 
00518         /* Wait for reset to complete */
00519         for ( i = 0 ; i < EHCI_RESET_MAX_WAIT_MS ; i++ ) {
00520 
00521                 /* Check if reset is complete */
00522                 usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
00523                 if ( ! ( usbcmd & EHCI_USBCMD_HCRST ) )
00524                         return 0;
00525 
00526                 /* Delay */
00527                 mdelay ( 1 );
00528         }
00529 
00530         DBGC ( ehci, "EHCI %s timed out waiting for reset\n", ehci->name );
00531         return -ETIMEDOUT;
00532 }
00533 
00534 /******************************************************************************
00535  *
00536  * Transfer descriptor rings
00537  *
00538  ******************************************************************************
00539  */
00540 
00541 /**
00542  * Allocate transfer descriptor ring
00543  *
00544  * @v ehci              EHCI device
00545  * @v ring              Transfer descriptor ring
00546  * @ret rc              Return status code
00547  */
00548 static int ehci_ring_alloc ( struct ehci_device *ehci,
00549                              struct ehci_ring *ring ) {
00550         struct ehci_transfer_descriptor *desc;
00551         struct ehci_transfer_descriptor *next;
00552         unsigned int i;
00553         size_t len;
00554         uint32_t link;
00555         int rc;
00556 
00557         /* Initialise structure */
00558         memset ( ring, 0, sizeof ( *ring ) );
00559 
00560         /* Allocate I/O buffers */
00561         ring->iobuf = zalloc ( EHCI_RING_COUNT * sizeof ( ring->iobuf[0] ) );
00562         if ( ! ring->iobuf ) {
00563                 rc = -ENOMEM;
00564                 goto err_alloc_iobuf;
00565         }
00566 
00567         /* Allocate queue head */
00568         ring->head = malloc_dma ( sizeof ( *ring->head ),
00569                                   ehci_align ( sizeof ( *ring->head ) ) );
00570         if ( ! ring->head ) {
00571                 rc = -ENOMEM;
00572                 goto err_alloc_queue;
00573         }
00574         if ( ( rc = ehci_ctrl_reachable ( ehci, ring->head ) ) != 0 ) {
00575                 DBGC ( ehci, "EHCI %s queue head unreachable\n", ehci->name );
00576                 goto err_unreachable_queue;
00577         }
00578         memset ( ring->head, 0, sizeof ( *ring->head ) );
00579 
00580         /* Allocate transfer descriptors */
00581         len = ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) );
00582         ring->desc = malloc_dma ( len, sizeof ( ring->desc[0] ) );
00583         if ( ! ring->desc ) {
00584                 rc = -ENOMEM;
00585                 goto err_alloc_desc;
00586         }
00587         memset ( ring->desc, 0, len );
00588 
00589         /* Initialise transfer descriptors */
00590         for ( i = 0 ; i < EHCI_RING_COUNT ; i++ ) {
00591                 desc = &ring->desc[i];
00592                 if ( ( rc = ehci_ctrl_reachable ( ehci, desc ) ) != 0 ) {
00593                         DBGC ( ehci, "EHCI %s descriptor unreachable\n",
00594                                ehci->name );
00595                         goto err_unreachable_desc;
00596                 }
00597                 next = &ring->desc[ ( i + 1 ) % EHCI_RING_COUNT ];
00598                 link = virt_to_phys ( next );
00599                 desc->next = cpu_to_le32 ( link );
00600                 desc->alt = cpu_to_le32 ( link );
00601         }
00602 
00603         /* Initialise queue head */
00604         link = virt_to_phys ( &ring->desc[0] );
00605         ring->head->cache.next = cpu_to_le32 ( link );
00606 
00607         return 0;
00608 
00609  err_unreachable_desc:
00610         free_dma ( ring->desc, len );
00611  err_alloc_desc:
00612  err_unreachable_queue:
00613         free_dma ( ring->head, sizeof ( *ring->head ) );
00614  err_alloc_queue:
00615         free ( ring->iobuf );
00616  err_alloc_iobuf:
00617         return rc;
00618 }
00619 
00620 /**
00621  * Free transfer descriptor ring
00622  *
00623  * @v ring              Transfer descriptor ring
00624  */
00625 static void ehci_ring_free ( struct ehci_ring *ring ) {
00626         unsigned int i;
00627 
00628         /* Sanity checks */
00629         assert ( ehci_ring_fill ( ring ) == 0 );
00630         for ( i = 0 ; i < EHCI_RING_COUNT ; i++ )
00631                 assert ( ring->iobuf[i] == NULL );
00632 
00633         /* Free transfer descriptors */
00634         free_dma ( ring->desc, ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) ) );
00635 
00636         /* Free queue head */
00637         free_dma ( ring->head, sizeof ( *ring->head ) );
00638 
00639         /* Free I/O buffers */
00640         free ( ring->iobuf );
00641 }
00642 
00643 /**
00644  * Enqueue transfer descriptors
00645  *
00646  * @v ehci              EHCI device
00647  * @v ring              Transfer descriptor ring
00648  * @v iobuf             I/O buffer
00649  * @v xfers             Transfers
00650  * @v count             Number of transfers
00651  * @ret rc              Return status code
00652  */
00653 static int ehci_enqueue ( struct ehci_device *ehci, struct ehci_ring *ring,
00654                           struct io_buffer *iobuf,
00655                           const struct ehci_transfer *xfer,
00656                           unsigned int count ) {
00657         struct ehci_transfer_descriptor *desc;
00658         physaddr_t phys;
00659         void *data;
00660         size_t len;
00661         size_t offset;
00662         size_t frag_len;
00663         unsigned int toggle;
00664         unsigned int index;
00665         unsigned int i;
00666 
00667         /* Sanity check */
00668         assert ( iobuf != NULL );
00669         assert ( count > 0 );
00670 
00671         /* Fail if ring does not have sufficient space */
00672         if ( ehci_ring_remaining ( ring ) < count )
00673                 return -ENOBUFS;
00674 
00675         /* Fail if any portion is unreachable */
00676         for ( i = 0 ; i < count ; i++ ) {
00677                 if ( ! xfer[i].len )
00678                         continue;
00679                 phys = ( virt_to_phys ( xfer[i].data ) + xfer[i].len - 1 );
00680                 if ( ( phys > 0xffffffffUL ) && ( ! ehci->addr64 ) )
00681                         return -ENOTSUP;
00682         }
00683 
00684         /* Enqueue each transfer, recording the I/O buffer with the last */
00685         for ( ; count ; ring->prod++, xfer++ ) {
00686 
00687                 /* Populate descriptor header */
00688                 index = ( ring->prod % EHCI_RING_COUNT );
00689                 desc = &ring->desc[index];
00690                 toggle = ( xfer->flags & EHCI_FL_TOGGLE );
00691                 assert ( xfer->len <= EHCI_LEN_MASK );
00692                 assert ( EHCI_FL_TOGGLE == EHCI_LEN_TOGGLE );
00693                 desc->len = cpu_to_le16 ( xfer->len | toggle );
00694                 desc->flags = ( xfer->flags | EHCI_FL_CERR_MAX );
00695 
00696                 /* Populate buffer pointers */
00697                 data = xfer->data;
00698                 len = xfer->len;
00699                 for ( i = 0 ; len ; i++ ) {
00700 
00701                         /* Calculate length of this fragment */
00702                         phys = virt_to_phys ( data );
00703                         offset = ( phys & ( EHCI_PAGE_ALIGN - 1 ) );
00704                         frag_len = ( EHCI_PAGE_ALIGN - offset );
00705                         if ( frag_len > len )
00706                                 frag_len = len;
00707 
00708                         /* Sanity checks */
00709                         assert ( ( i == 0 ) || ( offset == 0 ) );
00710                         assert ( i < ( sizeof ( desc->low ) /
00711                                        sizeof ( desc->low[0] ) ) );
00712 
00713                         /* Populate buffer pointer */
00714                         desc->low[i] = cpu_to_le32 ( phys );
00715                         if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) {
00716                                 desc->high[i] =
00717                                         cpu_to_le32 ( ((uint64_t) phys) >> 32 );
00718                         }
00719 
00720                         /* Move to next fragment */
00721                         data += frag_len;
00722                         len -= frag_len;
00723                 }
00724 
00725                 /* Ensure everything is valid before activating descriptor */
00726                 wmb();
00727                 desc->status = EHCI_STATUS_ACTIVE;
00728 
00729                 /* Record I/O buffer against last ring index */
00730                 if ( --count == 0 )
00731                         ring->iobuf[index] = iobuf;
00732         }
00733 
00734         return 0;
00735 }
00736 
00737 /**
00738  * Dequeue a transfer descriptor
00739  *
00740  * @v ring              Transfer descriptor ring
00741  * @ret iobuf           I/O buffer (or NULL)
00742  */
00743 static struct io_buffer * ehci_dequeue ( struct ehci_ring *ring ) {
00744         struct ehci_transfer_descriptor *desc;
00745         struct io_buffer *iobuf;
00746         unsigned int index = ( ring->cons % EHCI_RING_COUNT );
00747 
00748         /* Sanity check */
00749         assert ( ehci_ring_fill ( ring ) > 0 );
00750 
00751         /* Mark descriptor as inactive (and not halted) */
00752         desc = &ring->desc[index];
00753         desc->status = 0;
00754 
00755         /* Retrieve I/O buffer */
00756         iobuf = ring->iobuf[index];
00757         ring->iobuf[index] = NULL;
00758 
00759         /* Update consumer counter */
00760         ring->cons++;
00761 
00762         return iobuf;
00763 }
00764 
00765 /******************************************************************************
00766  *
00767  * Schedule management
00768  *
00769  ******************************************************************************
00770  */
00771 
00772 /**
00773  * Get link value for a queue head
00774  *
00775  * @v queue             Queue head
00776  * @ret link            Link value
00777  */
00778 static inline uint32_t ehci_link_qh ( struct ehci_queue_head *queue ) {
00779 
00780         return ( virt_to_phys ( queue ) | EHCI_LINK_TYPE_QH );
00781 }
00782 
00783 /**
00784  * (Re)build asynchronous schedule
00785  *
00786  * @v ehci              EHCI device
00787  */
00788 static void ehci_async_schedule ( struct ehci_device *ehci ) {
00789         struct ehci_endpoint *endpoint;
00790         struct ehci_queue_head *queue;
00791         uint32_t link;
00792 
00793         /* Build schedule in reverse order of execution.  Provided
00794          * that we only ever add or remove single endpoints, this can
00795          * safely run concurrently with hardware execution of the
00796          * schedule.
00797          */
00798         link = ehci_link_qh ( ehci->head );
00799         list_for_each_entry_reverse ( endpoint, &ehci->async, schedule ) {
00800                 queue = endpoint->ring.head;
00801                 queue->link = cpu_to_le32 ( link );
00802                 wmb();
00803                 link = ehci_link_qh ( queue );
00804         }
00805         ehci->head->link = cpu_to_le32 ( link );
00806         wmb();
00807 }
00808 
00809 /**
00810  * Add endpoint to asynchronous schedule
00811  *
00812  * @v endpoint          Endpoint
00813  */
00814 static void ehci_async_add ( struct ehci_endpoint *endpoint ) {
00815         struct ehci_device *ehci = endpoint->ehci;
00816 
00817         /* Add to end of schedule */
00818         list_add_tail ( &endpoint->schedule, &ehci->async );
00819 
00820         /* Rebuild schedule */
00821         ehci_async_schedule ( ehci );
00822 }
00823 
00824 /**
00825  * Remove endpoint from asynchronous schedule
00826  *
00827  * @v endpoint          Endpoint
00828  * @ret rc              Return status code
00829  */
00830 static int ehci_async_del ( struct ehci_endpoint *endpoint ) {
00831         struct ehci_device *ehci = endpoint->ehci;
00832         uint32_t usbcmd;
00833         uint32_t usbsts;
00834         unsigned int i;
00835 
00836         /* Remove from schedule */
00837         list_check_contains_entry ( endpoint, &ehci->async, schedule );
00838         list_del ( &endpoint->schedule );
00839 
00840         /* Rebuild schedule */
00841         ehci_async_schedule ( ehci );
00842 
00843         /* Request notification when asynchronous schedule advances */
00844         usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
00845         usbcmd |= EHCI_USBCMD_ASYNC_ADVANCE;
00846         writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
00847 
00848         /* Wait for asynchronous schedule to advance */
00849         for ( i = 0 ; i < EHCI_ASYNC_ADVANCE_MAX_WAIT_MS ; i++ ) {
00850 
00851                 /* Check for asynchronous schedule advancing */
00852                 usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
00853                 if ( usbsts & EHCI_USBSTS_ASYNC_ADVANCE ) {
00854                         usbsts &= ~EHCI_USBSTS_CHANGE;
00855                         usbsts |= EHCI_USBSTS_ASYNC_ADVANCE;
00856                         writel ( usbsts, ehci->op + EHCI_OP_USBSTS );
00857                         return 0;
00858                 }
00859 
00860                 /* Delay */
00861                 mdelay ( 1 );
00862         }
00863 
00864         /* Bad things will probably happen now */
00865         DBGC ( ehci, "EHCI %s timed out waiting for asynchronous schedule "
00866                "to advance\n", ehci->name );
00867         return -ETIMEDOUT;
00868 }
00869 
00870 /**
00871  * (Re)build periodic schedule
00872  *
00873  * @v ehci              EHCI device
00874  */
00875 static void ehci_periodic_schedule ( struct ehci_device *ehci ) {
00876         struct ehci_endpoint *endpoint;
00877         struct ehci_queue_head *queue;
00878         uint32_t link;
00879         unsigned int frames;
00880         unsigned int max_interval;
00881         unsigned int i;
00882 
00883         /* Build schedule in reverse order of execution.  Provided
00884          * that we only ever add or remove single endpoints, this can
00885          * safely run concurrently with hardware execution of the
00886          * schedule.
00887          */
00888         DBGCP ( ehci, "EHCI %s periodic schedule: ", ehci->name );
00889         link = EHCI_LINK_TERMINATE;
00890         list_for_each_entry_reverse ( endpoint, &ehci->periodic, schedule ) {
00891                 queue = endpoint->ring.head;
00892                 queue->link = cpu_to_le32 ( link );
00893                 wmb();
00894                 DBGCP ( ehci, "%s%d",
00895                         ( ( link == EHCI_LINK_TERMINATE ) ? "" : "<-" ),
00896                         endpoint->ep->interval );
00897                 link = ehci_link_qh ( queue );
00898         }
00899         DBGCP ( ehci, "\n" );
00900 
00901         /* Populate periodic frame list */
00902         DBGCP ( ehci, "EHCI %s periodic frame list:", ehci->name );
00903         frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
00904         for ( i = 0 ; i < frames ; i++ ) {
00905 
00906                 /* Calculate maximum interval (in microframes) which
00907                  * may appear as part of this frame list.
00908                  */
00909                 if ( i == 0 ) {
00910                         /* Start of list: include all endpoints */
00911                         max_interval = -1U;
00912                 } else {
00913                         /* Calculate highest power-of-two frame interval */
00914                         max_interval = ( 1 << ( ffs ( i ) - 1 ) );
00915                         /* Convert to microframes */
00916                         max_interval <<= 3;
00917                         /* Round up to nearest 2^n-1 */
00918                         max_interval = ( ( max_interval << 1 ) - 1 );
00919                 }
00920 
00921                 /* Find first endpoint in schedule satisfying this
00922                  * maximum interval constraint.
00923                  */
00924                 link = EHCI_LINK_TERMINATE;
00925                 list_for_each_entry ( endpoint, &ehci->periodic, schedule ) {
00926                         if ( endpoint->ep->interval <= max_interval ) {
00927                                 queue = endpoint->ring.head;
00928                                 link = ehci_link_qh ( queue );
00929                                 DBGCP ( ehci, " %d:%d",
00930                                         i, endpoint->ep->interval );
00931                                 break;
00932                         }
00933                 }
00934                 ehci->frame[i].link = cpu_to_le32 ( link );
00935         }
00936         wmb();
00937         DBGCP ( ehci, "\n" );
00938 }
00939 
00940 /**
00941  * Add endpoint to periodic schedule
00942  *
00943  * @v endpoint          Endpoint
00944  */
00945 static void ehci_periodic_add ( struct ehci_endpoint *endpoint ) {
00946         struct ehci_device *ehci = endpoint->ehci;
00947         struct ehci_endpoint *before;
00948         unsigned int interval = endpoint->ep->interval;
00949 
00950         /* Find first endpoint with a smaller interval */
00951         list_for_each_entry ( before, &ehci->periodic, schedule ) {
00952                 if ( before->ep->interval < interval )
00953                         break;
00954         }
00955         list_add_tail ( &endpoint->schedule, &before->schedule );
00956 
00957         /* Rebuild schedule */
00958         ehci_periodic_schedule ( ehci );
00959 }
00960 
00961 /**
00962  * Remove endpoint from periodic schedule
00963  *
00964  * @v endpoint          Endpoint
00965  * @ret rc              Return status code
00966  */
00967 static int ehci_periodic_del ( struct ehci_endpoint *endpoint ) {
00968         struct ehci_device *ehci = endpoint->ehci;
00969 
00970         /* Remove from schedule */
00971         list_check_contains_entry ( endpoint, &ehci->periodic, schedule );
00972         list_del ( &endpoint->schedule );
00973 
00974         /* Rebuild schedule */
00975         ehci_periodic_schedule ( ehci );
00976 
00977         /* Delay for a whole USB frame (with a 100% safety margin) */
00978         mdelay ( 2 );
00979 
00980         return 0;
00981 }
00982 
00983 /**
00984  * Add endpoint to appropriate schedule
00985  *
00986  * @v endpoint          Endpoint
00987  */
00988 static void ehci_schedule_add ( struct ehci_endpoint *endpoint ) {
00989         struct usb_endpoint *ep = endpoint->ep;
00990         unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
00991 
00992         if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
00993                 ehci_periodic_add ( endpoint );
00994         } else {
00995                 ehci_async_add ( endpoint );
00996         }
00997 }
00998 
00999 /**
01000  * Remove endpoint from appropriate schedule
01001  *
01002  * @v endpoint          Endpoint
01003  * @ret rc              Return status code
01004  */
01005 static int ehci_schedule_del ( struct ehci_endpoint *endpoint ) {
01006         struct usb_endpoint *ep = endpoint->ep;
01007         unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
01008 
01009         if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
01010                 return ehci_periodic_del ( endpoint );
01011         } else {
01012                 return ehci_async_del ( endpoint );
01013         }
01014 }
01015 
01016 /******************************************************************************
01017  *
01018  * Endpoint operations
01019  *
01020  ******************************************************************************
01021  */
01022 
01023 /**
01024  * Determine endpoint characteristics
01025  *
01026  * @v ep                USB endpoint
01027  * @ret chr             Endpoint characteristics
01028  */
01029 static uint32_t ehci_endpoint_characteristics ( struct usb_endpoint *ep ) {
01030         struct usb_device *usb = ep->usb;
01031         unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
01032         uint32_t chr;
01033 
01034         /* Determine basic characteristics */
01035         chr = ( EHCI_CHR_ADDRESS ( usb->address ) |
01036                 EHCI_CHR_ENDPOINT ( ep->address ) |
01037                 EHCI_CHR_MAX_LEN ( ep->mtu ) );
01038 
01039         /* Control endpoints require manual control of the data toggle */
01040         if ( attr == USB_ENDPOINT_ATTR_CONTROL )
01041                 chr |= EHCI_CHR_TOGGLE;
01042 
01043         /* Determine endpoint speed */
01044         if ( usb->speed == USB_SPEED_HIGH ) {
01045                 chr |= EHCI_CHR_EPS_HIGH;
01046         } else {
01047                 if ( usb->speed == USB_SPEED_FULL ) {
01048                         chr |= EHCI_CHR_EPS_FULL;
01049                 } else {
01050                         chr |= EHCI_CHR_EPS_LOW;
01051                 }
01052                 if ( attr == USB_ENDPOINT_ATTR_CONTROL )
01053                         chr |= EHCI_CHR_CONTROL;
01054         }
01055 
01056         return chr;
01057 }
01058 
01059 /**
01060  * Determine endpoint capabilities
01061  *
01062  * @v ep                USB endpoint
01063  * @ret cap             Endpoint capabilities
01064  */
01065 static uint32_t ehci_endpoint_capabilities ( struct usb_endpoint *ep ) {
01066         struct usb_device *usb = ep->usb;
01067         struct usb_port *tt = usb_transaction_translator ( usb );
01068         unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
01069         uint32_t cap;
01070         unsigned int i;
01071 
01072         /* Determine basic capabilities */
01073         cap = EHCI_CAP_MULT ( ep->burst + 1 );
01074 
01075         /* Determine interrupt schedule mask, if applicable */
01076         if ( ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) &&
01077              ( ( ep->interval != 0 ) /* avoid infinite loop */ ) ) {
01078                 for ( i = 0 ; i < 8 /* microframes per frame */ ;
01079                       i += ep->interval ) {
01080                         cap |= EHCI_CAP_INTR_SCHED ( i );
01081                 }
01082         }
01083 
01084         /* Set transaction translator hub address and port, if applicable */
01085         if ( tt ) {
01086                 assert ( tt->hub->usb );
01087                 cap |= ( EHCI_CAP_TT_HUB ( tt->hub->usb->address ) |
01088                          EHCI_CAP_TT_PORT ( tt->address ) );
01089                 if ( attr == USB_ENDPOINT_ATTR_INTERRUPT )
01090                         cap |= EHCI_CAP_SPLIT_SCHED_DEFAULT;
01091         }
01092 
01093         return cap;
01094 }
01095 
01096 /**
01097  * Update endpoint characteristics and capabilities
01098  *
01099  * @v ep                USB endpoint
01100  */
01101 static void ehci_endpoint_update ( struct usb_endpoint *ep ) {
01102         struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
01103         struct ehci_queue_head *head;
01104 
01105         /* Update queue characteristics and capabilities */
01106         head = endpoint->ring.head;
01107         head->chr = cpu_to_le32 ( ehci_endpoint_characteristics ( ep ) );
01108         head->cap = cpu_to_le32 ( ehci_endpoint_capabilities ( ep ) );
01109 }
01110 
01111 /**
01112  * Open endpoint
01113  *
01114  * @v ep                USB endpoint
01115  * @ret rc              Return status code
01116  */
01117 static int ehci_endpoint_open ( struct usb_endpoint *ep ) {
01118         struct usb_device *usb = ep->usb;
01119         struct ehci_device *ehci = usb_get_hostdata ( usb );
01120         struct ehci_endpoint *endpoint;
01121         int rc;
01122 
01123         /* Allocate and initialise structure */
01124         endpoint = zalloc ( sizeof ( *endpoint ) );
01125         if ( ! endpoint ) {
01126                 rc = -ENOMEM;
01127                 goto err_alloc;
01128         }
01129         endpoint->ehci = ehci;
01130         endpoint->ep = ep;
01131         usb_endpoint_set_hostdata ( ep, endpoint );
01132 
01133         /* Initialise descriptor ring */
01134         if ( ( rc = ehci_ring_alloc ( ehci, &endpoint->ring ) ) != 0 )
01135                 goto err_ring_alloc;
01136 
01137         /* Update queue characteristics and capabilities */
01138         ehci_endpoint_update ( ep );
01139 
01140         /* Add to list of endpoints */
01141         list_add_tail ( &endpoint->list, &ehci->endpoints );
01142 
01143         /* Add to schedule */
01144         ehci_schedule_add ( endpoint );
01145 
01146         return 0;
01147 
01148         ehci_ring_free ( &endpoint->ring );
01149  err_ring_alloc:
01150         free ( endpoint );
01151  err_alloc:
01152         return rc;
01153 }
01154 
01155 /**
01156  * Close endpoint
01157  *
01158  * @v ep                USB endpoint
01159  */
01160 static void ehci_endpoint_close ( struct usb_endpoint *ep ) {
01161         struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
01162         struct ehci_device *ehci = endpoint->ehci;
01163         struct usb_device *usb = ep->usb;
01164         struct io_buffer *iobuf;
01165         int rc;
01166 
01167         /* Remove from schedule */
01168         if ( ( rc = ehci_schedule_del ( endpoint ) ) != 0 ) {
01169                 /* No way to prevent hardware from continuing to
01170                  * access the memory, so leak it.
01171                  */
01172                 DBGC ( ehci, "EHCI %s %s could not unschedule: %s\n",
01173                        usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
01174                 return;
01175         }
01176 
01177         /* Cancel any incomplete transfers */
01178         while ( ehci_ring_fill ( &endpoint->ring ) ) {
01179                 iobuf = ehci_dequeue ( &endpoint->ring );
01180                 if ( iobuf )
01181                         usb_complete_err ( ep, iobuf, -ECANCELED );
01182         }
01183 
01184         /* Remove from list of endpoints */
01185         list_del ( &endpoint->list );
01186 
01187         /* Free descriptor ring */
01188         ehci_ring_free ( &endpoint->ring );
01189 
01190         /* Free endpoint */
01191         free ( endpoint );
01192 }
01193 
01194 /**
01195  * Reset endpoint
01196  *
01197  * @v ep                USB endpoint
01198  * @ret rc              Return status code
01199  */
01200 static int ehci_endpoint_reset ( struct usb_endpoint *ep ) {
01201         struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
01202         struct ehci_ring *ring = &endpoint->ring;
01203         struct ehci_transfer_descriptor *cache = &ring->head->cache;
01204         uint32_t link;
01205 
01206         /* Sanity checks */
01207         assert ( ! ( cache->status & EHCI_STATUS_ACTIVE ) );
01208         assert ( cache->status & EHCI_STATUS_HALTED );
01209 
01210         /* Reset residual count */
01211         ring->residual = 0;
01212 
01213         /* Reset data toggle */
01214         cache->len = 0;
01215 
01216         /* Prepare to restart at next unconsumed descriptor */
01217         link = virt_to_phys ( &ring->desc[ ring->cons % EHCI_RING_COUNT ] );
01218         cache->next = cpu_to_le32 ( link );
01219 
01220         /* Restart ring */
01221         wmb();
01222         cache->status = 0;
01223 
01224         return 0;
01225 }
01226 
01227 /**
01228  * Update MTU
01229  *
01230  * @v ep                USB endpoint
01231  * @ret rc              Return status code
01232  */
01233 static int ehci_endpoint_mtu ( struct usb_endpoint *ep ) {
01234 
01235         /* Update endpoint characteristics and capabilities */
01236         ehci_endpoint_update ( ep );
01237 
01238         return 0;
01239 }
01240 
01241 /**
01242  * Enqueue message transfer
01243  *
01244  * @v ep                USB endpoint
01245  * @v iobuf             I/O buffer
01246  * @ret rc              Return status code
01247  */
01248 static int ehci_endpoint_message ( struct usb_endpoint *ep,
01249                                    struct io_buffer *iobuf ) {
01250         struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
01251         struct ehci_device *ehci = endpoint->ehci;
01252         struct usb_setup_packet *packet;
01253         unsigned int input;
01254         struct ehci_transfer xfers[3];
01255         struct ehci_transfer *xfer = xfers;
01256         size_t len;
01257         int rc;
01258 
01259         /* Construct setup stage */
01260         assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
01261         packet = iobuf->data;
01262         iob_pull ( iobuf, sizeof ( *packet ) );
01263         xfer->data = packet;
01264         xfer->len = sizeof ( *packet );
01265         xfer->flags = EHCI_FL_PID_SETUP;
01266         xfer++;
01267 
01268         /* Construct data stage, if applicable */
01269         len = iob_len ( iobuf );
01270         input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
01271         if ( len ) {
01272                 xfer->data = iobuf->data;
01273                 xfer->len = len;
01274                 xfer->flags = ( EHCI_FL_TOGGLE |
01275                                 ( input ? EHCI_FL_PID_IN : EHCI_FL_PID_OUT ) );
01276                 xfer++;
01277         }
01278 
01279         /* Construct status stage */
01280         xfer->data = NULL;
01281         xfer->len = 0;
01282         xfer->flags = ( EHCI_FL_TOGGLE | EHCI_FL_IOC |
01283                         ( ( len && input ) ? EHCI_FL_PID_OUT : EHCI_FL_PID_IN));
01284         xfer++;
01285 
01286         /* Enqueue transfer */
01287         if ( ( rc = ehci_enqueue ( ehci, &endpoint->ring, iobuf, xfers,
01288                                    ( xfer - xfers ) ) ) != 0 )
01289                 return rc;
01290 
01291         return 0;
01292 }
01293 
01294 /**
01295  * Calculate number of transfer descriptors
01296  *
01297  * @v len               Length of data
01298  * @v zlp               Append a zero-length packet
01299  * @ret count           Number of transfer descriptors
01300  */
01301 static unsigned int ehci_endpoint_count ( size_t len, int zlp ) {
01302         unsigned int count;
01303 
01304         /* Split into 16kB transfers.  A single transfer can handle up
01305          * to 20kB if it happens to be page-aligned, or up to 16kB
01306          * with arbitrary alignment.  We simplify the code by assuming
01307          * that we can fit only 16kB into each transfer.
01308          */
01309         count = ( ( len + EHCI_MTU - 1 ) / EHCI_MTU );
01310 
01311         /* Append a zero-length transfer if applicable */
01312         if ( zlp || ( count == 0 ) )
01313                 count++;
01314 
01315         return count;
01316 }
01317 
01318 /**
01319  * Enqueue stream transfer
01320  *
01321  * @v ep                USB endpoint
01322  * @v iobuf             I/O buffer
01323  * @v zlp               Append a zero-length packet
01324  * @ret rc              Return status code
01325  */
01326 static int ehci_endpoint_stream ( struct usb_endpoint *ep,
01327                                   struct io_buffer *iobuf, int zlp ) {
01328         struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
01329         struct ehci_device *ehci = endpoint->ehci;
01330         void *data = iobuf->data;
01331         size_t len = iob_len ( iobuf );
01332         unsigned int count = ehci_endpoint_count ( len, zlp );
01333         unsigned int input = ( ep->address & USB_DIR_IN );
01334         unsigned int flags = ( input ? EHCI_FL_PID_IN : EHCI_FL_PID_OUT );
01335         struct ehci_transfer xfers[count];
01336         struct ehci_transfer *xfer = xfers;
01337         size_t xfer_len;
01338         unsigned int i;
01339         int rc;
01340 
01341         /* Create transfers */
01342         for ( i = 0 ; i < count ; i++ ) {
01343 
01344                 /* Calculate transfer length */
01345                 xfer_len = EHCI_MTU;
01346                 if ( xfer_len > len )
01347                         xfer_len = len;
01348 
01349                 /* Create transfer */
01350                 xfer->data = data;
01351                 xfer->len = xfer_len;
01352                 xfer->flags = flags;
01353 
01354                 /* Move to next transfer */
01355                 data += xfer_len;
01356                 len -= xfer_len;
01357                 xfer++;
01358         }
01359         xfer[-1].flags |= EHCI_FL_IOC;
01360 
01361         /* Enqueue transfer */
01362         if ( ( rc = ehci_enqueue ( ehci, &endpoint->ring, iobuf, xfers,
01363                                    count ) ) != 0 )
01364                 return rc;
01365 
01366         return 0;
01367 }
01368 
01369 /**
01370  * Poll for completions
01371  *
01372  * @v endpoint          Endpoint
01373  */
01374 static void ehci_endpoint_poll ( struct ehci_endpoint *endpoint ) {
01375         struct ehci_device *ehci = endpoint->ehci;
01376         struct ehci_ring *ring = &endpoint->ring;
01377         struct ehci_transfer_descriptor *desc;
01378         struct usb_endpoint *ep = endpoint->ep;
01379         struct usb_device *usb = ep->usb;
01380         struct io_buffer *iobuf;
01381         unsigned int index;
01382         unsigned int status;
01383         int rc;
01384 
01385         /* Consume all completed descriptors */
01386         while ( ehci_ring_fill ( &endpoint->ring ) ) {
01387 
01388                 /* Stop if we reach an uncompleted descriptor */
01389                 rmb();
01390                 index = ( ring->cons % EHCI_RING_COUNT );
01391                 desc = &ring->desc[index];
01392                 status = desc->status;
01393                 if ( status & EHCI_STATUS_ACTIVE )
01394                         break;
01395 
01396                 /* Consume this descriptor */
01397                 iobuf = ehci_dequeue ( ring );
01398 
01399                 /* If we have encountered an error, then consume all
01400                  * remaining descriptors in this transaction, report
01401                  * the error to the USB core, and stop further
01402                  * processing.
01403                  */
01404                 if ( status & EHCI_STATUS_HALTED ) {
01405                         rc = -EIO_STATUS ( status );
01406                         DBGC ( ehci, "EHCI %s %s completion %d failed (status "
01407                                "%02x): %s\n", usb->name,
01408                                usb_endpoint_name ( ep ), index, status,
01409                                strerror ( rc ) );
01410                         while ( ! iobuf )
01411                                 iobuf = ehci_dequeue ( ring );
01412                         usb_complete_err ( endpoint->ep, iobuf, rc );
01413                         return;
01414                 }
01415 
01416                 /* Accumulate residual data count */
01417                 ring->residual += ( le16_to_cpu ( desc->len ) & EHCI_LEN_MASK );
01418 
01419                 /* If this is not the end of a transaction (i.e. has
01420                  * no I/O buffer), then continue to next descriptor.
01421                  */
01422                 if ( ! iobuf )
01423                         continue;
01424 
01425                 /* Update I/O buffer length */
01426                 iob_unput ( iobuf, ring->residual );
01427                 ring->residual = 0;
01428 
01429                 /* Report completion to USB core */
01430                 usb_complete ( endpoint->ep, iobuf );
01431         }
01432 }
01433 
01434 /******************************************************************************
01435  *
01436  * Device operations
01437  *
01438  ******************************************************************************
01439  */
01440 
01441 /**
01442  * Open device
01443  *
01444  * @v usb               USB device
01445  * @ret rc              Return status code
01446  */
01447 static int ehci_device_open ( struct usb_device *usb ) {
01448         struct ehci_device *ehci = usb_bus_get_hostdata ( usb->port->hub->bus );
01449 
01450         usb_set_hostdata ( usb, ehci );
01451         return 0;
01452 }
01453 
01454 /**
01455  * Close device
01456  *
01457  * @v usb               USB device
01458  */
01459 static void ehci_device_close ( struct usb_device *usb ) {
01460         struct ehci_device *ehci = usb_get_hostdata ( usb );
01461         struct usb_bus *bus = ehci->bus;
01462 
01463         /* Free device address, if assigned */
01464         if ( usb->address )
01465                 usb_free_address ( bus, usb->address );
01466 }
01467 
01468 /**
01469  * Assign device address
01470  *
01471  * @v usb               USB device
01472  * @ret rc              Return status code
01473  */
01474 static int ehci_device_address ( struct usb_device *usb ) {
01475         struct ehci_device *ehci = usb_get_hostdata ( usb );
01476         struct usb_bus *bus = ehci->bus;
01477         struct usb_endpoint *ep0 = usb_endpoint ( usb, USB_EP0_ADDRESS );
01478         int address;
01479         int rc;
01480 
01481         /* Sanity checks */
01482         assert ( usb->address == 0 );
01483         assert ( ep0 != NULL );
01484 
01485         /* Allocate device address */
01486         address = usb_alloc_address ( bus );
01487         if ( address < 0 ) {
01488                 rc = address;
01489                 DBGC ( ehci, "EHCI %s could not allocate address: %s\n",
01490                        usb->name, strerror ( rc ) );
01491                 goto err_alloc_address;
01492         }
01493 
01494         /* Set address */
01495         if ( ( rc = usb_set_address ( usb, address ) ) != 0 )
01496                 goto err_set_address;
01497 
01498         /* Update device address */
01499         usb->address = address;
01500 
01501         /* Update control endpoint characteristics and capabilities */
01502         ehci_endpoint_update ( ep0 );
01503 
01504         return 0;
01505 
01506  err_set_address:
01507         usb_free_address ( bus, address );
01508  err_alloc_address:
01509         return rc;
01510 }
01511 
01512 /******************************************************************************
01513  *
01514  * Hub operations
01515  *
01516  ******************************************************************************
01517  */
01518 
01519 /**
01520  * Open hub
01521  *
01522  * @v hub               USB hub
01523  * @ret rc              Return status code
01524  */
01525 static int ehci_hub_open ( struct usb_hub *hub __unused ) {
01526 
01527         /* Nothing to do */
01528         return 0;
01529 }
01530 
01531 /**
01532  * Close hub
01533  *
01534  * @v hub               USB hub
01535  */
01536 static void ehci_hub_close ( struct usb_hub *hub __unused ) {
01537 
01538         /* Nothing to do */
01539 }
01540 
01541 /******************************************************************************
01542  *
01543  * Root hub operations
01544  *
01545  ******************************************************************************
01546  */
01547 
01548 /**
01549  * Open root hub
01550  *
01551  * @v hub               USB hub
01552  * @ret rc              Return status code
01553  */
01554 static int ehci_root_open ( struct usb_hub *hub ) {
01555         struct usb_bus *bus = hub->bus;
01556         struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
01557         uint32_t portsc;
01558         unsigned int i;
01559 
01560         /* Route all ports to EHCI controller */
01561         writel ( EHCI_CONFIGFLAG_CF, ehci->op + EHCI_OP_CONFIGFLAG );
01562 
01563         /* Enable power to all ports */
01564         for ( i = 1 ; i <= ehci->ports ; i++ ) {
01565                 portsc = readl ( ehci->op + EHCI_OP_PORTSC ( i ) );
01566                 portsc &= ~EHCI_PORTSC_CHANGE;
01567                 portsc |= EHCI_PORTSC_PP;
01568                 writel ( portsc, ehci->op + EHCI_OP_PORTSC ( i ) );
01569         }
01570 
01571         /* Wait 20ms after potentially enabling power to a port */
01572         mdelay ( EHCI_PORT_POWER_DELAY_MS );
01573 
01574         /* Record hub driver private data */
01575         usb_hub_set_drvdata ( hub, ehci );
01576 
01577         return 0;
01578 }
01579 
01580 /**
01581  * Close root hub
01582  *
01583  * @v hub               USB hub
01584  */
01585 static void ehci_root_close ( struct usb_hub *hub ) {
01586         struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
01587 
01588         /* Route all ports back to companion controllers */
01589         writel ( 0, ehci->op + EHCI_OP_CONFIGFLAG );
01590 
01591         /* Clear hub driver private data */
01592         usb_hub_set_drvdata ( hub, NULL );
01593 }
01594 
01595 /**
01596  * Enable port
01597  *
01598  * @v hub               USB hub
01599  * @v port              USB port
01600  * @ret rc              Return status code
01601  */
01602 static int ehci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
01603         struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
01604         uint32_t portsc;
01605         unsigned int line;
01606         unsigned int i;
01607 
01608         /* Check for a low-speed device */
01609         portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
01610         line = EHCI_PORTSC_LINE_STATUS ( portsc );
01611         if ( line == EHCI_PORTSC_LINE_STATUS_LOW ) {
01612                 DBGC ( ehci, "EHCI %s-%d detected low-speed device: "
01613                        "disowning\n", ehci->name, port->address );
01614                 goto disown;
01615         }
01616 
01617         /* Reset port */
01618         portsc &= ~( EHCI_PORTSC_PED | EHCI_PORTSC_CHANGE );
01619         portsc |= EHCI_PORTSC_PR;
01620         writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
01621         mdelay ( USB_RESET_DELAY_MS );
01622         portsc &= ~EHCI_PORTSC_PR;
01623         writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
01624 
01625         /* Wait for reset to complete */
01626         for ( i = 0 ; i < EHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
01627 
01628                 /* Check port status */
01629                 portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
01630                 if ( ! ( portsc & EHCI_PORTSC_PR ) ) {
01631                         if ( portsc & EHCI_PORTSC_PED )
01632                                 return 0;
01633                         DBGC ( ehci, "EHCI %s-%d not enabled after reset: "
01634                                "disowning\n", ehci->name, port->address );
01635                         goto disown;
01636                 }
01637 
01638                 /* Delay */
01639                 mdelay ( 1 );
01640         }
01641 
01642         DBGC ( ehci, "EHCI %s-%d timed out waiting for port to reset\n",
01643                ehci->name, port->address );
01644         return -ETIMEDOUT;
01645 
01646  disown:
01647         /* Disown port */
01648         portsc &= ~EHCI_PORTSC_CHANGE;
01649         portsc |= EHCI_PORTSC_OWNER;
01650         writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
01651 
01652         /* Delay to allow child companion controllers to settle */
01653         mdelay ( EHCI_DISOWN_DELAY_MS );
01654 
01655         /* Poll child companion controllers */
01656         ehci_poll_companions ( ehci );
01657 
01658         return -ENODEV;
01659 }
01660 
01661 /**
01662  * Disable port
01663  *
01664  * @v hub               USB hub
01665  * @v port              USB port
01666  * @ret rc              Return status code
01667  */
01668 static int ehci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
01669         struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
01670         uint32_t portsc;
01671 
01672         /* Disable port */
01673         portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
01674         portsc &= ~( EHCI_PORTSC_PED | EHCI_PORTSC_CHANGE );
01675         writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
01676 
01677         return 0;
01678 }
01679 
01680 /**
01681  * Update root hub port speed
01682  *
01683  * @v hub               USB hub
01684  * @v port              USB port
01685  * @ret rc              Return status code
01686  */
01687 static int ehci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
01688         struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
01689         uint32_t portsc;
01690         unsigned int speed;
01691         unsigned int line;
01692         int ccs;
01693         int csc;
01694         int ped;
01695 
01696         /* Read port status */
01697         portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
01698         DBGC2 ( ehci, "EHCI %s-%d status is %08x\n",
01699                 ehci->name, port->address, portsc );
01700         ccs = ( portsc & EHCI_PORTSC_CCS );
01701         csc = ( portsc & EHCI_PORTSC_CSC );
01702         ped = ( portsc & EHCI_PORTSC_PED );
01703         line = EHCI_PORTSC_LINE_STATUS ( portsc );
01704 
01705         /* Record disconnections and clear changes */
01706         port->disconnected |= csc;
01707         writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
01708 
01709         /* Determine port speed */
01710         if ( ! ccs ) {
01711                 /* Port not connected */
01712                 speed = USB_SPEED_NONE;
01713         } else if ( line == EHCI_PORTSC_LINE_STATUS_LOW ) {
01714                 /* Detected as low-speed */
01715                 speed = USB_SPEED_LOW;
01716         } else if ( ped ) {
01717                 /* Port already enabled: must be high-speed */
01718                 speed = USB_SPEED_HIGH;
01719         } else {
01720                 /* Not low-speed and not yet enabled.  Could be either
01721                  * full-speed or high-speed; we can't yet tell.
01722                  */
01723                 speed = USB_SPEED_FULL;
01724         }
01725         port->speed = speed;
01726         return 0;
01727 }
01728 
01729 /**
01730  * Clear transaction translator buffer
01731  *
01732  * @v hub               USB hub
01733  * @v port              USB port
01734  * @v ep                USB endpoint
01735  * @ret rc              Return status code
01736  */
01737 static int ehci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
01738                                 struct usb_endpoint *ep ) {
01739         struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
01740 
01741         /* Should never be called; this is a root hub */
01742         DBGC ( ehci, "EHCI %s-%d nonsensical CLEAR_TT for %s %s\n", ehci->name,
01743                port->address, ep->usb->name, usb_endpoint_name ( ep ) );
01744 
01745         return -ENOTSUP;
01746 }
01747 
01748 /**
01749  * Poll for port status changes
01750  *
01751  * @v hub               USB hub
01752  * @v port              USB port
01753  */
01754 static void ehci_root_poll ( struct usb_hub *hub, struct usb_port *port ) {
01755         struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
01756         uint32_t portsc;
01757         uint32_t change;
01758 
01759         /* Do nothing unless something has changed */
01760         portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
01761         change = ( portsc & EHCI_PORTSC_CHANGE );
01762         if ( ! change )
01763                 return;
01764 
01765         /* Record disconnections and clear changes */
01766         port->disconnected |= ( portsc & EHCI_PORTSC_CSC );
01767         writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
01768 
01769         /* Report port status change */
01770         usb_port_changed ( port );
01771 }
01772 
01773 /******************************************************************************
01774  *
01775  * Bus operations
01776  *
01777  ******************************************************************************
01778  */
01779 
01780 /**
01781  * Open USB bus
01782  *
01783  * @v bus               USB bus
01784  * @ret rc              Return status code
01785  */
01786 static int ehci_bus_open ( struct usb_bus *bus ) {
01787         struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
01788         unsigned int frames;
01789         size_t len;
01790         int rc;
01791 
01792         /* Sanity checks */
01793         assert ( list_empty ( &ehci->async ) );
01794         assert ( list_empty ( &ehci->periodic ) );
01795 
01796         /* Allocate and initialise asynchronous queue head */
01797         ehci->head = malloc_dma ( sizeof ( *ehci->head ),
01798                                   ehci_align ( sizeof ( *ehci->head ) ) );
01799         if ( ! ehci->head ) {
01800                 rc = -ENOMEM;
01801                 goto err_alloc_head;
01802         }
01803         memset ( ehci->head, 0, sizeof ( *ehci->head ) );
01804         ehci->head->chr = cpu_to_le32 ( EHCI_CHR_HEAD );
01805         ehci->head->cache.next = cpu_to_le32 ( EHCI_LINK_TERMINATE );
01806         ehci->head->cache.status = EHCI_STATUS_HALTED;
01807         ehci_async_schedule ( ehci );
01808         writel ( virt_to_phys ( ehci->head ),
01809                  ehci->op + EHCI_OP_ASYNCLISTADDR );
01810 
01811         /* Use async queue head to determine control data structure segment */
01812         ehci->ctrldssegment =
01813                 ( ( ( uint64_t ) virt_to_phys ( ehci->head ) ) >> 32 );
01814         if ( ehci->addr64 ) {
01815                 writel ( ehci->ctrldssegment, ehci->op + EHCI_OP_CTRLDSSEGMENT);
01816         } else if ( ehci->ctrldssegment ) {
01817                 DBGC ( ehci, "EHCI %s CTRLDSSEGMENT not supported\n",
01818                        ehci->name );
01819                 rc = -ENOTSUP;
01820                 goto err_ctrldssegment;
01821         }
01822 
01823         /* Allocate periodic frame list */
01824         frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
01825         len = ( frames * sizeof ( ehci->frame[0] ) );
01826         ehci->frame = malloc_dma ( len, EHCI_PAGE_ALIGN );
01827         if ( ! ehci->frame ) {
01828                 rc = -ENOMEM;
01829                 goto err_alloc_frame;
01830         }
01831         if ( ( rc = ehci_ctrl_reachable ( ehci, ehci->frame ) ) != 0 ) {
01832                 DBGC ( ehci, "EHCI %s frame list unreachable\n", ehci->name );
01833                 goto err_unreachable_frame;
01834         }
01835         ehci_periodic_schedule ( ehci );
01836         writel ( virt_to_phys ( ehci->frame ),
01837                  ehci->op + EHCI_OP_PERIODICLISTBASE );
01838 
01839         /* Start controller */
01840         ehci_run ( ehci );
01841 
01842         return 0;
01843 
01844         ehci_stop ( ehci );
01845  err_unreachable_frame:
01846         free_dma ( ehci->frame, len );
01847  err_alloc_frame:
01848  err_ctrldssegment:
01849         free_dma ( ehci->head, sizeof ( *ehci->head ) );
01850  err_alloc_head:
01851         return rc;
01852 }
01853 
01854 /**
01855  * Close USB bus
01856  *
01857  * @v bus               USB bus
01858  */
01859 static void ehci_bus_close ( struct usb_bus *bus ) {
01860         struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
01861         unsigned int frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
01862 
01863         /* Sanity checks */
01864         assert ( list_empty ( &ehci->async ) );
01865         assert ( list_empty ( &ehci->periodic ) );
01866 
01867         /* Stop controller */
01868         ehci_stop ( ehci );
01869 
01870         /* Free periodic frame list */
01871         free_dma ( ehci->frame, ( frames * sizeof ( ehci->frame[0] ) ) );
01872 
01873         /* Free asynchronous schedule */
01874         free_dma ( ehci->head, sizeof ( *ehci->head ) );
01875 }
01876 
01877 /**
01878  * Poll USB bus
01879  *
01880  * @v bus               USB bus
01881  */
01882 static void ehci_bus_poll ( struct usb_bus *bus ) {
01883         struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
01884         struct usb_hub *hub = bus->hub;
01885         struct ehci_endpoint *endpoint;
01886         unsigned int i;
01887         uint32_t usbsts;
01888         uint32_t change;
01889 
01890         /* Do nothing unless something has changed */
01891         usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
01892         assert ( usbsts & EHCI_USBSTS_ASYNC );
01893         assert ( usbsts & EHCI_USBSTS_PERIODIC );
01894         assert ( ! ( usbsts & EHCI_USBSTS_HCH ) );
01895         change = ( usbsts & EHCI_USBSTS_CHANGE );
01896         if ( ! change )
01897                 return;
01898 
01899         /* Acknowledge changes */
01900         writel ( usbsts, ehci->op + EHCI_OP_USBSTS );
01901 
01902         /* Process completions, if applicable */
01903         if ( change & ( EHCI_USBSTS_USBINT | EHCI_USBSTS_USBERRINT ) ) {
01904 
01905                 /* Iterate over all endpoints looking for completed
01906                  * descriptors.  We trust that completion handlers are
01907                  * minimal and will not do anything that could
01908                  * plausibly affect the endpoint list itself.
01909                  */
01910                 list_for_each_entry ( endpoint, &ehci->endpoints, list )
01911                         ehci_endpoint_poll ( endpoint );
01912         }
01913 
01914         /* Process port status changes, if applicable */
01915         if ( change & EHCI_USBSTS_PORT ) {
01916 
01917                 /* Iterate over all ports looking for status changes */
01918                 for ( i = 1 ; i <= ehci->ports ; i++ )
01919                         ehci_root_poll ( hub, usb_port ( hub, i ) );
01920         }
01921 
01922         /* Report fatal errors */
01923         if ( change & EHCI_USBSTS_SYSERR )
01924                 DBGC ( ehci, "EHCI %s host system error\n", ehci->name );
01925 }
01926 
01927 /******************************************************************************
01928  *
01929  * PCI interface
01930  *
01931  ******************************************************************************
01932  */
01933 
01934 /** USB host controller operations */
01935 static struct usb_host_operations ehci_operations = {
01936         .endpoint = {
01937                 .open = ehci_endpoint_open,
01938                 .close = ehci_endpoint_close,
01939                 .reset = ehci_endpoint_reset,
01940                 .mtu = ehci_endpoint_mtu,
01941                 .message = ehci_endpoint_message,
01942                 .stream = ehci_endpoint_stream,
01943         },
01944         .device = {
01945                 .open = ehci_device_open,
01946                 .close = ehci_device_close,
01947                 .address = ehci_device_address,
01948         },
01949         .bus = {
01950                 .open = ehci_bus_open,
01951                 .close = ehci_bus_close,
01952                 .poll = ehci_bus_poll,
01953         },
01954         .hub = {
01955                 .open = ehci_hub_open,
01956                 .close = ehci_hub_close,
01957         },
01958         .root = {
01959                 .open = ehci_root_open,
01960                 .close = ehci_root_close,
01961                 .enable = ehci_root_enable,
01962                 .disable = ehci_root_disable,
01963                 .speed = ehci_root_speed,
01964                 .clear_tt = ehci_root_clear_tt,
01965         },
01966 };
01967 
01968 /**
01969  * Probe PCI device
01970  *
01971  * @v pci               PCI device
01972  * @ret rc              Return status code
01973  */
01974 static int ehci_probe ( struct pci_device *pci ) {
01975         struct ehci_device *ehci;
01976         struct usb_port *port;
01977         unsigned long bar_start;
01978         size_t bar_size;
01979         unsigned int i;
01980         int rc;
01981 
01982         /* Allocate and initialise structure */
01983         ehci = zalloc ( sizeof ( *ehci ) );
01984         if ( ! ehci ) {
01985                 rc = -ENOMEM;
01986                 goto err_alloc;
01987         }
01988         ehci->name = pci->dev.name;
01989         INIT_LIST_HEAD ( &ehci->endpoints );
01990         INIT_LIST_HEAD ( &ehci->async );
01991         INIT_LIST_HEAD ( &ehci->periodic );
01992 
01993         /* Fix up PCI device */
01994         adjust_pci_device ( pci );
01995 
01996         /* Map registers */
01997         bar_start = pci_bar_start ( pci, EHCI_BAR );
01998         bar_size = pci_bar_size ( pci, EHCI_BAR );
01999         ehci->regs = ioremap ( bar_start, bar_size );
02000         if ( ! ehci->regs ) {
02001                 rc = -ENODEV;
02002                 goto err_ioremap;
02003         }
02004 
02005         /* Initialise EHCI device */
02006         ehci_init ( ehci, ehci->regs );
02007 
02008         /* Initialise USB legacy support and claim ownership */
02009         ehci_legacy_init ( ehci, pci );
02010         ehci_legacy_claim ( ehci, pci );
02011 
02012         /* Reset device */
02013         if ( ( rc = ehci_reset ( ehci ) ) != 0 )
02014                 goto err_reset;
02015 
02016         /* Allocate USB bus */
02017         ehci->bus = alloc_usb_bus ( &pci->dev, ehci->ports, EHCI_MTU,
02018                                     &ehci_operations );
02019         if ( ! ehci->bus ) {
02020                 rc = -ENOMEM;
02021                 goto err_alloc_bus;
02022         }
02023         usb_bus_set_hostdata ( ehci->bus, ehci );
02024         usb_hub_set_drvdata ( ehci->bus->hub, ehci );
02025 
02026         /* Set port protocols */
02027         for ( i = 1 ; i <= ehci->ports ; i++ ) {
02028                 port = usb_port ( ehci->bus->hub, i );
02029                 port->protocol = USB_PROTO_2_0;
02030         }
02031 
02032         /* Register USB bus */
02033         if ( ( rc = register_usb_bus ( ehci->bus ) ) != 0 )
02034                 goto err_register;
02035 
02036         pci_set_drvdata ( pci, ehci );
02037         return 0;
02038 
02039         unregister_usb_bus ( ehci->bus );
02040  err_register:
02041         free_usb_bus ( ehci->bus );
02042  err_alloc_bus:
02043         ehci_reset ( ehci );
02044  err_reset:
02045         ehci_legacy_release ( ehci, pci );
02046         iounmap ( ehci->regs );
02047  err_ioremap:
02048         free ( ehci );
02049  err_alloc:
02050         return rc;
02051 }
02052 
02053 /**
02054  * Remove PCI device
02055  *
02056  * @v pci               PCI device
02057  */
02058 static void ehci_remove ( struct pci_device *pci ) {
02059         struct ehci_device *ehci = pci_get_drvdata ( pci );
02060         struct usb_bus *bus = ehci->bus;
02061 
02062         unregister_usb_bus ( bus );
02063         assert ( list_empty ( &ehci->async ) );
02064         assert ( list_empty ( &ehci->periodic ) );
02065         free_usb_bus ( bus );
02066         ehci_reset ( ehci );
02067         ehci_legacy_release ( ehci, pci );
02068         iounmap ( ehci->regs );
02069         free ( ehci );
02070 }
02071 
02072 /** EHCI PCI device IDs */
02073 static struct pci_device_id ehci_ids[] = {
02074         PCI_ROM ( 0xffff, 0xffff, "ehci", "EHCI", 0 ),
02075 };
02076 
02077 /** EHCI PCI driver */
02078 struct pci_driver ehci_driver __pci_driver = {
02079         .ids = ehci_ids,
02080         .id_count = ( sizeof ( ehci_ids ) / sizeof ( ehci_ids[0] ) ),
02081         .class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
02082                                 PCI_CLASS_SERIAL_USB_EHCI ),
02083         .probe = ehci_probe,
02084         .remove = ehci_remove,
02085 };
02086 
02087 /**
02088  * Prepare for exit
02089  *
02090  * @v booting           System is shutting down for OS boot
02091  */
02092 static void ehci_shutdown ( int booting ) {
02093         /* If we are shutting down to boot an OS, then prevent the
02094          * release of ownership back to BIOS.
02095          */
02096         ehci_legacy_prevent_release = booting;
02097 }
02098 
02099 /** Startup/shutdown function */
02100 struct startup_fn ehci_startup __startup_fn ( STARTUP_LATE ) = {
02101         .shutdown = ehci_shutdown,
02102 };