iPXE
ena.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2018 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or (at your option) any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdint.h>
00027 #include <string.h>
00028 #include <unistd.h>
00029 #include <errno.h>
00030 #include <byteswap.h>
00031 #include <ipxe/netdevice.h>
00032 #include <ipxe/ethernet.h>
00033 #include <ipxe/if_ether.h>
00034 #include <ipxe/iobuf.h>
00035 #include <ipxe/malloc.h>
00036 #include <ipxe/pci.h>
00037 #include "ena.h"
00038 
00039 /** @file
00040  *
00041  * Amazon ENA network driver
00042  *
00043  */
00044 
00045 /**
00046  * Get direction name (for debugging)
00047  *
00048  * @v direction         Direction
00049  * @ret name            Direction name
00050  */
00051 static const char * ena_direction ( unsigned int direction ) {
00052 
00053         switch ( direction ) {
00054         case ENA_SQ_TX:         return "TX";
00055         case ENA_SQ_RX:         return "RX";
00056         default:                return "<UNKNOWN>";
00057         }
00058 }
00059 
00060 /******************************************************************************
00061  *
00062  * Device reset
00063  *
00064  ******************************************************************************
00065  */
00066 
00067 /**
00068  * Reset hardware
00069  *
00070  * @v ena               ENA device
00071  * @ret rc              Return status code
00072  */
00073 static int ena_reset ( struct ena_nic *ena ) {
00074         uint32_t stat;
00075         unsigned int i;
00076 
00077         /* Trigger reset */
00078         writel ( ENA_CTRL_RESET, ( ena->regs + ENA_CTRL ) );
00079 
00080         /* Wait for reset to complete */
00081         for ( i = 0 ; i < ENA_RESET_MAX_WAIT_MS ; i++ ) {
00082 
00083                 /* Check if device is ready */
00084                 stat = readl ( ena->regs + ENA_STAT );
00085                 if ( stat & ENA_STAT_READY )
00086                         return 0;
00087 
00088                 /* Delay */
00089                 mdelay ( 1 );
00090         }
00091 
00092         DBGC ( ena, "ENA %p timed out waiting for reset (status %#08x)\n",
00093                ena, stat );
00094         return -ETIMEDOUT;
00095 }
00096 
00097 /******************************************************************************
00098  *
00099  * Admin queue
00100  *
00101  ******************************************************************************
00102  */
00103 
00104 /**
00105  * Set queue base address
00106  *
00107  * @v ena               ENA device
00108  * @v offset            Register offset
00109  * @v address           Base address
00110  */
00111 static inline void ena_set_base ( struct ena_nic *ena, unsigned int offset,
00112                                   void *base ) {
00113         physaddr_t phys = virt_to_bus ( base );
00114 
00115         /* Program base address registers */
00116         writel ( ( phys & 0xffffffffUL ),
00117                  ( ena->regs + offset + ENA_BASE_LO ) );
00118         if ( sizeof ( phys ) > sizeof ( uint32_t ) ) {
00119                 writel ( ( ( ( uint64_t ) phys ) >> 32 ),
00120                          ( ena->regs + offset + ENA_BASE_HI ) );
00121         } else {
00122                 writel ( 0, ( ena->regs + offset + ENA_BASE_HI ) );
00123         }
00124 }
00125 
00126 /**
00127  * Set queue capabilities
00128  *
00129  * @v ena               ENA device
00130  * @v offset            Register offset
00131  * @v count             Number of entries
00132  * @v size              Size of each entry
00133  */
00134 static inline __attribute__ (( always_inline )) void
00135 ena_set_caps ( struct ena_nic *ena, unsigned int offset, unsigned int count,
00136                size_t size ) {
00137 
00138         /* Program capabilities register */
00139         writel ( ENA_CAPS ( count, size ), ( ena->regs + offset ) );
00140 }
00141 
00142 /**
00143  * Clear queue capabilities
00144  *
00145  * @v ena               ENA device
00146  * @v offset            Register offset
00147  */
00148 static inline __attribute__ (( always_inline )) void
00149 ena_clear_caps ( struct ena_nic *ena, unsigned int offset ) {
00150 
00151         /* Clear capabilities register */
00152         writel ( 0, ( ena->regs + offset ) );
00153 }
00154 
00155 /**
00156  * Create admin queues
00157  *
00158  * @v ena               ENA device
00159  * @ret rc              Return status code
00160  */
00161 static int ena_create_admin ( struct ena_nic *ena ) {
00162         size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
00163         size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
00164         int rc;
00165 
00166         /* Allocate admin completion queue */
00167         ena->acq.rsp = malloc_dma ( acq_len, acq_len );
00168         if ( ! ena->acq.rsp ) {
00169                 rc = -ENOMEM;
00170                 goto err_alloc_acq;
00171         }
00172         memset ( ena->acq.rsp, 0, acq_len );
00173 
00174         /* Allocate admin queue */
00175         ena->aq.req = malloc_dma ( aq_len, aq_len );
00176         if ( ! ena->aq.req ) {
00177                 rc = -ENOMEM;
00178                 goto err_alloc_aq;
00179         }
00180         memset ( ena->aq.req, 0, aq_len );
00181 
00182         /* Program queue addresses and capabilities */
00183         ena_set_base ( ena, ENA_ACQ_BASE, ena->acq.rsp );
00184         ena_set_caps ( ena, ENA_ACQ_CAPS, ENA_ACQ_COUNT,
00185                        sizeof ( ena->acq.rsp[0] ) );
00186         ena_set_base ( ena, ENA_AQ_BASE, ena->aq.req );
00187         ena_set_caps ( ena, ENA_AQ_CAPS, ENA_AQ_COUNT,
00188                        sizeof ( ena->aq.req[0] ) );
00189 
00190         DBGC ( ena, "ENA %p AQ [%08lx,%08lx) ACQ [%08lx,%08lx)\n",
00191                ena, virt_to_phys ( ena->aq.req ),
00192                ( virt_to_phys ( ena->aq.req ) + aq_len ),
00193                virt_to_phys ( ena->acq.rsp ),
00194                ( virt_to_phys ( ena->acq.rsp ) + acq_len ) );
00195         return 0;
00196 
00197         ena_clear_caps ( ena, ENA_AQ_CAPS );
00198         ena_clear_caps ( ena, ENA_ACQ_CAPS );
00199         free_dma ( ena->aq.req, aq_len );
00200  err_alloc_aq:
00201         free_dma ( ena->acq.rsp, acq_len );
00202  err_alloc_acq:
00203         return rc;
00204 }
00205 
00206 /**
00207  * Destroy admin queues
00208  *
00209  * @v ena               ENA device
00210  */
00211 static void ena_destroy_admin ( struct ena_nic *ena ) {
00212         size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
00213         size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
00214 
00215         /* Clear queue capabilities */
00216         ena_clear_caps ( ena, ENA_AQ_CAPS );
00217         ena_clear_caps ( ena, ENA_ACQ_CAPS );
00218         wmb();
00219 
00220         /* Free queues */
00221         free_dma ( ena->aq.req, aq_len );
00222         free_dma ( ena->acq.rsp, acq_len );
00223         DBGC ( ena, "ENA %p AQ and ACQ destroyed\n", ena );
00224 }
00225 
00226 /**
00227  * Get next available admin queue request
00228  *
00229  * @v ena               ENA device
00230  * @ret req             Admin queue request
00231  */
00232 static union ena_aq_req * ena_admin_req ( struct ena_nic *ena ) {
00233         union ena_aq_req *req;
00234         unsigned int index;
00235 
00236         /* Get next request */
00237         index = ( ena->aq.prod % ENA_AQ_COUNT );
00238         req = &ena->aq.req[index];
00239 
00240         /* Initialise request */
00241         memset ( ( ( ( void * ) req ) + sizeof ( req->header ) ), 0,
00242                  ( sizeof ( *req ) - sizeof ( req->header ) ) );
00243         req->header.id = ena->aq.prod;
00244 
00245         /* Increment producer counter */
00246         ena->aq.prod++;
00247 
00248         return req;
00249 }
00250 
00251 /**
00252  * Issue admin queue request
00253  *
00254  * @v ena               ENA device
00255  * @v req               Admin queue request
00256  * @v rsp               Admin queue response to fill in
00257  * @ret rc              Return status code
00258  */
00259 static int ena_admin ( struct ena_nic *ena, union ena_aq_req *req,
00260                        union ena_acq_rsp **rsp ) {
00261         unsigned int index;
00262         unsigned int i;
00263         int rc;
00264 
00265         /* Locate response */
00266         index = ( ena->acq.cons % ENA_ACQ_COUNT );
00267         *rsp = &ena->acq.rsp[index];
00268 
00269         /* Mark request as ready */
00270         req->header.flags ^= ENA_AQ_PHASE;
00271         wmb();
00272         DBGC2 ( ena, "ENA %p admin request %#x:\n",
00273                 ena, le16_to_cpu ( req->header.id ) );
00274         DBGC2_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
00275 
00276         /* Ring doorbell */
00277         writel ( ena->aq.prod, ( ena->regs + ENA_AQ_DB ) );
00278 
00279         /* Wait for response */
00280         for ( i = 0 ; i < ENA_ADMIN_MAX_WAIT_MS ; i++ ) {
00281 
00282                 /* Check for response */
00283                 if ( ( (*rsp)->header.flags ^ ena->acq.phase ) & ENA_ACQ_PHASE){
00284                         mdelay ( 1 );
00285                         continue;
00286                 }
00287                 DBGC2 ( ena, "ENA %p admin response %#x:\n",
00288                         ena, le16_to_cpu ( (*rsp)->header.id ) );
00289                 DBGC2_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ));
00290 
00291                 /* Increment consumer counter */
00292                 ena->acq.cons++;
00293                 if ( ( ena->acq.cons % ENA_ACQ_COUNT ) == 0 )
00294                         ena->acq.phase ^= ENA_ACQ_PHASE;
00295 
00296                 /* Check command identifier */
00297                 if ( (*rsp)->header.id != req->header.id ) {
00298                         DBGC ( ena, "ENA %p admin response %#x mismatch:\n",
00299                                ena, le16_to_cpu ( (*rsp)->header.id ) );
00300                         rc = -EILSEQ;
00301                         goto err;
00302                 }
00303 
00304                 /* Check status */
00305                 if ( (*rsp)->header.status != 0 ) {
00306                         DBGC ( ena, "ENA %p admin response %#x status %d:\n",
00307                                ena, le16_to_cpu ( (*rsp)->header.id ),
00308                                (*rsp)->header.status );
00309                         rc = -EIO;
00310                         goto err;
00311                 }
00312 
00313                 /* Success */
00314                 return 0;
00315         }
00316 
00317         rc = -ETIMEDOUT;
00318         DBGC ( ena, "ENA %p timed out waiting for admin request %#x:\n",
00319                ena, le16_to_cpu ( req->header.id ) );
00320  err:
00321         DBGC_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
00322         DBGC_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ) );
00323         return rc;
00324 }
00325 
00326 /**
00327  * Create submission queue
00328  *
00329  * @v ena               ENA device
00330  * @v sq                Submission queue
00331  * @v cq                Corresponding completion queue
00332  * @ret rc              Return status code
00333  */
00334 static int ena_create_sq ( struct ena_nic *ena, struct ena_sq *sq,
00335                            struct ena_cq *cq ) {
00336         union ena_aq_req *req;
00337         union ena_acq_rsp *rsp;
00338         int rc;
00339 
00340         /* Allocate submission queue entries */
00341         sq->sqe.raw = malloc_dma ( sq->len, ENA_ALIGN );
00342         if ( ! sq->sqe.raw ) {
00343                 rc = -ENOMEM;
00344                 goto err_alloc;
00345         }
00346         memset ( sq->sqe.raw, 0, sq->len );
00347 
00348         /* Construct request */
00349         req = ena_admin_req ( ena );
00350         req->header.opcode = ENA_CREATE_SQ;
00351         req->create_sq.direction = sq->direction;
00352         req->create_sq.policy = cpu_to_le16 ( ENA_SQ_HOST_MEMORY |
00353                                               ENA_SQ_CONTIGUOUS );
00354         req->create_sq.cq_id = cpu_to_le16 ( cq->id );
00355         req->create_sq.count = cpu_to_le16 ( sq->count );
00356         req->create_sq.address = cpu_to_le64 ( virt_to_bus ( sq->sqe.raw ) );
00357 
00358         /* Issue request */
00359         if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
00360                 goto err_admin;
00361 
00362         /* Parse response */
00363         sq->id = le16_to_cpu ( rsp->create_sq.id );
00364         sq->doorbell = le32_to_cpu ( rsp->create_sq.doorbell );
00365 
00366         /* Reset producer counter and phase */
00367         sq->prod = 0;
00368         sq->phase = ENA_SQE_PHASE;
00369 
00370         DBGC ( ena, "ENA %p %s SQ%d at [%08lx,%08lx) db +%04x CQ%d\n",
00371                ena, ena_direction ( sq->direction ), sq->id,
00372                virt_to_phys ( sq->sqe.raw ),
00373                ( virt_to_phys ( sq->sqe.raw ) + sq->len ),
00374                sq->doorbell, cq->id );
00375         return 0;
00376 
00377  err_admin:
00378         free_dma ( sq->sqe.raw, sq->len );
00379  err_alloc:
00380         return rc;
00381 }
00382 
00383 /**
00384  * Destroy submission queue
00385  *
00386  * @v ena               ENA device
00387  * @v sq                Submission queue
00388  * @ret rc              Return status code
00389  */
00390 static int ena_destroy_sq ( struct ena_nic *ena, struct ena_sq *sq ) {
00391         union ena_aq_req *req;
00392         union ena_acq_rsp *rsp;
00393         int rc;
00394 
00395         /* Construct request */
00396         req = ena_admin_req ( ena );
00397         req->header.opcode = ENA_DESTROY_SQ;
00398         req->destroy_sq.id = cpu_to_le16 ( sq->id );
00399         req->destroy_sq.direction = sq->direction;
00400 
00401         /* Issue request */
00402         if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
00403                 return rc;
00404 
00405         /* Free submission queue entries */
00406         free_dma ( sq->sqe.raw, sq->len );
00407 
00408         DBGC ( ena, "ENA %p %s SQ%d destroyed\n",
00409                ena, ena_direction ( sq->direction ), sq->id );
00410         return 0;
00411 }
00412 
00413 /**
00414  * Create completion queue
00415  *
00416  * @v ena               ENA device
00417  * @v cq                Completion queue
00418  * @ret rc              Return status code
00419  */
00420 static int ena_create_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
00421         union ena_aq_req *req;
00422         union ena_acq_rsp *rsp;
00423         int rc;
00424 
00425         /* Allocate completion queue entries */
00426         cq->cqe.raw = malloc_dma ( cq->len, ENA_ALIGN );
00427         if ( ! cq->cqe.raw ) {
00428                 rc = -ENOMEM;
00429                 goto err_alloc;
00430         }
00431         memset ( cq->cqe.raw, 0, cq->len );
00432 
00433         /* Construct request */
00434         req = ena_admin_req ( ena );
00435         req->header.opcode = ENA_CREATE_CQ;
00436         req->create_cq.size = cq->size;
00437         req->create_cq.count = cpu_to_le16 ( cq->requested );
00438         req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );
00439 
00440         /* Issue request */
00441         if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
00442                 goto err_admin;
00443 
00444         /* Parse response */
00445         cq->id = le16_to_cpu ( rsp->create_cq.id );
00446         cq->actual = le16_to_cpu ( rsp->create_cq.count );
00447         cq->doorbell = le32_to_cpu ( rsp->create_cq.doorbell );
00448         cq->mask = ( cq->actual - 1 );
00449         if ( cq->actual != cq->requested ) {
00450                 DBGC ( ena, "ENA %p CQ%d requested %d actual %d\n",
00451                        ena, cq->id, cq->requested, cq->actual );
00452         }
00453 
00454         /* Reset consumer counter and phase */
00455         cq->cons = 0;
00456         cq->phase = ENA_CQE_PHASE;
00457 
00458         DBGC ( ena, "ENA %p CQ%d at [%08lx,%08lx) db +%04x\n",
00459                ena, cq->id, virt_to_phys ( cq->cqe.raw ),
00460                ( virt_to_phys ( cq->cqe.raw ) + cq->len ), cq->doorbell );
00461         return 0;
00462 
00463  err_admin:
00464         free_dma ( cq->cqe.raw, cq->len );
00465  err_alloc:
00466         return rc;
00467 }
00468 
00469 /**
00470  * Destroy completion queue
00471  *
00472  * @v ena               ENA device
00473  * @v cq                Completion queue
00474  * @ret rc              Return status code
00475  */
00476 static int ena_destroy_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
00477         union ena_aq_req *req;
00478         union ena_acq_rsp *rsp;
00479         int rc;
00480 
00481         /* Construct request */
00482         req = ena_admin_req ( ena );
00483         req->header.opcode = ENA_DESTROY_CQ;
00484         req->destroy_cq.id = cpu_to_le16 ( cq->id );
00485 
00486         /* Issue request */
00487         if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
00488                 return rc;
00489 
00490         /* Free completion queue entries */
00491         free_dma ( cq->cqe.raw, cq->len );
00492 
00493         DBGC ( ena, "ENA %p CQ%d destroyed\n", ena, cq->id );
00494         return 0;
00495 }
00496 
00497 /**
00498  * Create queue pair
00499  *
00500  * @v ena               ENA device
00501  * @v qp                Queue pair
00502  * @ret rc              Return status code
00503  */
00504 static int ena_create_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
00505         int rc;
00506 
00507         /* Create completion queue */
00508         if ( ( rc = ena_create_cq ( ena, &qp->cq ) ) != 0 )
00509                 goto err_create_cq;
00510 
00511         /* Create submission queue */
00512         if ( ( rc = ena_create_sq ( ena, &qp->sq, &qp->cq ) ) != 0 )
00513                 goto err_create_sq;
00514 
00515         return 0;
00516 
00517         ena_destroy_sq ( ena, &qp->sq );
00518  err_create_sq:
00519         ena_destroy_cq ( ena, &qp->cq );
00520  err_create_cq:
00521         return rc;
00522 }
00523 
00524 /**
00525  * Destroy queue pair
00526  *
00527  * @v ena               ENA device
00528  * @v qp                Queue pair
00529  * @ret rc              Return status code
00530  */
00531 static int ena_destroy_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
00532 
00533         /* Destroy submission queue */
00534         ena_destroy_sq ( ena,  &qp->sq );
00535 
00536         /* Destroy completion queue */
00537         ena_destroy_cq ( ena, &qp->cq );
00538 
00539         return 0;
00540 }
00541 
00542 /**
00543  * Get device attributes
00544  *
00545  * @v netdev            Network device
00546  * @ret rc              Return status code
00547  */
00548 static int ena_get_device_attributes ( struct net_device *netdev ) {
00549         struct ena_nic *ena = netdev->priv;
00550         union ena_aq_req *req;
00551         union ena_acq_rsp *rsp;
00552         union ena_feature *feature;
00553         int rc;
00554 
00555         /* Construct request */
00556         req = ena_admin_req ( ena );
00557         req->header.opcode = ENA_GET_FEATURE;
00558         req->get_feature.id = ENA_DEVICE_ATTRIBUTES;
00559 
00560         /* Issue request */
00561         if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
00562                 return rc;
00563 
00564         /* Parse response */
00565         feature = &rsp->get_feature.feature;
00566         memcpy ( netdev->hw_addr, feature->device.mac, ETH_ALEN );
00567         netdev->max_pkt_len = le32_to_cpu ( feature->device.mtu );
00568         netdev->mtu = ( netdev->max_pkt_len - ETH_HLEN );
00569 
00570         DBGC ( ena, "ENA %p MAC %s MTU %zd\n",
00571                ena, eth_ntoa ( netdev->hw_addr ), netdev->max_pkt_len );
00572         return 0;
00573 }
00574 
00575 /**
00576  * Get statistics (for debugging)
00577  *
00578  * @v ena               ENA device
00579  * @ret rc              Return status code
00580  */
00581 static int ena_get_stats ( struct ena_nic *ena ) {
00582         union ena_aq_req *req;
00583         union ena_acq_rsp *rsp;
00584         struct ena_get_stats_rsp *stats;
00585         int rc;
00586 
00587         /* Do nothing unless debug messages are enabled */
00588         if ( ! DBG_LOG )
00589                 return 0;
00590 
00591         /* Construct request */
00592         req = ena_admin_req ( ena );
00593         req->header.opcode = ENA_GET_STATS;
00594         req->get_stats.type = ENA_STATS_TYPE_BASIC;
00595         req->get_stats.scope = ENA_STATS_SCOPE_ETH;
00596         req->get_stats.device = ENA_DEVICE_MINE;
00597 
00598         /* Issue request */
00599         if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
00600                 return rc;
00601 
00602         /* Parse response */
00603         stats = &rsp->get_stats;
00604         DBGC ( ena, "ENA %p TX bytes %#llx packets %#llx\n", ena,
00605                ( ( unsigned long long ) le64_to_cpu ( stats->tx_bytes ) ),
00606                ( ( unsigned long long ) le64_to_cpu ( stats->tx_packets ) ) );
00607         DBGC ( ena, "ENA %p RX bytes %#llx packets %#llx drops %#llx\n", ena,
00608                ( ( unsigned long long ) le64_to_cpu ( stats->rx_bytes ) ),
00609                ( ( unsigned long long ) le64_to_cpu ( stats->rx_packets ) ),
00610                ( ( unsigned long long ) le64_to_cpu ( stats->rx_drops ) ) );
00611 
00612         return 0;
00613 }
00614 
00615 /******************************************************************************
00616  *
00617  * Network device interface
00618  *
00619  ******************************************************************************
00620  */
00621 
00622 /**
00623  * Refill receive queue
00624  *
00625  * @v netdev            Network device
00626  */
00627 static void ena_refill_rx ( struct net_device *netdev ) {
00628         struct ena_nic *ena = netdev->priv;
00629         struct io_buffer *iobuf;
00630         struct ena_rx_sqe *sqe;
00631         unsigned int index;
00632         physaddr_t address;
00633         size_t len = netdev->max_pkt_len;
00634         unsigned int refilled = 0;
00635 
00636         /* Refill queue */
00637         while ( ( ena->rx.sq.prod - ena->rx.cq.cons ) < ENA_RX_COUNT ) {
00638 
00639                 /* Allocate I/O buffer */
00640                 iobuf = alloc_iob ( len );
00641                 if ( ! iobuf ) {
00642                         /* Wait for next refill */
00643                         break;
00644                 }
00645 
00646                 /* Get next submission queue entry */
00647                 index = ( ena->rx.sq.prod % ENA_RX_COUNT );
00648                 sqe = &ena->rx.sq.sqe.rx[index];
00649 
00650                 /* Construct submission queue entry */
00651                 address = virt_to_bus ( iobuf->data );
00652                 sqe->len = cpu_to_le16 ( len );
00653                 sqe->id = cpu_to_le16 ( ena->rx.sq.prod );
00654                 sqe->address = cpu_to_le64 ( address );
00655                 wmb();
00656                 sqe->flags = ( ENA_SQE_FIRST | ENA_SQE_LAST | ENA_SQE_CPL |
00657                                ena->rx.sq.phase );
00658 
00659                 /* Increment producer counter */
00660                 ena->rx.sq.prod++;
00661                 if ( ( ena->rx.sq.prod % ENA_RX_COUNT ) == 0 )
00662                         ena->rx.sq.phase ^= ENA_SQE_PHASE;
00663 
00664                 /* Record I/O buffer */
00665                 assert ( ena->rx_iobuf[index] == NULL );
00666                 ena->rx_iobuf[index] = iobuf;
00667 
00668                 DBGC2 ( ena, "ENA %p RX %d at [%08llx,%08llx)\n", ena, sqe->id,
00669                         ( ( unsigned long long ) address ),
00670                         ( ( unsigned long long ) address + len ) );
00671                 refilled++;
00672         }
00673 
00674         /* Ring doorbell, if applicable */
00675         if ( refilled ) {
00676                 wmb();
00677                 writel ( ena->rx.sq.prod, ( ena->regs + ena->rx.sq.doorbell ) );
00678         }
00679 }
00680 
00681 /**
00682  * Discard unused receive I/O buffers
00683  *
00684  * @v ena               ENA device
00685  */
00686 static void ena_empty_rx ( struct ena_nic *ena ) {
00687         unsigned int i;
00688 
00689         for ( i = 0 ; i < ENA_RX_COUNT ; i++ ) {
00690                 if ( ena->rx_iobuf[i] )
00691                         free_iob ( ena->rx_iobuf[i] );
00692                 ena->rx_iobuf[i] = NULL;
00693         }
00694 }
00695 
00696 /**
00697  * Open network device
00698  *
00699  * @v netdev            Network device
00700  * @ret rc              Return status code
00701  */
00702 static int ena_open ( struct net_device *netdev ) {
00703         struct ena_nic *ena = netdev->priv;
00704         int rc;
00705 
00706         /* Create transmit queue pair */
00707         if ( ( rc = ena_create_qp ( ena, &ena->tx ) ) != 0 )
00708                 goto err_create_tx;
00709 
00710         /* Create receive queue pair */
00711         if ( ( rc = ena_create_qp ( ena, &ena->rx ) ) != 0 )
00712                 goto err_create_rx;
00713 
00714         /* Refill receive queue */
00715         ena_refill_rx ( netdev );
00716 
00717         return 0;
00718 
00719         ena_destroy_qp ( ena, &ena->rx );
00720  err_create_rx:
00721         ena_destroy_qp ( ena, &ena->tx );
00722  err_create_tx:
00723         return rc;
00724 }
00725 
00726 /**
00727  * Close network device
00728  *
00729  * @v netdev            Network device
00730  */
00731 static void ena_close ( struct net_device *netdev ) {
00732         struct ena_nic *ena = netdev->priv;
00733 
00734         /* Dump statistics (for debugging) */
00735         ena_get_stats ( ena );
00736 
00737         /* Destroy receive queue pair */
00738         ena_destroy_qp ( ena, &ena->rx );
00739 
00740         /* Discard any unused receive buffers */
00741         ena_empty_rx ( ena );
00742 
00743         /* Destroy transmit queue pair */
00744         ena_destroy_qp ( ena, &ena->tx );
00745 }
00746 
00747 /**
00748  * Transmit packet
00749  *
00750  * @v netdev            Network device
00751  * @v iobuf             I/O buffer
00752  * @ret rc              Return status code
00753  */
00754 static int ena_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
00755         struct ena_nic *ena = netdev->priv;
00756         struct ena_tx_sqe *sqe;
00757         unsigned int index;
00758         physaddr_t address;
00759         size_t len;
00760 
00761         /* Get next submission queue entry */
00762         if ( ( ena->tx.sq.prod - ena->tx.cq.cons ) >= ENA_TX_COUNT ) {
00763                 DBGC ( ena, "ENA %p out of transmit descriptors\n", ena );
00764                 return -ENOBUFS;
00765         }
00766         index = ( ena->tx.sq.prod % ENA_TX_COUNT );
00767         sqe = &ena->tx.sq.sqe.tx[index];
00768 
00769         /* Construct submission queue entry */
00770         address = virt_to_bus ( iobuf->data );
00771         len = iob_len ( iobuf );
00772         sqe->len = cpu_to_le16 ( len );
00773         sqe->id = ena->tx.sq.prod;
00774         sqe->address = cpu_to_le64 ( address );
00775         wmb();
00776         sqe->flags = ( ENA_SQE_FIRST | ENA_SQE_LAST | ENA_SQE_CPL |
00777                        ena->tx.sq.phase );
00778         wmb();
00779 
00780         /* Increment producer counter */
00781         ena->tx.sq.prod++;
00782         if ( ( ena->tx.sq.prod % ENA_TX_COUNT ) == 0 )
00783                 ena->tx.sq.phase ^= ENA_SQE_PHASE;
00784 
00785         /* Ring doorbell */
00786         writel ( ena->tx.sq.prod, ( ena->regs + ena->tx.sq.doorbell ) );
00787 
00788         DBGC2 ( ena, "ENA %p TX %d at [%08llx,%08llx)\n", ena, sqe->id,
00789                 ( ( unsigned long long ) address ),
00790                 ( ( unsigned long long ) address + len ) );
00791         return 0;
00792 }
00793 
00794 /**
00795  * Poll for completed transmissions
00796  *
00797  * @v netdev            Network device
00798  */
00799 static void ena_poll_tx ( struct net_device *netdev ) {
00800         struct ena_nic *ena = netdev->priv;
00801         struct ena_tx_cqe *cqe;
00802         unsigned int index;
00803 
00804         /* Check for completed packets */
00805         while ( ena->tx.cq.cons != ena->tx.sq.prod ) {
00806 
00807                 /* Get next completion queue entry */
00808                 index = ( ena->tx.cq.cons & ena->tx.cq.mask );
00809                 cqe = &ena->tx.cq.cqe.tx[index];
00810 
00811                 /* Stop if completion queue entry is empty */
00812                 if ( ( cqe->flags ^ ena->tx.cq.phase ) & ENA_CQE_PHASE )
00813                         return;
00814                 DBGC2 ( ena, "ENA %p TX %d complete\n", ena,
00815                         ( le16_to_cpu ( cqe->id ) >> 2 /* Don't ask */ ) );
00816 
00817                 /* Increment consumer counter */
00818                 ena->tx.cq.cons++;
00819                 if ( ! ( ena->tx.cq.cons & ena->tx.cq.mask ) )
00820                         ena->tx.cq.phase ^= ENA_CQE_PHASE;
00821 
00822                 /* Complete transmit */
00823                 netdev_tx_complete_next ( netdev );
00824         }
00825 }
00826 
00827 /**
00828  * Poll for received packets
00829  *
00830  * @v netdev            Network device
00831  */
00832 static void ena_poll_rx ( struct net_device *netdev ) {
00833         struct ena_nic *ena = netdev->priv;
00834         struct ena_rx_cqe *cqe;
00835         struct io_buffer *iobuf;
00836         unsigned int index;
00837         size_t len;
00838 
00839         /* Check for received packets */
00840         while ( ena->rx.cq.cons != ena->rx.sq.prod ) {
00841 
00842                 /* Get next completion queue entry */
00843                 index = ( ena->rx.cq.cons % ENA_RX_COUNT );
00844                 cqe = &ena->rx.cq.cqe.rx[index];
00845 
00846                 /* Stop if completion queue entry is empty */
00847                 if ( ( cqe->flags ^ ena->rx.cq.phase ) & ENA_CQE_PHASE )
00848                         return;
00849 
00850                 /* Increment consumer counter */
00851                 ena->rx.cq.cons++;
00852                 if ( ! ( ena->rx.cq.cons & ena->rx.cq.mask ) )
00853                         ena->rx.cq.phase ^= ENA_CQE_PHASE;
00854 
00855                 /* Populate I/O buffer */
00856                 iobuf = ena->rx_iobuf[index];
00857                 ena->rx_iobuf[index] = NULL;
00858                 len = le16_to_cpu ( cqe->len );
00859                 iob_put ( iobuf, len );
00860 
00861                 /* Hand off to network stack */
00862                 DBGC2 ( ena, "ENA %p RX %d complete (length %zd)\n",
00863                         ena, le16_to_cpu ( cqe->id ), len );
00864                 netdev_rx ( netdev, iobuf );
00865         }
00866 }
00867 
00868 /**
00869  * Poll for completed and received packets
00870  *
00871  * @v netdev            Network device
00872  */
00873 static void ena_poll ( struct net_device *netdev ) {
00874 
00875         /* Poll for transmit completions */
00876         ena_poll_tx ( netdev );
00877 
00878         /* Poll for receive completions */
00879         ena_poll_rx ( netdev );
00880 
00881         /* Refill receive ring */
00882         ena_refill_rx ( netdev );
00883 }
00884 
00885 /** ENA network device operations */
00886 static struct net_device_operations ena_operations = {
00887         .open           = ena_open,
00888         .close          = ena_close,
00889         .transmit       = ena_transmit,
00890         .poll           = ena_poll,
00891 };
00892 
00893 /******************************************************************************
00894  *
00895  * PCI interface
00896  *
00897  ******************************************************************************
00898  */
00899 
00900 /**
00901  * Probe PCI device
00902  *
00903  * @v pci               PCI device
00904  * @ret rc              Return status code
00905  */
00906 static int ena_probe ( struct pci_device *pci ) {
00907         struct net_device *netdev;
00908         struct ena_nic *ena;
00909         int rc;
00910 
00911         /* Allocate and initialise net device */
00912         netdev = alloc_etherdev ( sizeof ( *ena ) );
00913         if ( ! netdev ) {
00914                 rc = -ENOMEM;
00915                 goto err_alloc;
00916         }
00917         netdev_init ( netdev, &ena_operations );
00918         ena = netdev->priv;
00919         pci_set_drvdata ( pci, netdev );
00920         netdev->dev = &pci->dev;
00921         memset ( ena, 0, sizeof ( *ena ) );
00922         ena->acq.phase = ENA_ACQ_PHASE;
00923         ena_cq_init ( &ena->tx.cq, ENA_TX_COUNT,
00924                       sizeof ( ena->tx.cq.cqe.tx[0] ) );
00925         ena_sq_init ( &ena->tx.sq, ENA_SQ_TX, ENA_TX_COUNT,
00926                       sizeof ( ena->tx.sq.sqe.tx[0] ) );
00927         ena_cq_init ( &ena->rx.cq, ENA_RX_COUNT,
00928                       sizeof ( ena->rx.cq.cqe.rx[0] ) );
00929         ena_sq_init ( &ena->rx.sq, ENA_SQ_RX, ENA_RX_COUNT,
00930                       sizeof ( ena->rx.sq.sqe.rx[0] ) );
00931 
00932         /* Fix up PCI device */
00933         adjust_pci_device ( pci );
00934 
00935         /* Map registers */
00936         ena->regs = ioremap ( pci->membase, ENA_BAR_SIZE );
00937         if ( ! ena->regs ) {
00938                 rc = -ENODEV;
00939                 goto err_ioremap;
00940         }
00941 
00942         /* Reset the NIC */
00943         if ( ( rc = ena_reset ( ena ) ) != 0 )
00944                 goto err_reset;
00945 
00946         /* Create admin queues */
00947         if ( ( rc = ena_create_admin ( ena ) ) != 0 )
00948                 goto err_create_admin;
00949 
00950         /* Fetch MAC address */
00951         if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
00952                 goto err_get_device_attributes;
00953 
00954         /* Register network device */
00955         if ( ( rc = register_netdev ( netdev ) ) != 0 )
00956                 goto err_register_netdev;
00957 
00958         /* Mark as link up, since we have no way to test link state on
00959          * this hardware.
00960          */
00961         netdev_link_up ( netdev );
00962 
00963         return 0;
00964 
00965         unregister_netdev ( netdev );
00966  err_register_netdev:
00967  err_get_device_attributes:
00968         ena_destroy_admin ( ena );
00969  err_create_admin:
00970         ena_reset ( ena );
00971  err_reset:
00972         iounmap ( ena->regs );
00973  err_ioremap:
00974         netdev_nullify ( netdev );
00975         netdev_put ( netdev );
00976  err_alloc:
00977         return rc;
00978 }
00979 
00980 /**
00981  * Remove PCI device
00982  *
00983  * @v pci               PCI device
00984  */
00985 static void ena_remove ( struct pci_device *pci ) {
00986         struct net_device *netdev = pci_get_drvdata ( pci );
00987         struct ena_nic *ena = netdev->priv;
00988 
00989         /* Unregister network device */
00990         unregister_netdev ( netdev );
00991 
00992         /* Destroy admin queues */
00993         ena_destroy_admin ( ena );
00994 
00995         /* Reset card */
00996         ena_reset ( ena );
00997 
00998         /* Free network device */
00999         iounmap ( ena->regs );
01000         netdev_nullify ( netdev );
01001         netdev_put ( netdev );
01002 }
01003 
01004 /** ENA PCI device IDs */
01005 static struct pci_device_id ena_nics[] = {
01006         PCI_ROM ( 0x1d0f, 0xec20, "ena-vf", "ENA VF", 0 ),
01007         PCI_ROM ( 0x1d0f, 0xec21, "ena-vf-llq", "ENA VF (LLQ)", 0 ),
01008 };
01009 
01010 /** ENA PCI driver */
01011 struct pci_driver ena_driver __pci_driver = {
01012         .ids = ena_nics,
01013         .id_count = ( sizeof ( ena_nics ) / sizeof ( ena_nics[0] ) ),
01014         .probe = ena_probe,
01015         .remove = ena_remove,
01016 };