iPXE
thunderx.c
Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
00003  *
00004  * This program is free software; you can redistribute it and/or
00005  * modify it under the terms of the GNU General Public License as
00006  * published by the Free Software Foundation; either version 2 of the
00007  * License, or (at your option) any later version.
00008  *
00009  * This program is distributed in the hope that it will be useful, but
00010  * WITHOUT ANY WARRANTY; without even the implied warranty of
00011  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012  * General Public License for more details.
00013  *
00014  * You should have received a copy of the GNU General Public License
00015  * along with this program; if not, write to the Free Software
00016  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00017  * 02110-1301, USA.
00018  *
00019  * You can also choose to distribute this program under the terms of
00020  * the Unmodified Binary Distribution Licence (as given in the file
00021  * COPYING.UBDL), provided that you have satisfied its requirements.
00022  */
00023 
00024 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
00025 
00026 #include <stdint.h>
00027 #include <string.h>
00028 #include <strings.h>
00029 #include <stdio.h>
00030 #include <unistd.h>
00031 #include <errno.h>
00032 #include <assert.h>
00033 #include <byteswap.h>
00034 #include <ipxe/netdevice.h>
00035 #include <ipxe/ethernet.h>
00036 #include <ipxe/if_ether.h>
00037 #include <ipxe/iobuf.h>
00038 #include <ipxe/malloc.h>
00039 #include <ipxe/pci.h>
00040 #include <ipxe/pciea.h>
00041 #include <ipxe/umalloc.h>
00042 #include "thunderx.h"
00043 #include "thunderxcfg.h"
00044 
00045 /** @file
00046  *
00047  * Cavium ThunderX Ethernet driver
00048  *
00049  */
00050 
00051 /** List of BGX Ethernet interfaces */
00052 static LIST_HEAD ( txnic_bgxs );
00053 
00054 /** List of physical functions */
00055 static LIST_HEAD ( txnic_pfs );
00056 
00057 /** Debug colour for physical function and BGX messages */
00058 #define TXNICCOL(x) ( &txnic_pfs + (x)->node )
00059 
00060 /** Board configuration protocol */
00061 static EFI_THUNDER_CONFIG_PROTOCOL *txcfg;
00062 EFI_REQUEST_PROTOCOL ( EFI_THUNDER_CONFIG_PROTOCOL, &txcfg );
00063 
00064 /******************************************************************************
00065  *
00066  * Diagnostics
00067  *
00068  ******************************************************************************
00069  */
00070 
00071 /**
00072  * Show virtual NIC diagnostics (for debugging)
00073  *
00074  * @v vnic              Virtual NIC
00075  */
00076 static __attribute__ (( unused )) void txnic_diag ( struct txnic *vnic ) {
00077 
00078         DBGC ( vnic, "TXNIC %s SQ %05zx(%05llx)/%05zx(%05llx) %08llx\n",
00079                vnic->name,
00080                ( ( vnic->sq.prod % TXNIC_SQES ) * TXNIC_SQ_STRIDE ),
00081                readq ( vnic->regs + TXNIC_QS_SQ_TAIL(0) ),
00082                ( ( vnic->sq.cons % TXNIC_SQES ) * TXNIC_SQ_STRIDE ),
00083                readq ( vnic->regs + TXNIC_QS_SQ_HEAD(0) ),
00084                readq ( vnic->regs + TXNIC_QS_SQ_STATUS(0) ) );
00085         DBGC ( vnic, "TXNIC %s RQ %05zx(%05llx)/%05zx(%05llx) %016llx\n",
00086                vnic->name,
00087                ( ( vnic->rq.prod % TXNIC_RQES ) * TXNIC_RQ_STRIDE ),
00088                readq ( vnic->regs + TXNIC_QS_RBDR_TAIL(0) ),
00089                ( ( vnic->rq.cons % TXNIC_RQES ) * TXNIC_RQ_STRIDE ),
00090                readq ( vnic->regs + TXNIC_QS_RBDR_HEAD(0) ),
00091                readq ( vnic->regs + TXNIC_QS_RBDR_STATUS0(0) ) );
00092         DBGC ( vnic, "TXNIC %s CQ xxxxx(%05llx)/%05x(%05llx) %08llx:%08llx\n",
00093                vnic->name, readq ( vnic->regs + TXNIC_QS_CQ_TAIL(0) ),
00094                ( ( vnic->cq.cons % TXNIC_CQES ) * TXNIC_CQ_STRIDE ),
00095                readq ( vnic->regs + TXNIC_QS_CQ_HEAD(0) ),
00096                readq ( vnic->regs + TXNIC_QS_CQ_STATUS(0) ),
00097                readq ( vnic->regs + TXNIC_QS_CQ_STATUS2(0) ) );
00098 }
00099 
00100 /******************************************************************************
00101  *
00102  * Send queue
00103  *
00104  ******************************************************************************
00105  */
00106 
00107 /**
00108  * Create send queue
00109  *
00110  * @v vnic              Virtual NIC
00111  * @ret rc              Return status code
00112  */
00113 static int txnic_create_sq ( struct txnic *vnic ) {
00114 
00115         /* Reset send queue */
00116         vnic->sq.prod = 0;
00117         vnic->sq.cons = 0;
00118         writeq ( TXNIC_QS_SQ_CFG_RESET, ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
00119 
00120         /* Configure and enable send queue */
00121         writeq ( user_to_phys ( vnic->sq.sqe, 0 ),
00122                  ( vnic->regs + TXNIC_QS_SQ_BASE(0) ) );
00123         writeq ( ( TXNIC_QS_SQ_CFG_ENA | TXNIC_QS_SQ_CFG_QSIZE_1K ),
00124                  ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
00125 
00126         DBGC ( vnic, "TXNIC %s SQ at [%08lx,%08lx)\n",
00127                vnic->name, user_to_phys ( vnic->sq.sqe, 0 ),
00128                user_to_phys ( vnic->sq.sqe, TXNIC_SQ_SIZE ) );
00129         return 0;
00130 }
00131 
00132 /**
00133  * Disable send queue
00134  *
00135  * @v vnic              Virtual NIC
00136  * @ret rc              Return status code
00137  */
00138 static int txnic_disable_sq ( struct txnic *vnic ) {
00139         uint64_t status;
00140         unsigned int i;
00141 
00142         /* Disable send queue */
00143         writeq ( 0, ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
00144 
00145         /* Wait for send queue to be stopped */
00146         for ( i = 0 ; i < TXNIC_SQ_STOP_MAX_WAIT_MS ; i++ ) {
00147 
00148                 /* Check if send queue is stopped */
00149                 status = readq ( vnic->regs + TXNIC_QS_SQ_STATUS(0) );
00150                 if ( status & TXNIC_QS_SQ_STATUS_STOPPED )
00151                         return 0;
00152 
00153                 /* Delay */
00154                 mdelay ( 1 );
00155         }
00156 
00157         DBGC ( vnic, "TXNIC %s SQ disable timed out\n", vnic->name );
00158         return -ETIMEDOUT;
00159 }
00160 
00161 /**
00162  * Destroy send queue
00163  *
00164  * @v vnic              Virtual NIC
00165  */
00166 static void txnic_destroy_sq ( struct txnic *vnic ) {
00167         int rc;
00168 
00169         /* Disable send queue */
00170         if ( ( rc = txnic_disable_sq ( vnic ) ) != 0 ) {
00171                 /* Nothing else we can do */
00172                 return;
00173         }
00174 
00175         /* Reset send queue */
00176         writeq ( TXNIC_QS_SQ_CFG_RESET, ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
00177 }
00178 
00179 /**
00180  * Send packet
00181  *
00182  * @v vnic              Virtual NIC
00183  * @v iobuf             I/O buffer
00184  * @ret rc              Return status code
00185  */
00186 static int txnic_send ( struct txnic *vnic, struct io_buffer *iobuf ) {
00187         struct txnic_sqe sqe;
00188         unsigned int sq_idx;
00189         size_t offset;
00190         size_t len;
00191 
00192         /* Get next send queue entry */
00193         if ( ( vnic->sq.prod - vnic->sq.cons ) >= TXNIC_SQ_FILL ) {
00194                 DBGC ( vnic, "TXNIC %s out of send queue entries\n",
00195                        vnic->name );
00196                 return -ENOBUFS;
00197         }
00198         sq_idx = ( vnic->sq.prod++ % TXNIC_SQES );
00199         offset = ( sq_idx * TXNIC_SQ_STRIDE );
00200 
00201         /* Populate send descriptor */
00202         len = iob_len ( iobuf );
00203         memset ( &sqe, 0, sizeof ( sqe ) );
00204         sqe.hdr.total = cpu_to_le32 ( ( len >= ETH_ZLEN ) ? len : ETH_ZLEN );
00205         sqe.hdr.subdcnt = ( TXNIC_SQE_SUBDESCS - 1 );
00206         sqe.hdr.flags = TXNIC_SEND_HDR_FLAGS;
00207         sqe.gather.size = cpu_to_le16 ( len );
00208         sqe.gather.flags = TXNIC_SEND_GATHER_FLAGS;
00209         sqe.gather.addr = cpu_to_le64 ( virt_to_bus ( iobuf->data ) );
00210         DBGC2 ( vnic, "TXNIC %s SQE %#03x is [%08lx,%08lx)\n",
00211                 vnic->name, sq_idx, virt_to_bus ( iobuf->data ),
00212                 ( virt_to_bus ( iobuf->data ) + len ) );
00213 
00214         /* Copy send descriptor to ring */
00215         copy_to_user ( vnic->sq.sqe, offset, &sqe, sizeof ( sqe ) );
00216 
00217         /* Ring doorbell */
00218         wmb();
00219         writeq ( TXNIC_SQE_SUBDESCS, ( vnic->regs + TXNIC_QS_SQ_DOOR(0) ) );
00220 
00221         return 0;
00222 }
00223 
00224 /**
00225  * Complete send queue entry
00226  *
00227  * @v vnic              Virtual NIC
00228  * @v cqe               Send completion queue entry
00229  */
00230 static void txnic_complete_sqe ( struct txnic *vnic,
00231                                  struct txnic_cqe_send *cqe ) {
00232         struct net_device *netdev = vnic->netdev;
00233         unsigned int sq_idx;
00234         unsigned int status;
00235 
00236         /* Parse completion */
00237         sq_idx = ( le16_to_cpu ( cqe->sqe_ptr ) / TXNIC_SQE_SUBDESCS );
00238         status = cqe->send_status;
00239 
00240         /* Sanity check */
00241         assert ( sq_idx == ( vnic->sq.cons % TXNIC_SQES ) );
00242 
00243         /* Free send queue entry */
00244         vnic->sq.cons++;
00245 
00246         /* Complete transmission */
00247         if ( status ) {
00248                 DBGC ( vnic, "TXNIC %s SQE %#03x complete (status %#02x)\n",
00249                        vnic->name, sq_idx, status );
00250                 netdev_tx_complete_next_err ( netdev, -EIO );
00251         } else {
00252                 DBGC2 ( vnic, "TXNIC %s SQE %#03x complete\n",
00253                         vnic->name, sq_idx );
00254                 netdev_tx_complete_next ( netdev );
00255         }
00256 }
00257 
00258 /******************************************************************************
00259  *
00260  * Receive queue
00261  *
00262  ******************************************************************************
00263  */
00264 
00265 /**
00266  * Create receive queue
00267  *
00268  * @v vnic              Virtual NIC
00269  * @ret rc              Return status code
00270  */
00271 static int txnic_create_rq ( struct txnic *vnic ) {
00272 
00273         /* Reset receive buffer descriptor ring */
00274         vnic->rq.prod = 0;
00275         vnic->rq.cons = 0;
00276         writeq ( TXNIC_QS_RBDR_CFG_RESET,
00277                  ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
00278 
00279         /* Configure and enable receive buffer descriptor ring */
00280         writeq ( user_to_phys ( vnic->rq.rqe, 0 ),
00281                  ( vnic->regs + TXNIC_QS_RBDR_BASE(0) ) );
00282         writeq ( ( TXNIC_QS_RBDR_CFG_ENA | TXNIC_QS_RBDR_CFG_QSIZE_8K |
00283                    TXNIC_QS_RBDR_CFG_LINES ( TXNIC_RQE_SIZE /
00284                                              TXNIC_LINE_SIZE ) ),
00285                  ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
00286 
00287         /* Enable receive queue */
00288         writeq ( TXNIC_QS_RQ_CFG_ENA, ( vnic->regs + TXNIC_QS_RQ_CFG(0) ) );
00289 
00290         DBGC ( vnic, "TXNIC %s RQ at [%08lx,%08lx)\n",
00291                vnic->name, user_to_phys ( vnic->rq.rqe, 0 ),
00292                user_to_phys ( vnic->rq.rqe, TXNIC_RQ_SIZE ) );
00293         return 0;
00294 }
00295 
00296 /**
00297  * Disable receive queue
00298  *
00299  * @v vnic              Virtual NIC
00300  * @ret rc              Return status code
00301  */
00302 static int txnic_disable_rq ( struct txnic *vnic ) {
00303         uint64_t cfg;
00304         unsigned int i;
00305 
00306         /* Disable receive queue */
00307         writeq ( 0, ( vnic->regs + TXNIC_QS_RQ_CFG(0) ) );
00308 
00309         /* Wait for receive queue to be disabled */
00310         for ( i = 0 ; i < TXNIC_RQ_DISABLE_MAX_WAIT_MS ; i++ ) {
00311 
00312                 /* Check if receive queue is disabled */
00313                 cfg = readq ( vnic->regs + TXNIC_QS_RQ_CFG(0) );
00314                 if ( ! ( cfg & TXNIC_QS_RQ_CFG_ENA ) )
00315                         return 0;
00316 
00317                 /* Delay */
00318                 mdelay ( 1 );
00319         }
00320 
00321         DBGC ( vnic, "TXNIC %s RQ disable timed out\n", vnic->name );
00322         return -ETIMEDOUT;
00323 }
00324 
00325 /**
00326  * Destroy receive queue
00327  *
00328  * @v vnic              Virtual NIC
00329  */
00330 static void txnic_destroy_rq ( struct txnic *vnic ) {
00331         unsigned int i;
00332         int rc;
00333 
00334         /* Disable receive queue */
00335         if ( ( rc = txnic_disable_rq ( vnic ) ) != 0 ) {
00336                 /* Leak memory; there's nothing else we can do */
00337                 return;
00338         }
00339 
00340         /* Disable receive buffer descriptor ring */
00341         writeq ( 0, ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
00342 
00343         /* Reset receive buffer descriptor ring */
00344         writeq ( TXNIC_QS_RBDR_CFG_RESET,
00345                  ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
00346 
00347         /* Free any unused I/O buffers */
00348         for ( i = 0 ; i < TXNIC_RQ_FILL ; i++ ) {
00349                 if ( vnic->rq.iobuf[i] )
00350                         free_iob ( vnic->rq.iobuf[i] );
00351                 vnic->rq.iobuf[i] = NULL;
00352         }
00353 }
00354 
00355 /**
00356  * Refill receive queue
00357  *
00358  * @v vnic              Virtual NIC
00359  */
00360 static void txnic_refill_rq ( struct txnic *vnic ) {
00361         struct io_buffer *iobuf;
00362         struct txnic_rqe rqe;
00363         unsigned int rq_idx;
00364         unsigned int rq_iobuf_idx;
00365         unsigned int refilled = 0;
00366         size_t offset;
00367 
00368         /* Refill ring */
00369         while ( ( vnic->rq.prod - vnic->rq.cons ) < TXNIC_RQ_FILL ) {
00370 
00371                 /* Allocate I/O buffer */
00372                 iobuf = alloc_iob ( TXNIC_RQE_SIZE );
00373                 if ( ! iobuf ) {
00374                         /* Wait for next refill */
00375                         break;
00376                 }
00377 
00378                 /* Get next receive descriptor */
00379                 rq_idx = ( vnic->rq.prod++ % TXNIC_RQES );
00380                 offset = ( rq_idx * TXNIC_RQ_STRIDE );
00381 
00382                 /* Populate receive descriptor */
00383                 rqe.rbdre.addr = cpu_to_le64 ( virt_to_bus ( iobuf->data ) );
00384                 DBGC2 ( vnic, "TXNIC %s RQE %#03x is [%08lx,%08lx)\n",
00385                         vnic->name, rq_idx, virt_to_bus ( iobuf->data ),
00386                         ( virt_to_bus ( iobuf->data ) + TXNIC_RQE_SIZE ) );
00387 
00388                 /* Copy receive descriptor to ring */
00389                 copy_to_user ( vnic->rq.rqe, offset, &rqe, sizeof ( rqe ) );
00390                 refilled++;
00391 
00392                 /* Record I/O buffer */
00393                 rq_iobuf_idx = ( rq_idx % TXNIC_RQ_FILL );
00394                 assert ( vnic->rq.iobuf[rq_iobuf_idx] == NULL );
00395                 vnic->rq.iobuf[rq_iobuf_idx] = iobuf;
00396         }
00397 
00398         /* Ring doorbell */
00399         wmb();
00400         writeq ( refilled, ( vnic->regs + TXNIC_QS_RBDR_DOOR(0) ) );
00401 }
00402 
00403 /**
00404  * Complete receive queue entry
00405  *
00406  * @v vnic              Virtual NIC
00407  * @v cqe               Receive completion queue entry
00408  */
00409 static void txnic_complete_rqe ( struct txnic *vnic,
00410                                  struct txnic_cqe_rx *cqe ) {
00411         struct net_device *netdev = vnic->netdev;
00412         struct io_buffer *iobuf;
00413         unsigned int errop;
00414         unsigned int rq_idx;
00415         unsigned int rq_iobuf_idx;
00416         size_t apad_len;
00417         size_t len;
00418 
00419         /* Parse completion */
00420         errop = cqe->errop;
00421         apad_len = TXNIC_CQE_RX_APAD_LEN ( cqe->apad );
00422         len = le16_to_cpu ( cqe->len );
00423 
00424         /* Get next receive I/O buffer */
00425         rq_idx = ( vnic->rq.cons++ % TXNIC_RQES );
00426         rq_iobuf_idx = ( rq_idx % TXNIC_RQ_FILL );
00427         iobuf = vnic->rq.iobuf[rq_iobuf_idx];
00428         vnic->rq.iobuf[rq_iobuf_idx] = NULL;
00429 
00430         /* Populate I/O buffer */
00431         iob_reserve ( iobuf, apad_len );
00432         iob_put ( iobuf, len );
00433 
00434         /* Hand off to network stack */
00435         if ( errop ) {
00436                 DBGC ( vnic, "TXNIC %s RQE %#03x error (length %zd, errop "
00437                        "%#02x)\n", vnic->name, rq_idx, len, errop );
00438                 netdev_rx_err ( netdev, iobuf, -EIO );
00439         } else {
00440                 DBGC2 ( vnic, "TXNIC %s RQE %#03x complete (length %zd)\n",
00441                         vnic->name, rq_idx, len );
00442                 netdev_rx ( netdev, iobuf );
00443         }
00444 }
00445 
00446 /******************************************************************************
00447  *
00448  * Completion queue
00449  *
00450  ******************************************************************************
00451  */
00452 
00453 /**
00454  * Create completion queue
00455  *
00456  * @v vnic              Virtual NIC
00457  * @ret rc              Return status code
00458  */
00459 static int txnic_create_cq ( struct txnic *vnic ) {
00460 
00461         /* Reset completion queue */
00462         vnic->cq.cons = 0;
00463         writeq ( TXNIC_QS_CQ_CFG_RESET, ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
00464 
00465         /* Configure and enable completion queue */
00466         writeq ( user_to_phys ( vnic->cq.cqe, 0 ),
00467                  ( vnic->regs + TXNIC_QS_CQ_BASE(0) ) );
00468         writeq ( ( TXNIC_QS_CQ_CFG_ENA | TXNIC_QS_CQ_CFG_QSIZE_256 ),
00469                  ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
00470 
00471         DBGC ( vnic, "TXNIC %s CQ at [%08lx,%08lx)\n",
00472                vnic->name, user_to_phys ( vnic->cq.cqe, 0 ),
00473                user_to_phys ( vnic->cq.cqe, TXNIC_CQ_SIZE ) );
00474         return 0;
00475 }
00476 
00477 /**
00478  * Disable completion queue
00479  *
00480  * @v vnic              Virtual NIC
00481  * @ret rc              Return status code
00482  */
00483 static int txnic_disable_cq ( struct txnic *vnic ) {
00484         uint64_t cfg;
00485         unsigned int i;
00486 
00487         /* Disable completion queue */
00488         writeq ( 0, ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
00489 
00490         /* Wait for completion queue to be disabled */
00491         for ( i = 0 ; i < TXNIC_CQ_DISABLE_MAX_WAIT_MS ; i++ ) {
00492 
00493                 /* Check if completion queue is disabled */
00494                 cfg = readq ( vnic->regs + TXNIC_QS_CQ_CFG(0) );
00495                 if ( ! ( cfg & TXNIC_QS_CQ_CFG_ENA ) )
00496                         return 0;
00497 
00498                 /* Delay */
00499                 mdelay ( 1 );
00500         }
00501 
00502         DBGC ( vnic, "TXNIC %s CQ disable timed out\n", vnic->name );
00503         return -ETIMEDOUT;
00504 }
00505 
00506 /**
00507  * Destroy completion queue
00508  *
00509  * @v vnic              Virtual NIC
00510  */
00511 static void txnic_destroy_cq ( struct txnic *vnic ) {
00512         int rc;
00513 
00514         /* Disable completion queue */
00515         if ( ( rc = txnic_disable_cq ( vnic ) ) != 0 ) {
00516                 /* Leak memory; there's nothing else we can do */
00517                 return;
00518         }
00519 
00520         /* Reset completion queue */
00521         writeq ( TXNIC_QS_CQ_CFG_RESET, ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
00522 }
00523 
00524 /**
00525  * Poll completion queue
00526  *
00527  * @v vnic              Virtual NIC
00528  */
00529 static void txnic_poll_cq ( struct txnic *vnic ) {
00530         union txnic_cqe cqe;
00531         uint64_t status;
00532         size_t offset;
00533         unsigned int qcount;
00534         unsigned int cq_idx;
00535         unsigned int i;
00536 
00537         /* Get number of completions */
00538         status = readq ( vnic->regs + TXNIC_QS_CQ_STATUS(0) );
00539         qcount = TXNIC_QS_CQ_STATUS_QCOUNT ( status );
00540         if ( ! qcount )
00541                 return;
00542 
00543         /* Process completion queue entries */
00544         for ( i = 0 ; i < qcount ; i++ ) {
00545 
00546                 /* Get completion queue entry */
00547                 cq_idx = ( vnic->cq.cons++ % TXNIC_CQES );
00548                 offset = ( cq_idx * TXNIC_CQ_STRIDE );
00549                 copy_from_user ( &cqe, vnic->cq.cqe, offset, sizeof ( cqe ) );
00550 
00551                 /* Process completion queue entry */
00552                 switch ( cqe.common.cqe_type ) {
00553                 case TXNIC_CQE_TYPE_SEND:
00554                         txnic_complete_sqe ( vnic, &cqe.send );
00555                         break;
00556                 case TXNIC_CQE_TYPE_RX:
00557                         txnic_complete_rqe ( vnic, &cqe.rx );
00558                         break;
00559                 default:
00560                         DBGC ( vnic, "TXNIC %s unknown completion type %d\n",
00561                                vnic->name, cqe.common.cqe_type );
00562                         DBGC_HDA ( vnic, user_to_phys ( vnic->cq.cqe, offset ),
00563                                    &cqe, sizeof ( cqe ) );
00564                         break;
00565                 }
00566         }
00567 
00568         /* Ring doorbell */
00569         writeq ( qcount, ( vnic->regs + TXNIC_QS_CQ_DOOR(0) ) );
00570 }
00571 
00572 /******************************************************************************
00573  *
00574  * Virtual NIC
00575  *
00576  ******************************************************************************
00577  */
00578 
00579 /**
00580  * Open virtual NIC
00581  *
00582  * @v vnic              Virtual NIC
00583  * @ret rc              Return status code
00584  */
00585 static int txnic_open ( struct txnic *vnic ) {
00586         int rc;
00587 
00588         /* Create completion queue */
00589         if ( ( rc = txnic_create_cq ( vnic ) ) != 0 )
00590                 goto err_create_cq;
00591 
00592         /* Create send queue */
00593         if ( ( rc = txnic_create_sq ( vnic ) ) != 0 )
00594                 goto err_create_sq;
00595 
00596         /* Create receive queue */
00597         if ( ( rc = txnic_create_rq ( vnic ) ) != 0 )
00598                 goto err_create_rq;
00599 
00600         /* Refill receive queue */
00601         txnic_refill_rq ( vnic );
00602 
00603         return 0;
00604 
00605         txnic_destroy_rq ( vnic );
00606  err_create_rq:
00607         txnic_destroy_sq ( vnic );
00608  err_create_sq:
00609         txnic_destroy_cq ( vnic );
00610  err_create_cq:
00611         return rc;
00612 }
00613 
00614 /**
00615  * Close virtual NIC
00616  *
00617  * @v vnic              Virtual NIC
00618  */
00619 static void txnic_close ( struct txnic *vnic ) {
00620 
00621         /* Destroy receive queue */
00622         txnic_destroy_rq ( vnic );
00623 
00624         /* Destroy send queue */
00625         txnic_destroy_sq ( vnic );
00626 
00627         /* Destroy completion queue */
00628         txnic_destroy_cq ( vnic );
00629 }
00630 
00631 /**
00632  * Poll virtual NIC
00633  *
00634  * @v vnic              Virtual NIC
00635  */
00636 static void txnic_poll ( struct txnic *vnic ) {
00637 
00638         /* Poll completion queue */
00639         txnic_poll_cq ( vnic );
00640 
00641         /* Refill receive queue */
00642         txnic_refill_rq ( vnic );
00643 }
00644 
00645 /**
00646  * Allocate virtual NIC
00647  *
00648  * @v dev               Underlying device
00649  * @v membase           Register base address
00650  * @ret vnic            Virtual NIC, or NULL on failure
00651  */
00652 static struct txnic * txnic_alloc ( struct device *dev,
00653                                     unsigned long membase ) {
00654         struct net_device *netdev;
00655         struct txnic *vnic;
00656 
00657         /* Allocate network device */
00658         netdev = alloc_etherdev ( sizeof ( *vnic ) );
00659         if ( ! netdev )
00660                 goto err_alloc_netdev;
00661         netdev->dev = dev;
00662         vnic = netdev->priv;
00663         vnic->netdev = netdev;
00664         vnic->name = dev->name;
00665 
00666         /* Allow caller to reuse netdev->priv.  (The generic virtual
00667          * NIC code never assumes that netdev->priv==vnic.)
00668          */
00669         netdev->priv = NULL;
00670 
00671         /* Allocate completion queue */
00672         vnic->cq.cqe = umalloc ( TXNIC_CQ_SIZE );
00673         if ( ! vnic->cq.cqe )
00674                 goto err_alloc_cq;
00675 
00676         /* Allocate send queue */
00677         vnic->sq.sqe = umalloc ( TXNIC_SQ_SIZE );
00678         if ( ! vnic->sq.sqe )
00679                 goto err_alloc_sq;
00680 
00681         /* Allocate receive queue */
00682         vnic->rq.rqe = umalloc ( TXNIC_RQ_SIZE );
00683         if ( ! vnic->rq.rqe )
00684                 goto err_alloc_rq;
00685 
00686         /* Map registers */
00687         vnic->regs = ioremap ( membase, TXNIC_VF_BAR_SIZE );
00688         if ( ! vnic->regs )
00689                 goto err_ioremap;
00690 
00691         return vnic;
00692 
00693         iounmap ( vnic->regs );
00694  err_ioremap:
00695         ufree ( vnic->rq.rqe );
00696  err_alloc_rq:
00697         ufree ( vnic->sq.sqe );
00698  err_alloc_sq:
00699         ufree ( vnic->cq.cqe );
00700  err_alloc_cq:
00701         netdev_nullify ( netdev );
00702         netdev_put ( netdev );
00703  err_alloc_netdev:
00704         return NULL;
00705 }
00706 
00707 /**
00708  * Free virtual NIC
00709  *
00710  * @v vnic              Virtual NIC
00711  */
00712 static void txnic_free ( struct txnic *vnic ) {
00713         struct net_device *netdev = vnic->netdev;
00714 
00715         /* Unmap registers */
00716         iounmap ( vnic->regs );
00717 
00718         /* Free receive queue */
00719         ufree ( vnic->rq.rqe );
00720 
00721         /* Free send queue */
00722         ufree ( vnic->sq.sqe );
00723 
00724         /* Free completion queue */
00725         ufree ( vnic->cq.cqe );
00726 
00727         /* Free network device */
00728         netdev_nullify ( netdev );
00729         netdev_put ( netdev );
00730 }
00731 
00732 /******************************************************************************
00733  *
00734  * Logical MAC virtual NICs
00735  *
00736  ******************************************************************************
00737  */
00738 
00739 /**
00740  * Show LMAC diagnostics (for debugging)
00741  *
00742  * @v lmac              Logical MAC
00743  */
00744 static __attribute__ (( unused )) void
00745 txnic_lmac_diag ( struct txnic_lmac *lmac ) {
00746         struct txnic *vnic = lmac->vnic;
00747         uint64_t status1;
00748         uint64_t status2;
00749         uint64_t br_status1;
00750         uint64_t br_status2;
00751         uint64_t br_algn_status;
00752         uint64_t br_pmd_status;
00753         uint64_t an_status;
00754 
00755         /* Read status (clearing latching bits) */
00756         writeq ( BGX_SPU_STATUS1_RCV_LNK, ( lmac->regs + BGX_SPU_STATUS1 ) );
00757         writeq ( BGX_SPU_STATUS2_RCVFLT, ( lmac->regs + BGX_SPU_STATUS2 ) );
00758         status1 = readq ( lmac->regs + BGX_SPU_STATUS1 );
00759         status2 = readq ( lmac->regs + BGX_SPU_STATUS2 );
00760         DBGC ( vnic, "TXNIC %s SPU %02llx:%04llx%s%s%s\n",
00761                vnic->name, status1, status2,
00762                ( ( status1 & BGX_SPU_STATUS1_FLT ) ? " FLT" : "" ),
00763                ( ( status1 & BGX_SPU_STATUS1_RCV_LNK ) ? " RCV_LNK" : "" ),
00764                ( ( status2 & BGX_SPU_STATUS2_RCVFLT ) ? " RCVFLT" : "" ) );
00765 
00766         /* Read BASE-R status (clearing latching bits) */
00767         writeq ( ( BGX_SPU_BR_STATUS2_LATCHED_LOCK |
00768                    BGX_SPU_BR_STATUS2_LATCHED_BER ),
00769                  ( lmac->regs + BGX_SPU_BR_STATUS2 ) );
00770         br_status1 = readq ( lmac->regs + BGX_SPU_BR_STATUS1 );
00771         br_status2 = readq ( lmac->regs + BGX_SPU_BR_STATUS2 );
00772         DBGC ( vnic, "TXNIC %s BR %04llx:%04llx%s%s%s%s%s\n",
00773                vnic->name, br_status2, br_status2,
00774                ( ( br_status1 & BGX_SPU_BR_STATUS1_RCV_LNK ) ? " RCV_LNK" : ""),
00775                ( ( br_status1 & BGX_SPU_BR_STATUS1_HI_BER ) ? " HI_BER" : "" ),
00776                ( ( br_status1 & BGX_SPU_BR_STATUS1_BLK_LOCK ) ?
00777                  " BLK_LOCK" : "" ),
00778                ( ( br_status2 & BGX_SPU_BR_STATUS2_LATCHED_LOCK ) ?
00779                  " LATCHED_LOCK" : "" ),
00780                ( ( br_status2 & BGX_SPU_BR_STATUS2_LATCHED_BER ) ?
00781                  " LATCHED_BER" : "" ) );
00782 
00783         /* Read BASE-R alignment status */
00784         br_algn_status = readq ( lmac->regs + BGX_SPU_BR_ALGN_STATUS );
00785         DBGC ( vnic, "TXNIC %s BR ALGN %016llx%s\n", vnic->name, br_algn_status,
00786                ( ( br_algn_status & BGX_SPU_BR_ALGN_STATUS_ALIGND ) ?
00787                  " ALIGND" : "" ) );
00788 
00789         /* Read BASE-R link training status */
00790         br_pmd_status = readq ( lmac->regs + BGX_SPU_BR_PMD_STATUS );
00791         DBGC ( vnic, "TXNIC %s BR PMD %04llx\n", vnic->name, br_pmd_status );
00792 
00793         /* Read autonegotiation status (clearing latching bits) */
00794         writeq ( ( BGX_SPU_AN_STATUS_PAGE_RX | BGX_SPU_AN_STATUS_LINK_STATUS ),
00795                  ( lmac->regs + BGX_SPU_AN_STATUS ) );
00796         an_status = readq ( lmac->regs + BGX_SPU_AN_STATUS );
00797         DBGC ( vnic, "TXNIC %s BR AN %04llx%s%s%s%s%s\n", vnic->name, an_status,
00798                ( ( an_status & BGX_SPU_AN_STATUS_XNP_STAT ) ? " XNP_STAT" : ""),
00799                ( ( an_status & BGX_SPU_AN_STATUS_PAGE_RX ) ? " PAGE_RX" : "" ),
00800                ( ( an_status & BGX_SPU_AN_STATUS_AN_COMPLETE ) ?
00801                  " AN_COMPLETE" : "" ),
00802                ( ( an_status & BGX_SPU_AN_STATUS_LINK_STATUS ) ?
00803                  " LINK_STATUS" : "" ),
00804                ( ( an_status & BGX_SPU_AN_STATUS_LP_AN_ABLE ) ?
00805                  " LP_AN_ABLE" : "" ) );
00806 
00807         /* Read transmit statistics */
00808         DBGC ( vnic, "TXNIC %s TXF xc %#llx xd %#llx mc %#llx sc %#llx ok "
00809                "%#llx bc %#llx mc %#llx un %#llx pa %#llx\n", vnic->name,
00810                readq ( lmac->regs + BGX_CMR_TX_STAT0 ),
00811                readq ( lmac->regs + BGX_CMR_TX_STAT1 ),
00812                readq ( lmac->regs + BGX_CMR_TX_STAT2 ),
00813                readq ( lmac->regs + BGX_CMR_TX_STAT3 ),
00814                readq ( lmac->regs + BGX_CMR_TX_STAT5 ),
00815                readq ( lmac->regs + BGX_CMR_TX_STAT14 ),
00816                readq ( lmac->regs + BGX_CMR_TX_STAT15 ),
00817                readq ( lmac->regs + BGX_CMR_TX_STAT16 ),
00818                readq ( lmac->regs + BGX_CMR_TX_STAT17 ) );
00819         DBGC ( vnic, "TXNIC %s TXB ok %#llx hist %#llx:%#llx:%#llx:%#llx:"
00820                "%#llx:%#llx:%#llx:%#llx\n", vnic->name,
00821                readq ( lmac->regs + BGX_CMR_TX_STAT4 ),
00822                readq ( lmac->regs + BGX_CMR_TX_STAT6 ),
00823                readq ( lmac->regs + BGX_CMR_TX_STAT7 ),
00824                readq ( lmac->regs + BGX_CMR_TX_STAT8 ),
00825                readq ( lmac->regs + BGX_CMR_TX_STAT9 ),
00826                readq ( lmac->regs + BGX_CMR_TX_STAT10 ),
00827                readq ( lmac->regs + BGX_CMR_TX_STAT11 ),
00828                readq ( lmac->regs + BGX_CMR_TX_STAT12 ),
00829                readq ( lmac->regs + BGX_CMR_TX_STAT13 ) );
00830 
00831         /* Read receive statistics */
00832         DBGC ( vnic, "TXNIC %s RXF ok %#llx pa %#llx nm %#llx ov %#llx er "
00833                "%#llx nc %#llx\n", vnic->name,
00834                readq ( lmac->regs + BGX_CMR_RX_STAT0 ),
00835                readq ( lmac->regs + BGX_CMR_RX_STAT2 ),
00836                readq ( lmac->regs + BGX_CMR_RX_STAT4 ),
00837                readq ( lmac->regs + BGX_CMR_RX_STAT6 ),
00838                readq ( lmac->regs + BGX_CMR_RX_STAT8 ),
00839                readq ( lmac->regs + BGX_CMR_RX_STAT9 ) );
00840         DBGC ( vnic, "TXNIC %s RXB ok %#llx pa %#llx nm %#llx ov %#llx nc "
00841                "%#llx\n", vnic->name,
00842                readq ( lmac->regs + BGX_CMR_RX_STAT1 ),
00843                readq ( lmac->regs + BGX_CMR_RX_STAT3 ),
00844                readq ( lmac->regs + BGX_CMR_RX_STAT5 ),
00845                readq ( lmac->regs + BGX_CMR_RX_STAT7 ),
00846                readq ( lmac->regs + BGX_CMR_RX_STAT10 ) );
00847 }
00848 
00849 /**
00850  * Update LMAC link state
00851  *
00852  * @v lmac              Logical MAC
00853  */
00854 static void txnic_lmac_update_link ( struct txnic_lmac *lmac ) {
00855         struct txnic *vnic = lmac->vnic;
00856         struct net_device *netdev = vnic->netdev;
00857         uint64_t status1;
00858 
00859         /* Read status (clearing latching bits) */
00860         writeq ( BGX_SPU_STATUS1_RCV_LNK, ( lmac->regs + BGX_SPU_STATUS1 ) );
00861         status1 = readq ( lmac->regs + BGX_SPU_STATUS1 );
00862 
00863         /* Report link status */
00864         if ( status1 & BGX_SPU_STATUS1_RCV_LNK ) {
00865                 netdev_link_up ( netdev );
00866         } else {
00867                 netdev_link_down ( netdev );
00868         }
00869 }
00870 
00871 /**
00872  * Poll LMAC link state
00873  *
00874  * @v lmac              Logical MAC
00875  */
00876 static void txnic_lmac_poll_link ( struct txnic_lmac *lmac ) {
00877         struct txnic *vnic = lmac->vnic;
00878         uint64_t intr;
00879 
00880         /* Get interrupt status */
00881         intr = readq ( lmac->regs + BGX_SPU_INT );
00882         if ( ! intr )
00883                 return;
00884         DBGC ( vnic, "TXNIC %s INT %04llx%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
00885                vnic->name, intr,
00886                ( ( intr & BGX_SPU_INT_TRAINING_FAIL ) ? " TRAINING_FAIL" : "" ),
00887                ( ( intr & BGX_SPU_INT_TRAINING_DONE ) ? " TRAINING_DONE" : "" ),
00888                ( ( intr & BGX_SPU_INT_AN_COMPLETE ) ? " AN_COMPLETE" : "" ),
00889                ( ( intr & BGX_SPU_INT_AN_LINK_GOOD ) ? " AN_LINK_GOOD" : "" ),
00890                ( ( intr & BGX_SPU_INT_AN_PAGE_RX ) ? " AN_PAGE_RX" : "" ),
00891                ( ( intr & BGX_SPU_INT_FEC_UNCORR ) ? " FEC_UNCORR" : "" ),
00892                ( ( intr & BGX_SPU_INT_FEC_CORR ) ? " FEC_CORR" : "" ),
00893                ( ( intr & BGX_SPU_INT_BIP_ERR ) ? " BIP_ERR" : "" ),
00894                ( ( intr & BGX_SPU_INT_DBG_SYNC ) ? " DBG_SYNC" : "" ),
00895                ( ( intr & BGX_SPU_INT_ALGNLOS ) ? " ALGNLOS" : "" ),
00896                ( ( intr & BGX_SPU_INT_SYNLOS ) ? " SYNLOS" : "" ),
00897                ( ( intr & BGX_SPU_INT_BITLCKLS ) ? " BITLCKLS" : "" ),
00898                ( ( intr & BGX_SPU_INT_ERR_BLK ) ? " ERR_BLK" : "" ),
00899                ( ( intr & BGX_SPU_INT_RX_LINK_DOWN ) ? " RX_LINK_DOWN" : "" ),
00900                ( ( intr & BGX_SPU_INT_RX_LINK_UP ) ? " RX_LINK_UP" : "" ) );
00901 
00902         /* Clear interrupt status */
00903         writeq ( intr, ( lmac->regs + BGX_SPU_INT ) );
00904 
00905         /* Update link state */
00906         txnic_lmac_update_link ( lmac );
00907 }
00908 
00909 /**
00910  * Reset LMAC
00911  *
00912  * @v lmac              Logical MAC
00913  */
00914 static void txnic_lmac_reset ( struct txnic_lmac *lmac ) {
00915         struct txnic_bgx *bgx = lmac->bgx;
00916         struct txnic_pf *pf = bgx->pf;
00917         void *qsregs = ( pf->regs + TXNIC_PF_QS ( lmac->idx ) );
00918 
00919         /* There is no reset available for the physical function
00920          * aspects of a virtual NIC; we have to explicitly reload a
00921          * sensible set of default values.
00922          */
00923         writeq ( 0, ( qsregs + TXNIC_PF_QS_CFG ) );
00924         writeq ( 0, ( qsregs + TXNIC_PF_QS_RQ_CFG(0) ) );
00925         writeq ( 0, ( qsregs + TXNIC_PF_QS_RQ_DROP_CFG(0) ) );
00926         writeq ( 0, ( qsregs + TXNIC_PF_QS_RQ_BP_CFG(0) ) );
00927         writeq ( 0, ( qsregs + TXNIC_PF_QS_SQ_CFG(0) ) );
00928 }
00929 
00930 /**
00931  * Open network device
00932  *
00933  * @v netdev            Network device
00934  * @ret rc              Return status code
00935  */
00936 static int txnic_lmac_open ( struct net_device *netdev ) {
00937         struct txnic_lmac *lmac = netdev->priv;
00938         struct txnic_bgx *bgx = lmac->bgx;
00939         struct txnic_pf *pf = bgx->pf;
00940         struct txnic *vnic = lmac->vnic;
00941         unsigned int vnic_idx = lmac->idx;
00942         unsigned int chan_idx = TXNIC_CHAN_IDX ( vnic_idx );
00943         unsigned int tl4_idx = TXNIC_TL4_IDX ( vnic_idx );
00944         unsigned int tl3_idx = TXNIC_TL3_IDX ( vnic_idx );
00945         unsigned int tl2_idx = TXNIC_TL2_IDX ( vnic_idx );
00946         void *lmregs = ( pf->regs + TXNIC_PF_LMAC ( vnic_idx ) );
00947         void *chregs = ( pf->regs + TXNIC_PF_CHAN ( chan_idx ) );
00948         void *qsregs = ( pf->regs + TXNIC_PF_QS ( vnic_idx ) );
00949         size_t max_pkt_size;
00950         int rc;
00951 
00952         /* Configure channel/match parse indices */
00953         writeq ( ( TXNIC_PF_MPI_CFG_VNIC ( vnic_idx ) |
00954                    TXNIC_PF_MPI_CFG_RSSI_BASE ( vnic_idx ) ),
00955                  ( TXNIC_PF_MPI_CFG ( vnic_idx ) + pf->regs ) );
00956         writeq ( ( TXNIC_PF_RSSI_RQ_RQ_QS ( vnic_idx ) ),
00957                  ( TXNIC_PF_RSSI_RQ ( vnic_idx ) + pf->regs ) );
00958 
00959         /* Configure LMAC */
00960         max_pkt_size = ( netdev->max_pkt_len + 4 /* possible VLAN */ );
00961         writeq ( ( TXNIC_PF_LMAC_CFG_ADJUST_DEFAULT |
00962                    TXNIC_PF_LMAC_CFG_MIN_PKT_SIZE ( ETH_ZLEN ) ),
00963                  ( TXNIC_PF_LMAC_CFG + lmregs ) );
00964         writeq ( ( TXNIC_PF_LMAC_CFG2_MAX_PKT_SIZE ( max_pkt_size ) ),
00965                  ( TXNIC_PF_LMAC_CFG2 + lmregs ) );
00966         writeq ( ( TXNIC_PF_LMAC_CREDIT_CC_UNIT_CNT_DEFAULT |
00967                    TXNIC_PF_LMAC_CREDIT_CC_PACKET_CNT_DEFAULT |
00968                    TXNIC_PF_LMAC_CREDIT_CC_ENABLE ),
00969                  ( TXNIC_PF_LMAC_CREDIT + lmregs ) );
00970 
00971         /* Configure channels */
00972         writeq ( ( TXNIC_PF_CHAN_TX_CFG_BP_ENA ),
00973                  ( TXNIC_PF_CHAN_TX_CFG + chregs ) );
00974         writeq ( ( TXNIC_PF_CHAN_RX_CFG_CPI_BASE ( vnic_idx ) ),
00975                  ( TXNIC_PF_CHAN_RX_CFG + chregs ) );
00976         writeq ( ( TXNIC_PF_CHAN_RX_BP_CFG_ENA |
00977                    TXNIC_PF_CHAN_RX_BP_CFG_BPID ( vnic_idx ) ),
00978                  ( TXNIC_PF_CHAN_RX_BP_CFG + chregs ) );
00979 
00980         /* Configure traffic limiters */
00981         writeq ( ( TXNIC_PF_TL2_CFG_RR_QUANTUM_DEFAULT ),
00982                  ( TXNIC_PF_TL2_CFG ( tl2_idx ) + pf->regs ) );
00983         writeq ( ( TXNIC_PF_TL3_CFG_RR_QUANTUM_DEFAULT ),
00984                  ( TXNIC_PF_TL3_CFG ( tl3_idx ) + pf->regs ) );
00985         writeq ( ( TXNIC_PF_TL3_CHAN_CHAN ( chan_idx ) ),
00986                  ( TXNIC_PF_TL3_CHAN ( tl3_idx ) + pf->regs ) );
00987         writeq ( ( TXNIC_PF_TL4_CFG_SQ_QS ( vnic_idx ) |
00988                    TXNIC_PF_TL4_CFG_RR_QUANTUM_DEFAULT ),
00989                  ( TXNIC_PF_TL4_CFG ( tl4_idx ) + pf->regs ) );
00990 
00991         /* Configure send queue */
00992         writeq ( ( TXNIC_PF_QS_SQ_CFG_CQ_QS ( vnic_idx ) ),
00993                  ( TXNIC_PF_QS_SQ_CFG(0) + qsregs ) );
00994         writeq ( ( TXNIC_PF_QS_SQ_CFG2_TL4 ( tl4_idx ) ),
00995                  ( TXNIC_PF_QS_SQ_CFG2(0) + qsregs ) );
00996 
00997         /* Configure receive queue */
00998         writeq ( ( TXNIC_PF_QS_RQ_CFG_CACHING_ALL |
00999                    TXNIC_PF_QS_RQ_CFG_CQ_QS ( vnic_idx ) |
01000                    TXNIC_PF_QS_RQ_CFG_RBDR_CONT_QS ( vnic_idx ) |
01001                    TXNIC_PF_QS_RQ_CFG_RBDR_STRT_QS ( vnic_idx ) ),
01002                  ( TXNIC_PF_QS_RQ_CFG(0) + qsregs ) );
01003         writeq ( ( TXNIC_PF_QS_RQ_BP_CFG_RBDR_BP_ENA |
01004                    TXNIC_PF_QS_RQ_BP_CFG_CQ_BP_ENA |
01005                    TXNIC_PF_QS_RQ_BP_CFG_BPID ( vnic_idx ) ),
01006                  ( TXNIC_PF_QS_RQ_BP_CFG(0) + qsregs ) );
01007 
01008         /* Enable queue set */
01009         writeq ( ( TXNIC_PF_QS_CFG_ENA | TXNIC_PF_QS_CFG_VNIC ( vnic_idx ) ),
01010                  ( TXNIC_PF_QS_CFG + qsregs ) );
01011 
01012         /* Open virtual NIC */
01013         if ( ( rc = txnic_open ( vnic ) ) != 0 )
01014                 goto err_open;
01015 
01016         /* Update link state */
01017         txnic_lmac_update_link ( lmac );
01018 
01019         return 0;
01020 
01021         txnic_close ( vnic );
01022  err_open:
01023         writeq ( 0, ( qsregs + TXNIC_PF_QS_CFG ) );
01024         return rc;
01025 }
01026 
01027 /**
01028  * Close network device
01029  *
01030  * @v netdev            Network device
01031  */
01032 static void txnic_lmac_close ( struct net_device *netdev ) {
01033         struct txnic_lmac *lmac = netdev->priv;
01034         struct txnic_bgx *bgx = lmac->bgx;
01035         struct txnic_pf *pf = bgx->pf;
01036         struct txnic *vnic = lmac->vnic;
01037         void *qsregs = ( pf->regs + TXNIC_PF_QS ( lmac->idx ) );
01038 
01039         /* Close virtual NIC */
01040         txnic_close ( vnic );
01041 
01042         /* Disable queue set */
01043         writeq ( 0, ( qsregs + TXNIC_PF_QS_CFG ) );
01044 }
01045 
01046 /**
01047  * Transmit packet
01048  *
01049  * @v netdev            Network device
01050  * @v iobuf             I/O buffer
01051  * @ret rc              Return status code
01052  */
01053 static int txnic_lmac_transmit ( struct net_device *netdev,
01054                                  struct io_buffer *iobuf ) {
01055         struct txnic_lmac *lmac = netdev->priv;
01056         struct txnic *vnic = lmac->vnic;
01057 
01058         return txnic_send ( vnic, iobuf );
01059 }
01060 
01061 /**
01062  * Poll network device
01063  *
01064  * @v netdev            Network device
01065  */
01066 static void txnic_lmac_poll ( struct net_device *netdev ) {
01067         struct txnic_lmac *lmac = netdev->priv;
01068         struct txnic *vnic = lmac->vnic;
01069 
01070         /* Poll virtual NIC */
01071         txnic_poll ( vnic );
01072 
01073         /* Poll link state */
01074         txnic_lmac_poll_link ( lmac );
01075 }
01076 
01077 /** Network device operations */
01078 static struct net_device_operations txnic_lmac_operations = {
01079         .open = txnic_lmac_open,
01080         .close = txnic_lmac_close,
01081         .transmit = txnic_lmac_transmit,
01082         .poll = txnic_lmac_poll,
01083 };
01084 
01085 /**
01086  * Probe logical MAC virtual NIC
01087  *
01088  * @v lmac              Logical MAC
01089  * @ret rc              Return status code
01090  */
01091 static int txnic_lmac_probe ( struct txnic_lmac *lmac ) {
01092         struct txnic_bgx *bgx = lmac->bgx;
01093         struct txnic_pf *pf = bgx->pf;
01094         struct txnic *vnic;
01095         struct net_device *netdev;
01096         unsigned long membase;
01097         int rc;
01098 
01099         /* Sanity check */
01100         assert ( lmac->vnic == NULL );
01101 
01102         /* Calculate register base address */
01103         membase = ( pf->vf_membase + ( lmac->idx * pf->vf_stride ) );
01104 
01105         /* Allocate and initialise network device */
01106         vnic = txnic_alloc ( &bgx->pci->dev, membase );
01107         if ( ! vnic ) {
01108                 rc = -ENOMEM;
01109                 goto err_alloc;
01110         }
01111         netdev = vnic->netdev;
01112         netdev_init ( netdev, &txnic_lmac_operations );
01113         netdev->priv = lmac;
01114         lmac->vnic = vnic;
01115 
01116         /* Reset device */
01117         txnic_lmac_reset ( lmac );
01118 
01119         /* Set MAC address */
01120         memcpy ( netdev->hw_addr, lmac->mac.raw, ETH_ALEN );
01121 
01122         /* Register network device */
01123         if ( ( rc = register_netdev ( netdev ) ) != 0 )
01124                 goto err_register;
01125         vnic->name = netdev->name;
01126         DBGC ( TXNICCOL ( pf ), "TXNIC %d/%d/%d is %s (%s)\n", pf->node,
01127                bgx->idx, lmac->idx, vnic->name, eth_ntoa ( lmac->mac.raw ) );
01128 
01129         /* Update link state */
01130         txnic_lmac_update_link ( lmac );
01131 
01132         return 0;
01133 
01134         unregister_netdev ( netdev );
01135  err_register:
01136         txnic_lmac_reset ( lmac );
01137         txnic_free ( vnic );
01138         lmac->vnic = NULL;
01139  err_alloc:
01140         return rc;
01141 }
01142 
01143 /**
01144  * Remove logical MAC virtual NIC
01145  *
01146  * @v lmac              Logical MAC
01147  */
01148 static void txnic_lmac_remove ( struct txnic_lmac *lmac ) {
01149         uint64_t config;
01150 
01151         /* Sanity check */
01152         assert ( lmac->vnic != NULL );
01153 
01154         /* Disable packet receive and transmit */
01155         config = readq ( lmac->regs + BGX_CMR_CONFIG );
01156         config &= ~( BGX_CMR_CONFIG_DATA_PKT_TX_EN |
01157                      BGX_CMR_CONFIG_DATA_PKT_RX_EN );
01158         writeq ( config, ( lmac->regs + BGX_CMR_CONFIG ) );
01159 
01160         /* Unregister network device */
01161         unregister_netdev ( lmac->vnic->netdev );
01162 
01163         /* Reset device */
01164         txnic_lmac_reset ( lmac );
01165 
01166         /* Free virtual NIC */
01167         txnic_free ( lmac->vnic );
01168         lmac->vnic = NULL;
01169 }
01170 
01171 /**
01172  * Probe all LMACs on a BGX Ethernet interface
01173  *
01174  * @v pf                Physical function
01175  * @v bgx               BGX Ethernet interface
01176  * @ret rc              Return status code
01177  */
01178 static int txnic_lmac_probe_all ( struct txnic_pf *pf, struct txnic_bgx *bgx ) {
01179         unsigned int bgx_idx;
01180         int lmac_idx;
01181         int count;
01182         int rc;
01183 
01184         /* Sanity checks */
01185         bgx_idx = bgx->idx;
01186         assert ( pf->node == bgx->node );
01187         assert ( pf->bgx[bgx_idx] == NULL );
01188         assert ( bgx->pf == NULL );
01189 
01190         /* Associate BGX with physical function */
01191         pf->bgx[bgx_idx] = bgx;
01192         bgx->pf = pf;
01193 
01194         /* Probe all LMACs */
01195         count = bgx->count;
01196         for ( lmac_idx = 0 ; lmac_idx < count ; lmac_idx++ ) {
01197                 if ( ( rc = txnic_lmac_probe ( &bgx->lmac[lmac_idx] ) ) != 0 )
01198                         goto err_probe;
01199         }
01200 
01201         return 0;
01202 
01203         lmac_idx = count;
01204  err_probe:
01205         for ( lmac_idx-- ; lmac_idx >= 0 ; lmac_idx-- )
01206                 txnic_lmac_remove ( &bgx->lmac[lmac_idx] );
01207         pf->bgx[bgx_idx] = NULL;
01208         bgx->pf = NULL;
01209         return rc;
01210 }
01211 
01212 /**
01213  * Remove all LMACs on a BGX Ethernet interface
01214  *
01215  * @v pf                Physical function
01216  * @v bgx               BGX Ethernet interface
01217  */
01218 static void txnic_lmac_remove_all ( struct txnic_pf *pf,
01219                                     struct txnic_bgx *bgx ) {
01220         unsigned int lmac_idx;
01221 
01222         /* Sanity checks */
01223         assert ( pf->bgx[bgx->idx] == bgx );
01224         assert ( bgx->pf == pf );
01225 
01226         /* Remove all LMACs */
01227         for ( lmac_idx = 0 ; lmac_idx < bgx->count ; lmac_idx++ )
01228                 txnic_lmac_remove ( &bgx->lmac[lmac_idx] );
01229 
01230         /* Disassociate BGX from physical function */
01231         pf->bgx[bgx->idx] = NULL;
01232         bgx->pf = NULL;
01233 }
01234 
01235 /******************************************************************************
01236  *
01237  * NIC physical function interface
01238  *
01239  ******************************************************************************
01240  */
01241 
01242 /**
01243  * Probe PCI device
01244  *
01245  * @v pci               PCI device
01246  * @ret rc              Return status code
01247  */
01248 static int txnic_pf_probe ( struct pci_device *pci ) {
01249         struct txnic_pf *pf;
01250         struct txnic_bgx *bgx;
01251         unsigned long membase;
01252         unsigned int i;
01253         int rc;
01254 
01255         /* Allocate and initialise structure */
01256         pf = zalloc ( sizeof ( *pf ) );
01257         if ( ! pf ) {
01258                 rc = -ENOMEM;
01259                 goto err_alloc;
01260         }
01261         pf->pci = pci;
01262         pci_set_drvdata ( pci, pf );
01263 
01264         /* Get base addresses */
01265         membase = pciea_bar_start ( pci, PCIEA_BEI_BAR_0 );
01266         pf->vf_membase = pciea_bar_start ( pci, PCIEA_BEI_VF_BAR_0 );
01267         pf->vf_stride = pciea_bar_size ( pci, PCIEA_BEI_VF_BAR_0 );
01268 
01269         /* Calculate node ID */
01270         pf->node = txnic_address_node ( membase );
01271         DBGC ( TXNICCOL ( pf ), "TXNIC %d/*/* PF %s at %#lx (VF %#lx+%#lx)\n",
01272                pf->node, pci->dev.name, membase, pf->vf_membase, pf->vf_stride);
01273 
01274         /* Fix up PCI device */
01275         adjust_pci_device ( pci );
01276 
01277         /* Map registers */
01278         pf->regs = ioremap ( membase, TXNIC_PF_BAR_SIZE );
01279         if ( ! pf->regs ) {
01280                 rc = -ENODEV;
01281                 goto err_ioremap;
01282         }
01283 
01284         /* Configure physical function */
01285         writeq ( TXNIC_PF_CFG_ENA, ( pf->regs + TXNIC_PF_CFG ) );
01286         writeq ( ( TXNIC_PF_BP_CFG_BP_POLL_ENA |
01287                    TXNIC_PF_BP_CFG_BP_POLL_DLY_DEFAULT ),
01288                  ( pf->regs + TXNIC_PF_BP_CFG ) );
01289         for ( i = 0 ; i < TXNIC_NUM_BGX ; i++ ) {
01290                 writeq ( ( TXNIC_PF_INTF_SEND_CFG_BLOCK_BGX |
01291                            TXNIC_PF_INTF_SEND_CFG_BLOCK ( i ) ),
01292                          ( pf->regs + TXNIC_PF_INTF_SEND_CFG ( i ) ) );
01293                 writeq ( ( TXNIC_PF_INTF_BP_CFG_BP_ENA |
01294                            TXNIC_PF_INTF_BP_CFG_BP_ID_BGX |
01295                            TXNIC_PF_INTF_BP_CFG_BP_ID ( i ) ),
01296                          ( pf->regs + TXNIC_PF_INTF_BP_CFG ( i ) ) );
01297         }
01298         writeq ( ( TXNIC_PF_PKIND_CFG_LENERR_EN |
01299                    TXNIC_PF_PKIND_CFG_MAXLEN_DISABLE |
01300                    TXNIC_PF_PKIND_CFG_MINLEN_DISABLE ),
01301                  ( pf->regs + TXNIC_PF_PKIND_CFG(0) ) );
01302 
01303         /* Add to list of physical functions */
01304         list_add_tail ( &pf->list, &txnic_pfs );
01305 
01306         /* Probe all LMACs, if applicable */
01307         list_for_each_entry ( bgx, &txnic_bgxs, list ) {
01308                 if ( bgx->node != pf->node )
01309                         continue;
01310                 if ( ( rc = txnic_lmac_probe_all ( pf, bgx ) ) != 0 )
01311                         goto err_probe;
01312         }
01313 
01314         return 0;
01315 
01316  err_probe:
01317         for ( i = 0 ; i < TXNIC_NUM_BGX ; i++ ) {
01318                 if ( pf->bgx[i] )
01319                         txnic_lmac_remove_all ( pf, pf->bgx[i] );
01320         }
01321         list_del ( &pf->list );
01322         writeq ( 0, ( pf->regs + TXNIC_PF_CFG ) );
01323         iounmap ( pf->regs );
01324  err_ioremap:
01325         free ( pf );
01326  err_alloc:
01327         return rc;
01328 }
01329 
01330 /**
01331  * Remove PCI device
01332  *
01333  * @v pci               PCI device
01334  */
01335 static void txnic_pf_remove ( struct pci_device *pci ) {
01336         struct txnic_pf *pf = pci_get_drvdata ( pci );
01337         unsigned int i;
01338 
01339         /* Remove all LMACs, if applicable */
01340         for ( i = 0 ; i < TXNIC_NUM_BGX ; i++ ) {
01341                 if ( pf->bgx[i] )
01342                         txnic_lmac_remove_all ( pf, pf->bgx[i] );
01343         }
01344 
01345         /* Remove from list of physical functions */
01346         list_del ( &pf->list );
01347 
01348         /* Unmap registers */
01349         iounmap ( pf->regs );
01350 
01351         /* Free physical function */
01352         free ( pf );
01353 }
01354 
01355 /** NIC physical function PCI device IDs */
01356 static struct pci_device_id txnic_pf_ids[] = {
01357         PCI_ROM ( 0x177d, 0xa01e, "thunder-pf", "ThunderX NIC PF", 0 ),
01358 };
01359 
01360 /** NIC physical function PCI driver */
01361 struct pci_driver txnic_pf_driver __pci_driver = {
01362         .ids = txnic_pf_ids,
01363         .id_count = ( sizeof ( txnic_pf_ids ) / sizeof ( txnic_pf_ids[0] ) ),
01364         .probe = txnic_pf_probe,
01365         .remove = txnic_pf_remove,
01366 };
01367 
01368 /******************************************************************************
01369  *
01370  * BGX interface
01371  *
01372  ******************************************************************************
01373  */
01374 
01375 /** LMAC types */
01376 static struct txnic_lmac_type txnic_lmac_types[] = {
01377         [TXNIC_LMAC_XAUI] = {
01378                 .name = "XAUI",
01379                 .count = 1,
01380                 .lane_to_sds = 0xe4,
01381         },
01382         [TXNIC_LMAC_RXAUI] = {
01383                 .name = "RXAUI",
01384                 .count = 2,
01385                 .lane_to_sds = 0x0e04,
01386         },
01387         [TXNIC_LMAC_10G_R] = {
01388                 .name = "10GBASE-R",
01389                 .count = 4,
01390                 .lane_to_sds = 0x00000000,
01391         },
01392         [TXNIC_LMAC_40G_R] = {
01393                 .name = "40GBASE-R",
01394                 .count = 1,
01395                 .lane_to_sds = 0xe4,
01396         },
01397 };
01398 
01399 /**
01400  * Detect BGX Ethernet interface LMAC type
01401  *
01402  * @v bgx               BGX Ethernet interface
01403  * @ret type            LMAC type, or negative error
01404  */
01405 static int txnic_bgx_detect ( struct txnic_bgx *bgx ) {
01406         uint64_t config;
01407         uint64_t br_pmd_control;
01408         uint64_t rx_lmacs;
01409         unsigned int type;
01410 
01411         /* We assume that the early (pre-UEFI) firmware will have
01412          * configured at least the LMAC 0 type and use of link
01413          * training, and may have overridden the number of LMACs.
01414          */
01415 
01416         /* Determine type from LMAC 0 */
01417         config = readq ( bgx->regs + BGX_CMR_CONFIG );
01418         type = BGX_CMR_CONFIG_LMAC_TYPE_GET ( config );
01419         if ( ( type >= ( sizeof ( txnic_lmac_types ) /
01420                          sizeof ( txnic_lmac_types[0] ) ) ) ||
01421              ( txnic_lmac_types[type].count == 0 ) ) {
01422                 DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/* BGX unknown type %d\n",
01423                        bgx->node, bgx->idx, type );
01424                 return -ENOTTY;
01425         }
01426         bgx->type = &txnic_lmac_types[type];
01427 
01428         /* Check whether link training is required */
01429         br_pmd_control = readq ( bgx->regs + BGX_SPU_BR_PMD_CONTROL );
01430         bgx->training =
01431                 ( !! ( br_pmd_control & BGX_SPU_BR_PMD_CONTROL_TRAIN_EN ) );
01432 
01433         /* Determine number of LMACs */
01434         rx_lmacs = readq ( bgx->regs + BGX_CMR_RX_LMACS );
01435         bgx->count = BGX_CMR_RX_LMACS_LMACS_GET ( rx_lmacs );
01436         if ( ( bgx->count == TXNIC_NUM_LMAC ) &&
01437              ( bgx->type->count != TXNIC_NUM_LMAC ) ) {
01438                 DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/* assuming %d LMACs\n",
01439                        bgx->node, bgx->idx, bgx->type->count );
01440                 bgx->count = bgx->type->count;
01441         }
01442 
01443         return type;
01444 }
01445 
01446 /**
01447  * Initialise BGX Ethernet interface
01448  *
01449  * @v bgx               BGX Ethernet interface
01450  * @v type              LMAC type
01451  */
01452 static void txnic_bgx_init ( struct txnic_bgx *bgx, unsigned int type ) {
01453         uint64_t global_config;
01454         uint32_t lane_to_sds;
01455         unsigned int i;
01456 
01457         /* Set number of LMACs */
01458         writeq ( BGX_CMR_RX_LMACS_LMACS_SET ( bgx->count ),
01459                  ( bgx->regs + BGX_CMR_RX_LMACS ) );
01460         writeq ( BGX_CMR_TX_LMACS_LMACS_SET ( bgx->count ),
01461                  ( bgx->regs + BGX_CMR_TX_LMACS ) );
01462 
01463         /* Set LMAC types and lane mappings, and disable all LMACs */
01464         lane_to_sds = bgx->type->lane_to_sds;
01465         for ( i = 0 ; i < bgx->count ; i++ ) {
01466                 writeq ( ( BGX_CMR_CONFIG_LMAC_TYPE_SET ( type ) |
01467                            BGX_CMR_CONFIG_LANE_TO_SDS ( lane_to_sds ) ),
01468                          ( bgx->regs + BGX_LMAC ( i ) + BGX_CMR_CONFIG ) );
01469                 lane_to_sds >>= 8;
01470         }
01471 
01472         /* Reset all MAC address filtering */
01473         for ( i = 0 ; i < TXNIC_NUM_DMAC ; i++ )
01474                 writeq ( 0, ( bgx->regs + BGX_CMR_RX_DMAC_CAM ( i ) ) );
01475 
01476         /* Reset NCSI steering */
01477         for ( i = 0 ; i < TXNIC_NUM_STEERING ; i++ )
01478                 writeq ( 0, ( bgx->regs + BGX_CMR_RX_STEERING ( i ) ) );
01479 
01480         /* Enable backpressure to all channels */
01481         writeq ( BGX_CMR_CHAN_MSK_AND_ALL ( bgx->count ),
01482                  ( bgx->regs + BGX_CMR_CHAN_MSK_AND ) );
01483 
01484         /* Strip FCS */
01485         global_config = readq ( bgx->regs + BGX_CMR_GLOBAL_CONFIG );
01486         global_config |= BGX_CMR_GLOBAL_CONFIG_FCS_STRIP;
01487         writeq ( global_config, ( bgx->regs + BGX_CMR_GLOBAL_CONFIG ) );
01488 }
01489 
01490 /**
01491  * Get MAC address
01492  *
01493  * @v lmac              Logical MAC
01494  */
01495 static void txnic_bgx_mac ( struct txnic_lmac *lmac ) {
01496         struct txnic_bgx *bgx = lmac->bgx;
01497         unsigned int lmac_idx = TXNIC_LMAC_IDX ( lmac->idx );
01498         uint64_t mac;
01499         EFI_STATUS efirc;
01500         int rc;
01501 
01502         /* Extract MAC from Board Configuration protocol, if available */
01503         if ( txcfg ) {
01504                 if ( ( efirc = txcfg->GetLmacProp ( txcfg, bgx->node, bgx->idx,
01505                                                     lmac_idx, MAC_ADDRESS,
01506                                                     sizeof ( mac ),
01507                                                     &mac ) ) == 0 ) {
01508                         lmac->mac.be64 = cpu_to_be64 ( mac );
01509                 } else {
01510                         rc = -EEFI ( efirc );
01511                         DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/%d could not get "
01512                                "MAC address: %s\n", bgx->node, bgx->idx,
01513                                lmac->idx, strerror ( rc ) );
01514                 }
01515         } else {
01516                 DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/%d has no board "
01517                        "configuration protocol\n", bgx->node, bgx->idx,
01518                        lmac->idx );
01519         }
01520 
01521         /* Use random MAC address if none available */
01522         if ( ! lmac->mac.be64 ) {
01523                 DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/%d has no MAC address\n",
01524                        bgx->node, bgx->idx, lmac->idx );
01525                 eth_random_addr ( lmac->mac.raw );
01526         }
01527 }
01528 
01529 /**
01530  * Initialise Super PHY Unit (SPU)
01531  *
01532  * @v lmac              Logical MAC
01533  */
01534 static void txnic_bgx_spu_init ( struct txnic_lmac *lmac ) {
01535         struct txnic_bgx *bgx = lmac->bgx;
01536 
01537         /* Reset PHY */
01538         writeq ( BGX_SPU_CONTROL1_RESET, ( lmac->regs + BGX_SPU_CONTROL1 ) );
01539         mdelay ( BGX_SPU_RESET_DELAY_MS );
01540 
01541         /* Power down PHY */
01542         writeq ( BGX_SPU_CONTROL1_LO_PWR, ( lmac->regs + BGX_SPU_CONTROL1 ) );
01543 
01544         /* Configure training, if applicable */
01545         if ( bgx->training ) {
01546                 writeq ( 0, ( lmac->regs + BGX_SPU_BR_PMD_LP_CUP ) );
01547                 writeq ( 0, ( lmac->regs + BGX_SPU_BR_PMD_LD_CUP ) );
01548                 writeq ( 0, ( lmac->regs + BGX_SPU_BR_PMD_LD_REP ) );
01549                 writeq ( BGX_SPU_BR_PMD_CONTROL_TRAIN_EN,
01550                          ( lmac->regs + BGX_SPU_BR_PMD_CONTROL ) );
01551         }
01552 
01553         /* Disable forward error correction */
01554         writeq ( 0, ( lmac->regs + BGX_SPU_FEC_CONTROL ) );
01555 
01556         /* Disable autonegotiation */
01557         writeq ( 0, ( lmac->regs + BGX_SPU_AN_CONTROL ) );
01558 
01559         /* Power up PHY */
01560         writeq ( 0, ( lmac->regs + BGX_SPU_CONTROL1 ) );
01561 }
01562 
01563 /**
01564  * Initialise LMAC
01565  *
01566  * @v bgx               BGX Ethernet interface
01567  * @v lmac_idx          LMAC index
01568  */
01569 static void txnic_bgx_lmac_init ( struct txnic_bgx *bgx,
01570                                   unsigned int lmac_idx ) {
01571         struct txnic_lmac *lmac = &bgx->lmac[lmac_idx];
01572         uint64_t config;
01573 
01574         /* Record associated BGX */
01575         lmac->bgx = bgx;
01576 
01577         /* Set register base address (already mapped) */
01578         lmac->regs = ( bgx->regs + BGX_LMAC ( lmac_idx ) );
01579 
01580         /* Calculate virtual NIC index */
01581         lmac->idx = TXNIC_VNIC_IDX ( bgx->idx, lmac_idx );
01582 
01583         /* Set MAC address */
01584         txnic_bgx_mac ( lmac );
01585 
01586         /* Initialise PHY */
01587         txnic_bgx_spu_init ( lmac );
01588 
01589         /* Accept all multicasts and broadcasts */
01590         writeq ( ( BGX_CMR_RX_DMAC_CTL_MCST_MODE_ACCEPT |
01591                    BGX_CMR_RX_DMAC_CTL_BCST_ACCEPT ),
01592                  ( lmac->regs + BGX_CMR_RX_DMAC_CTL ) );
01593 
01594         /* Enable LMAC */
01595         config = readq ( lmac->regs + BGX_CMR_CONFIG );
01596         config |= ( BGX_CMR_CONFIG_ENABLE |
01597                     BGX_CMR_CONFIG_DATA_PKT_RX_EN |
01598                     BGX_CMR_CONFIG_DATA_PKT_TX_EN );
01599         writeq ( config, ( lmac->regs + BGX_CMR_CONFIG ) );
01600 }
01601 
01602 /**
01603  * Probe PCI device
01604  *
01605  * @v pci               PCI device
01606  * @ret rc              Return status code
01607  */
01608 static int txnic_bgx_probe ( struct pci_device *pci ) {
01609         struct txnic_bgx *bgx;
01610         struct txnic_pf *pf;
01611         unsigned long membase;
01612         unsigned int i;
01613         int type;
01614         int rc;
01615 
01616         /* Allocate and initialise structure */
01617         bgx = zalloc ( sizeof ( *bgx ) );
01618         if ( ! bgx ) {
01619                 rc = -ENOMEM;
01620                 goto err_alloc;
01621         }
01622         bgx->pci = pci;
01623         pci_set_drvdata ( pci, bgx );
01624 
01625         /* Get base address */
01626         membase = pciea_bar_start ( pci, PCIEA_BEI_BAR_0 );
01627 
01628         /* Calculate node ID and index */
01629         bgx->node = txnic_address_node ( membase );
01630         bgx->idx = txnic_address_bgx ( membase );
01631 
01632         /* Fix up PCI device */
01633         adjust_pci_device ( pci );
01634 
01635         /* Map registers */
01636         bgx->regs = ioremap ( membase, TXNIC_BGX_BAR_SIZE );
01637         if ( ! bgx->regs ) {
01638                 rc = -ENODEV;
01639                 goto err_ioremap;
01640         }
01641 
01642         /* Detect LMAC type */
01643         if ( ( type = txnic_bgx_detect ( bgx ) ) < 0 ) {
01644                 rc = type;
01645                 goto err_detect;
01646         }
01647         DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/* BGX %s at %#lx %dx %s%s\n",
01648                bgx->node, bgx->idx, pci->dev.name, membase, bgx->count,
01649                bgx->type->name, ( bgx->training ? "(training)" : "" ) );
01650 
01651         /* Initialise interface */
01652         txnic_bgx_init ( bgx, type );
01653 
01654         /* Initialise all LMACs */
01655         for ( i = 0 ; i < bgx->count ; i++ )
01656                 txnic_bgx_lmac_init ( bgx, i );
01657 
01658         /* Add to list of BGX devices */
01659         list_add_tail ( &bgx->list, &txnic_bgxs );
01660 
01661         /* Probe all LMACs, if applicable */
01662         list_for_each_entry ( pf, &txnic_pfs, list ) {
01663                 if ( pf->node != bgx->node )
01664                         continue;
01665                 if ( ( rc = txnic_lmac_probe_all ( pf, bgx ) ) != 0 )
01666                         goto err_probe;
01667         }
01668 
01669         return 0;
01670 
01671         if ( bgx->pf )
01672                 txnic_lmac_remove_all ( bgx->pf, bgx );
01673         list_del ( &bgx->list );
01674  err_probe:
01675  err_detect:
01676         iounmap ( bgx->regs );
01677  err_ioremap:
01678         free ( bgx );
01679  err_alloc:
01680         return rc;
01681 }
01682 
01683 /**
01684  * Remove PCI device
01685  *
01686  * @v pci               PCI device
01687  */
01688 static void txnic_bgx_remove ( struct pci_device *pci ) {
01689         struct txnic_bgx *bgx = pci_get_drvdata ( pci );
01690 
01691         /* Remove all LMACs, if applicable */
01692         if ( bgx->pf )
01693                 txnic_lmac_remove_all ( bgx->pf, bgx );
01694 
01695         /* Remove from list of BGX devices */
01696         list_del ( &bgx->list );
01697 
01698         /* Unmap registers */
01699         iounmap ( bgx->regs );
01700 
01701         /* Free BGX device */
01702         free ( bgx );
01703 }
01704 
01705 /** BGX PCI device IDs */
01706 static struct pci_device_id txnic_bgx_ids[] = {
01707         PCI_ROM ( 0x177d, 0xa026, "thunder-bgx", "ThunderX BGX", 0 ),
01708 };
01709 
01710 /** BGX PCI driver */
01711 struct pci_driver txnic_bgx_driver __pci_driver = {
01712         .ids = txnic_bgx_ids,
01713         .id_count = ( sizeof ( txnic_bgx_ids ) / sizeof ( txnic_bgx_ids[0] ) ),
01714         .probe = txnic_bgx_probe,
01715         .remove = txnic_bgx_remove,
01716 };