iPXE
Functions | Variables
ena.c File Reference

Amazon ENA network driver. More...

#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include <ipxe/if_ether.h>
#include <ipxe/iobuf.h>
#include <ipxe/malloc.h>
#include <ipxe/pci.h>
#include "ena.h"

Go to the source code of this file.

Functions

 FILE_LICENCE (GPL2_OR_LATER_OR_UBDL)
static const char * ena_direction (unsigned int direction)
 Get direction name (for debugging)
static int ena_reset (struct ena_nic *ena)
 Reset hardware.
static void ena_set_base (struct ena_nic *ena, unsigned int offset, void *base)
 Set queue base address.
static void ena_set_caps (struct ena_nic *ena, unsigned int offset, unsigned int count, size_t size)
 Set queue capabilities.
static void ena_clear_caps (struct ena_nic *ena, unsigned int offset)
 Clear queue capabilities.
static int ena_create_admin (struct ena_nic *ena)
 Create admin queues.
static void ena_destroy_admin (struct ena_nic *ena)
 Destroy admin queues.
static union ena_aq_reqena_admin_req (struct ena_nic *ena)
 Get next available admin queue request.
static int ena_admin (struct ena_nic *ena, union ena_aq_req *req, union ena_acq_rsp **rsp)
 Issue admin queue request.
static int ena_create_sq (struct ena_nic *ena, struct ena_sq *sq, struct ena_cq *cq)
 Create submission queue.
static int ena_destroy_sq (struct ena_nic *ena, struct ena_sq *sq)
 Destroy submission queue.
static int ena_create_cq (struct ena_nic *ena, struct ena_cq *cq)
 Create completion queue.
static int ena_destroy_cq (struct ena_nic *ena, struct ena_cq *cq)
 Destroy completion queue.
static int ena_create_qp (struct ena_nic *ena, struct ena_qp *qp)
 Create queue pair.
static int ena_destroy_qp (struct ena_nic *ena, struct ena_qp *qp)
 Destroy queue pair.
static int ena_get_device_attributes (struct net_device *netdev)
 Get device attributes.
static int ena_get_stats (struct ena_nic *ena)
 Get statistics (for debugging)
static void ena_refill_rx (struct net_device *netdev)
 Refill receive queue.
static void ena_empty_rx (struct ena_nic *ena)
 Discard unused receive I/O buffers.
static int ena_open (struct net_device *netdev)
 Open network device.
static void ena_close (struct net_device *netdev)
 Close network device.
static int ena_transmit (struct net_device *netdev, struct io_buffer *iobuf)
 Transmit packet.
static void ena_poll_tx (struct net_device *netdev)
 Poll for completed transmissions.
static void ena_poll_rx (struct net_device *netdev)
 Poll for received packets.
static void ena_poll (struct net_device *netdev)
 Poll for completed and received packets.
static int ena_probe (struct pci_device *pci)
 Probe PCI device.
static void ena_remove (struct pci_device *pci)
 Remove PCI device.

Variables

static struct net_device_operations ena_operations
 ENA network device operations.
static struct pci_device_id ena_nics []
 ENA PCI device IDs.
struct pci_driver ena_driver __pci_driver
 ENA PCI driver.

Detailed Description

Amazon ENA network driver.

Definition in file ena.c.


Function Documentation

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL  )
static const char* ena_direction ( unsigned int  direction) [static]

Get direction name (for debugging)

Parameters:
directionDirection
Return values:
nameDirection name

Definition at line 51 of file ena.c.

References ENA_SQ_RX, and ENA_SQ_TX.

Referenced by ena_create_sq(), and ena_destroy_sq().

                                                             {

        switch ( direction ) {
        case ENA_SQ_TX:         return "TX";
        case ENA_SQ_RX:         return "RX";
        default:                return "<UNKNOWN>";
        }
}
static int ena_reset ( struct ena_nic ena) [static]

Reset hardware.

Parameters:
enaENA device
Return values:
rcReturn status code

Definition at line 73 of file ena.c.

References DBGC, ENA_CTRL, ENA_CTRL_RESET, ENA_RESET_MAX_WAIT_MS, ENA_STAT, ENA_STAT_READY, ETIMEDOUT, mdelay(), readl(), ena_nic::regs, and writel().

Referenced by ena_probe(), and ena_remove().

                                             {
        uint32_t stat;
        unsigned int i;

        /* Trigger reset */
        writel ( ENA_CTRL_RESET, ( ena->regs + ENA_CTRL ) );

        /* Wait for reset to complete */
        for ( i = 0 ; i < ENA_RESET_MAX_WAIT_MS ; i++ ) {

                /* Check if device is ready */
                stat = readl ( ena->regs + ENA_STAT );
                if ( stat & ENA_STAT_READY )
                        return 0;

                /* Delay */
                mdelay ( 1 );
        }

        DBGC ( ena, "ENA %p timed out waiting for reset (status %#08x)\n",
               ena, stat );
        return -ETIMEDOUT;
}
static void ena_set_base ( struct ena_nic ena,
unsigned int  offset,
void *  base 
) [inline, static]

Set queue base address.

Parameters:
enaENA device
offsetRegister offset
addressBase address

Definition at line 111 of file ena.c.

References ENA_BASE_HI, ENA_BASE_LO, phys, ena_nic::regs, virt_to_bus(), and writel().

Referenced by ena_create_admin().

                                               {
        physaddr_t phys = virt_to_bus ( base );

        /* Program base address registers */
        writel ( ( phys & 0xffffffffUL ),
                 ( ena->regs + offset + ENA_BASE_LO ) );
        if ( sizeof ( phys ) > sizeof ( uint32_t ) ) {
                writel ( ( ( ( uint64_t ) phys ) >> 32 ),
                         ( ena->regs + offset + ENA_BASE_HI ) );
        } else {
                writel ( 0, ( ena->regs + offset + ENA_BASE_HI ) );
        }
}
static void ena_set_caps ( struct ena_nic ena,
unsigned int  offset,
unsigned int  count,
size_t  size 
) [inline, static]

Set queue capabilities.

Parameters:
enaENA device
offsetRegister offset
countNumber of entries
sizeSize of each entry

Definition at line 135 of file ena.c.

References ENA_CAPS, and writel().

Referenced by ena_create_admin().

                             {

        /* Program capabilities register */
        writel ( ENA_CAPS ( count, size ), ( ena->regs + offset ) );
}
static void ena_clear_caps ( struct ena_nic ena,
unsigned int  offset 
) [inline, static]

Clear queue capabilities.

Parameters:
enaENA device
offsetRegister offset

Definition at line 149 of file ena.c.

References writel().

Referenced by ena_create_admin(), and ena_destroy_admin().

                                                            {

        /* Clear capabilities register */
        writel ( 0, ( ena->regs + offset ) );
}
static int ena_create_admin ( struct ena_nic ena) [static]

Create admin queues.

Parameters:
enaENA device
Return values:
rcReturn status code

Definition at line 161 of file ena.c.

References ena_nic::acq, ena_nic::aq, DBGC, ENA_ACQ_BASE, ENA_ACQ_CAPS, ENA_ACQ_COUNT, ENA_AQ_BASE, ENA_AQ_CAPS, ENA_AQ_COUNT, ena_clear_caps(), ena_set_base(), ena_set_caps(), ENOMEM, free_dma(), malloc_dma(), memset(), rc, ena_aq::req, ena_acq::rsp, and virt_to_phys().

Referenced by ena_probe().

                                                    {
        size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
        size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
        int rc;

        /* Allocate admin completion queue */
        ena->acq.rsp = malloc_dma ( acq_len, acq_len );
        if ( ! ena->acq.rsp ) {
                rc = -ENOMEM;
                goto err_alloc_acq;
        }
        memset ( ena->acq.rsp, 0, acq_len );

        /* Allocate admin queue */
        ena->aq.req = malloc_dma ( aq_len, aq_len );
        if ( ! ena->aq.req ) {
                rc = -ENOMEM;
                goto err_alloc_aq;
        }
        memset ( ena->aq.req, 0, aq_len );

        /* Program queue addresses and capabilities */
        ena_set_base ( ena, ENA_ACQ_BASE, ena->acq.rsp );
        ena_set_caps ( ena, ENA_ACQ_CAPS, ENA_ACQ_COUNT,
                       sizeof ( ena->acq.rsp[0] ) );
        ena_set_base ( ena, ENA_AQ_BASE, ena->aq.req );
        ena_set_caps ( ena, ENA_AQ_CAPS, ENA_AQ_COUNT,
                       sizeof ( ena->aq.req[0] ) );

        DBGC ( ena, "ENA %p AQ [%08lx,%08lx) ACQ [%08lx,%08lx)\n",
               ena, virt_to_phys ( ena->aq.req ),
               ( virt_to_phys ( ena->aq.req ) + aq_len ),
               virt_to_phys ( ena->acq.rsp ),
               ( virt_to_phys ( ena->acq.rsp ) + acq_len ) );
        return 0;

        ena_clear_caps ( ena, ENA_AQ_CAPS );
        ena_clear_caps ( ena, ENA_ACQ_CAPS );
        free_dma ( ena->aq.req, aq_len );
 err_alloc_aq:
        free_dma ( ena->acq.rsp, acq_len );
 err_alloc_acq:
        return rc;
}
static void ena_destroy_admin ( struct ena_nic ena) [static]

Destroy admin queues.

Parameters:
enaENA device

Definition at line 211 of file ena.c.

References ena_nic::acq, ena_nic::aq, DBGC, ENA_ACQ_CAPS, ENA_ACQ_COUNT, ENA_AQ_CAPS, ENA_AQ_COUNT, ena_clear_caps(), free_dma(), ena_aq::req, ena_acq::rsp, and wmb.

Referenced by ena_probe(), and ena_remove().

                                                      {
        size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
        size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );

        /* Clear queue capabilities */
        ena_clear_caps ( ena, ENA_AQ_CAPS );
        ena_clear_caps ( ena, ENA_ACQ_CAPS );
        wmb();

        /* Free queues */
        free_dma ( ena->aq.req, aq_len );
        free_dma ( ena->acq.rsp, acq_len );
        DBGC ( ena, "ENA %p AQ and ACQ destroyed\n", ena );
}
static union ena_aq_req* ena_admin_req ( struct ena_nic ena) [static, write]

Get next available admin queue request.

Parameters:
enaENA device
Return values:
reqAdmin queue request

Definition at line 232 of file ena.c.

References ena_nic::aq, ENA_AQ_COUNT, index, memset(), ena_aq::prod, and ena_aq::req.

Referenced by ena_create_cq(), ena_create_sq(), ena_destroy_cq(), ena_destroy_sq(), ena_get_device_attributes(), and ena_get_stats().

                                                                {
        union ena_aq_req *req;
        unsigned int index;

        /* Get next request */
        index = ( ena->aq.prod % ENA_AQ_COUNT );
        req = &ena->aq.req[index];

        /* Initialise request */
        memset ( ( ( ( void * ) req ) + sizeof ( req->header ) ), 0,
                 ( sizeof ( *req ) - sizeof ( req->header ) ) );
        req->header.id = ena->aq.prod;

        /* Increment producer counter */
        ena->aq.prod++;

        return req;
}
static int ena_admin ( struct ena_nic ena,
union ena_aq_req req,
union ena_acq_rsp **  rsp 
) [static]

Issue admin queue request.

Parameters:
enaENA device
reqAdmin queue request
rspAdmin queue response to fill in
Return values:
rcReturn status code

Definition at line 259 of file ena.c.

References ena_nic::acq, ena_nic::aq, ena_acq::cons, DBGC, DBGC2, DBGC2_HDA, DBGC_HDA, EILSEQ, EIO, ENA_ACQ_COUNT, ENA_ACQ_PHASE, ENA_ADMIN_MAX_WAIT_MS, ENA_AQ_DB, ENA_AQ_PHASE, ETIMEDOUT, ena_aq_header::flags, ena_aq_req::header, ena_aq_header::id, index, le16_to_cpu, mdelay(), ena_acq::phase, ena_aq::prod, rc, ena_nic::regs, ena_acq::rsp, virt_to_phys(), wmb, and writel().

Referenced by ena_create_cq(), ena_create_sq(), ena_destroy_cq(), ena_destroy_sq(), ena_get_device_attributes(), and ena_get_stats().

                                                 {
        unsigned int index;
        unsigned int i;
        int rc;

        /* Locate response */
        index = ( ena->acq.cons % ENA_ACQ_COUNT );
        *rsp = &ena->acq.rsp[index];

        /* Mark request as ready */
        req->header.flags ^= ENA_AQ_PHASE;
        wmb();
        DBGC2 ( ena, "ENA %p admin request %#x:\n",
                ena, le16_to_cpu ( req->header.id ) );
        DBGC2_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );

        /* Ring doorbell */
        writel ( ena->aq.prod, ( ena->regs + ENA_AQ_DB ) );

        /* Wait for response */
        for ( i = 0 ; i < ENA_ADMIN_MAX_WAIT_MS ; i++ ) {

                /* Check for response */
                if ( ( (*rsp)->header.flags ^ ena->acq.phase ) & ENA_ACQ_PHASE){
                        mdelay ( 1 );
                        continue;
                }
                DBGC2 ( ena, "ENA %p admin response %#x:\n",
                        ena, le16_to_cpu ( (*rsp)->header.id ) );
                DBGC2_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ));

                /* Increment consumer counter */
                ena->acq.cons++;
                if ( ( ena->acq.cons % ENA_ACQ_COUNT ) == 0 )
                        ena->acq.phase ^= ENA_ACQ_PHASE;

                /* Check command identifier */
                if ( (*rsp)->header.id != req->header.id ) {
                        DBGC ( ena, "ENA %p admin response %#x mismatch:\n",
                               ena, le16_to_cpu ( (*rsp)->header.id ) );
                        rc = -EILSEQ;
                        goto err;
                }

                /* Check status */
                if ( (*rsp)->header.status != 0 ) {
                        DBGC ( ena, "ENA %p admin response %#x status %d:\n",
                               ena, le16_to_cpu ( (*rsp)->header.id ),
                               (*rsp)->header.status );
                        rc = -EIO;
                        goto err;
                }

                /* Success */
                return 0;
        }

        rc = -ETIMEDOUT;
        DBGC ( ena, "ENA %p timed out waiting for admin request %#x:\n",
               ena, le16_to_cpu ( req->header.id ) );
 err:
        DBGC_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
        DBGC_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ) );
        return rc;
}
static int ena_create_sq ( struct ena_nic ena,
struct ena_sq sq,
struct ena_cq *  cq 
) [static]

Create submission queue.

Parameters:
enaENA device
sqSubmission queue
cqCorresponding completion queue
Return values:
rcReturn status code

Definition at line 334 of file ena.c.

References ena_create_sq_req::address, ena_create_sq_req::count, ena_sq::count, cpu_to_le16, cpu_to_le64, ena_create_sq_req::cq_id, ena_aq_req::create_sq, ena_acq_rsp::create_sq, DBGC, ena_create_sq_req::direction, ena_sq::direction, ena_create_sq_rsp::doorbell, ena_sq::doorbell, ena_admin(), ena_admin_req(), ENA_ALIGN, ENA_CREATE_SQ, ena_direction(), ENA_SQ_CONTIGUOUS, ENA_SQ_HOST_MEMORY, ENA_SQE_PHASE, ENOMEM, free_dma(), ena_aq_req::header, ena_create_sq_rsp::id, ena_sq::id, le16_to_cpu, le32_to_cpu, ena_sq::len, malloc_dma(), memset(), ena_aq_header::opcode, ena_sq::phase, ena_create_sq_req::policy, ena_sq::prod, ena_sq::raw, rc, rsp, ena_sq::sqe, virt_to_bus(), and virt_to_phys().

Referenced by ena_create_qp().

                                               {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Allocate submission queue entries */
        sq->sqe.raw = malloc_dma ( sq->len, ENA_ALIGN );
        if ( ! sq->sqe.raw ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        memset ( sq->sqe.raw, 0, sq->len );

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_CREATE_SQ;
        req->create_sq.direction = sq->direction;
        req->create_sq.policy = cpu_to_le16 ( ENA_SQ_HOST_MEMORY |
                                              ENA_SQ_CONTIGUOUS );
        req->create_sq.cq_id = cpu_to_le16 ( cq->id );
        req->create_sq.count = cpu_to_le16 ( sq->count );
        req->create_sq.address = cpu_to_le64 ( virt_to_bus ( sq->sqe.raw ) );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
                goto err_admin;

        /* Parse response */
        sq->id = le16_to_cpu ( rsp->create_sq.id );
        sq->doorbell = le32_to_cpu ( rsp->create_sq.doorbell );

        /* Reset producer counter and phase */
        sq->prod = 0;
        sq->phase = ENA_SQE_PHASE;

        DBGC ( ena, "ENA %p %s SQ%d at [%08lx,%08lx) db +%04x CQ%d\n",
               ena, ena_direction ( sq->direction ), sq->id,
               virt_to_phys ( sq->sqe.raw ),
               ( virt_to_phys ( sq->sqe.raw ) + sq->len ),
               sq->doorbell, cq->id );
        return 0;

 err_admin:
        free_dma ( sq->sqe.raw, sq->len );
 err_alloc:
        return rc;
}
static int ena_destroy_sq ( struct ena_nic ena,
struct ena_sq sq 
) [static]

Destroy submission queue.

Parameters:
enaENA device
sqSubmission queue
Return values:
rcReturn status code

Definition at line 390 of file ena.c.

References cpu_to_le16, DBGC, ena_aq_req::destroy_sq, ena_destroy_sq_req::direction, ena_sq::direction, ena_admin(), ena_admin_req(), ENA_DESTROY_SQ, ena_direction(), free_dma(), ena_aq_req::header, ena_destroy_sq_req::id, ena_sq::id, ena_sq::len, ena_aq_header::opcode, ena_sq::raw, rc, rsp, and ena_sq::sqe.

Referenced by ena_create_qp(), and ena_destroy_qp().

                                                                     {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_DESTROY_SQ;
        req->destroy_sq.id = cpu_to_le16 ( sq->id );
        req->destroy_sq.direction = sq->direction;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
                return rc;

        /* Free submission queue entries */
        free_dma ( sq->sqe.raw, sq->len );

        DBGC ( ena, "ENA %p %s SQ%d destroyed\n",
               ena, ena_direction ( sq->direction ), sq->id );
        return 0;
}
static int ena_create_cq ( struct ena_nic ena,
struct ena_cq *  cq 
) [static]

Create completion queue.

Parameters:
enaENA device
cqCompletion queue
Return values:
rcReturn status code

Definition at line 420 of file ena.c.

References ena_create_cq_req::address, ena_create_cq_req::count, ena_create_cq_rsp::count, cpu_to_le16, cpu_to_le64, ena_aq_req::create_cq, ena_acq_rsp::create_cq, DBGC, ena_create_cq_rsp::doorbell, ena_admin(), ena_admin_req(), ENA_ALIGN, ENA_CQE_PHASE, ENA_CREATE_CQ, ENOMEM, free_dma(), ena_aq_req::header, ena_create_cq_rsp::id, le16_to_cpu, le32_to_cpu, malloc_dma(), memset(), ena_aq_header::opcode, rc, rsp, ena_create_cq_req::size, virt_to_bus(), and virt_to_phys().

Referenced by ena_create_qp().

                                                                    {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Allocate completion queue entries */
        cq->cqe.raw = malloc_dma ( cq->len, ENA_ALIGN );
        if ( ! cq->cqe.raw ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        memset ( cq->cqe.raw, 0, cq->len );

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_CREATE_CQ;
        req->create_cq.size = cq->size;
        req->create_cq.count = cpu_to_le16 ( cq->requested );
        req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
                goto err_admin;

        /* Parse response */
        cq->id = le16_to_cpu ( rsp->create_cq.id );
        cq->actual = le16_to_cpu ( rsp->create_cq.count );
        cq->doorbell = le32_to_cpu ( rsp->create_cq.doorbell );
        cq->mask = ( cq->actual - 1 );
        if ( cq->actual != cq->requested ) {
                DBGC ( ena, "ENA %p CQ%d requested %d actual %d\n",
                       ena, cq->id, cq->requested, cq->actual );
        }

        /* Reset consumer counter and phase */
        cq->cons = 0;
        cq->phase = ENA_CQE_PHASE;

        DBGC ( ena, "ENA %p CQ%d at [%08lx,%08lx) db +%04x\n",
               ena, cq->id, virt_to_phys ( cq->cqe.raw ),
               ( virt_to_phys ( cq->cqe.raw ) + cq->len ), cq->doorbell );
        return 0;

 err_admin:
        free_dma ( cq->cqe.raw, cq->len );
 err_alloc:
        return rc;
}
static int ena_destroy_cq ( struct ena_nic ena,
struct ena_cq *  cq 
) [static]

Destroy completion queue.

Parameters:
enaENA device
cqCompletion queue
Return values:
rcReturn status code

Definition at line 476 of file ena.c.

References cpu_to_le16, DBGC, ena_aq_req::destroy_cq, ena_admin(), ena_admin_req(), ENA_DESTROY_CQ, free_dma(), ena_aq_req::header, ena_destroy_cq_req::id, ena_aq_header::opcode, rc, and rsp.

Referenced by ena_create_qp(), and ena_destroy_qp().

                                                                     {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_DESTROY_CQ;
        req->destroy_cq.id = cpu_to_le16 ( cq->id );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
                return rc;

        /* Free completion queue entries */
        free_dma ( cq->cqe.raw, cq->len );

        DBGC ( ena, "ENA %p CQ%d destroyed\n", ena, cq->id );
        return 0;
}
static int ena_create_qp ( struct ena_nic ena,
struct ena_qp *  qp 
) [static]

Create queue pair.

Parameters:
enaENA device
qpQueue pair
Return values:
rcReturn status code

Definition at line 504 of file ena.c.

References ena_create_cq(), ena_create_sq(), ena_destroy_cq(), ena_destroy_sq(), and rc.

Referenced by ena_open().

                                                                    {
        int rc;

        /* Create completion queue */
        if ( ( rc = ena_create_cq ( ena, &qp->cq ) ) != 0 )
                goto err_create_cq;

        /* Create submission queue */
        if ( ( rc = ena_create_sq ( ena, &qp->sq, &qp->cq ) ) != 0 )
                goto err_create_sq;

        return 0;

        ena_destroy_sq ( ena, &qp->sq );
 err_create_sq:
        ena_destroy_cq ( ena, &qp->cq );
 err_create_cq:
        return rc;
}
static int ena_destroy_qp ( struct ena_nic ena,
struct ena_qp *  qp 
) [static]

Destroy queue pair.

Parameters:
enaENA device
qpQueue pair
Return values:
rcReturn status code

Definition at line 531 of file ena.c.

References ena_destroy_cq(), and ena_destroy_sq().

Referenced by ena_close(), and ena_open().

                                                                     {

        /* Destroy submission queue */
        ena_destroy_sq ( ena,  &qp->sq );

        /* Destroy completion queue */
        ena_destroy_cq ( ena, &qp->cq );

        return 0;
}
static int ena_get_device_attributes ( struct net_device netdev) [static]

Get device attributes.

Parameters:
netdevNetwork device
Return values:
rcReturn status code

Definition at line 548 of file ena.c.

References DBGC, ena_feature::device, ena_admin(), ena_admin_req(), ENA_DEVICE_ATTRIBUTES, ENA_GET_FEATURE, ETH_ALEN, ETH_HLEN, eth_ntoa(), ena_get_feature_rsp::feature, feature, ena_aq_req::get_feature, ena_acq_rsp::get_feature, ena_aq_req::header, net_device::hw_addr, ena_get_feature_req::id, le32_to_cpu, ena_device_attributes::mac, net_device::max_pkt_len, memcpy(), ena_device_attributes::mtu, net_device::mtu, ena_aq_header::opcode, net_device::priv, rc, and rsp.

Referenced by ena_probe().

                                                                   {
        struct ena_nic *ena = netdev->priv;
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        union ena_feature *feature;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_GET_FEATURE;
        req->get_feature.id = ENA_DEVICE_ATTRIBUTES;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
                return rc;

        /* Parse response */
        feature = &rsp->get_feature.feature;
        memcpy ( netdev->hw_addr, feature->device.mac, ETH_ALEN );
        netdev->max_pkt_len = le32_to_cpu ( feature->device.mtu );
        netdev->mtu = ( netdev->max_pkt_len - ETH_HLEN );

        DBGC ( ena, "ENA %p MAC %s MTU %zd\n",
               ena, eth_ntoa ( netdev->hw_addr ), netdev->max_pkt_len );
        return 0;
}
static int ena_get_stats ( struct ena_nic ena) [static]

Get statistics (for debugging)

Parameters:
enaENA device
Return values:
rcReturn status code

Definition at line 581 of file ena.c.

References DBG_LOG, DBGC, ena_get_stats_req::device, ena_admin(), ena_admin_req(), ENA_DEVICE_MINE, ENA_GET_STATS, ENA_STATS_SCOPE_ETH, ENA_STATS_TYPE_BASIC, ena_aq_req::get_stats, ena_acq_rsp::get_stats, ena_aq_req::header, le64_to_cpu, ena_aq_header::opcode, rc, rsp, ena_get_stats_rsp::rx_bytes, ena_get_stats_rsp::rx_drops, ena_get_stats_rsp::rx_packets, ena_get_stats_req::scope, ena_get_stats_rsp::tx_bytes, ena_get_stats_rsp::tx_packets, and ena_get_stats_req::type.

Referenced by ena_close().

                                                 {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        struct ena_get_stats_rsp *stats;
        int rc;

        /* Do nothing unless debug messages are enabled */
        if ( ! DBG_LOG )
                return 0;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_GET_STATS;
        req->get_stats.type = ENA_STATS_TYPE_BASIC;
        req->get_stats.scope = ENA_STATS_SCOPE_ETH;
        req->get_stats.device = ENA_DEVICE_MINE;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
                return rc;

        /* Parse response */
        stats = &rsp->get_stats;
        DBGC ( ena, "ENA %p TX bytes %#llx packets %#llx\n", ena,
               ( ( unsigned long long ) le64_to_cpu ( stats->tx_bytes ) ),
               ( ( unsigned long long ) le64_to_cpu ( stats->tx_packets ) ) );
        DBGC ( ena, "ENA %p RX bytes %#llx packets %#llx drops %#llx\n", ena,
               ( ( unsigned long long ) le64_to_cpu ( stats->rx_bytes ) ),
               ( ( unsigned long long ) le64_to_cpu ( stats->rx_packets ) ),
               ( ( unsigned long long ) le64_to_cpu ( stats->rx_drops ) ) );

        return 0;
}
static void ena_refill_rx ( struct net_device netdev) [static]

Refill receive queue.

Parameters:
netdevNetwork device

Definition at line 627 of file ena.c.

References address, ena_rx_sqe::address, alloc_iob(), assert, cpu_to_le16, cpu_to_le64, io_buffer::data, DBGC2, ENA_RX_COUNT, ENA_SQE_CPL, ENA_SQE_FIRST, ENA_SQE_LAST, ENA_SQE_PHASE, ena_rx_sqe::flags, ena_rx_sqe::id, index, len, ena_rx_sqe::len, net_device::max_pkt_len, NULL, net_device::priv, ena_nic::regs, ena_nic::rx, ena_nic::rx_iobuf, virt_to_bus(), wmb, and writel().

Referenced by ena_open(), and ena_poll().

                                                        {
        struct ena_nic *ena = netdev->priv;
        struct io_buffer *iobuf;
        struct ena_rx_sqe *sqe;
        unsigned int index;
        physaddr_t address;
        size_t len = netdev->max_pkt_len;
        unsigned int refilled = 0;

        /* Refill queue */
        while ( ( ena->rx.sq.prod - ena->rx.cq.cons ) < ENA_RX_COUNT ) {

                /* Allocate I/O buffer */
                iobuf = alloc_iob ( len );
                if ( ! iobuf ) {
                        /* Wait for next refill */
                        break;
                }

                /* Get next submission queue entry */
                index = ( ena->rx.sq.prod % ENA_RX_COUNT );
                sqe = &ena->rx.sq.sqe.rx[index];

                /* Construct submission queue entry */
                address = virt_to_bus ( iobuf->data );
                sqe->len = cpu_to_le16 ( len );
                sqe->id = cpu_to_le16 ( ena->rx.sq.prod );
                sqe->address = cpu_to_le64 ( address );
                wmb();
                sqe->flags = ( ENA_SQE_FIRST | ENA_SQE_LAST | ENA_SQE_CPL |
                               ena->rx.sq.phase );

                /* Increment producer counter */
                ena->rx.sq.prod++;
                if ( ( ena->rx.sq.prod % ENA_RX_COUNT ) == 0 )
                        ena->rx.sq.phase ^= ENA_SQE_PHASE;

                /* Record I/O buffer */
                assert ( ena->rx_iobuf[index] == NULL );
                ena->rx_iobuf[index] = iobuf;

                DBGC2 ( ena, "ENA %p RX %d at [%08llx,%08llx)\n", ena, sqe->id,
                        ( ( unsigned long long ) address ),
                        ( ( unsigned long long ) address + len ) );
                refilled++;
        }

        /* Ring doorbell, if applicable */
        if ( refilled ) {
                wmb();
                writel ( ena->rx.sq.prod, ( ena->regs + ena->rx.sq.doorbell ) );
        }
}
static void ena_empty_rx ( struct ena_nic ena) [static]

Discard unused receive I/O buffers.

Parameters:
enaENA device

Definition at line 686 of file ena.c.

References ENA_RX_COUNT, free_iob(), NULL, and ena_nic::rx_iobuf.

Referenced by ena_close().

                                                 {
        unsigned int i;

        for ( i = 0 ; i < ENA_RX_COUNT ; i++ ) {
                if ( ena->rx_iobuf[i] )
                        free_iob ( ena->rx_iobuf[i] );
                ena->rx_iobuf[i] = NULL;
        }
}
static int ena_open ( struct net_device netdev) [static]

Open network device.

Parameters:
netdevNetwork device
Return values:
rcReturn status code

Definition at line 702 of file ena.c.

References ena_create_qp(), ena_destroy_qp(), ena_refill_rx(), net_device::priv, rc, ena_nic::rx, and ena_nic::tx.

                                                  {
        struct ena_nic *ena = netdev->priv;
        int rc;

        /* Create transmit queue pair */
        if ( ( rc = ena_create_qp ( ena, &ena->tx ) ) != 0 )
                goto err_create_tx;

        /* Create receive queue pair */
        if ( ( rc = ena_create_qp ( ena, &ena->rx ) ) != 0 )
                goto err_create_rx;

        /* Refill receive queue */
        ena_refill_rx ( netdev );

        return 0;

        ena_destroy_qp ( ena, &ena->rx );
 err_create_rx:
        ena_destroy_qp ( ena, &ena->tx );
 err_create_tx:
        return rc;
}
static void ena_close ( struct net_device netdev) [static]

Close network device.

Parameters:
netdevNetwork device

Definition at line 731 of file ena.c.

References ena_destroy_qp(), ena_empty_rx(), ena_get_stats(), net_device::priv, ena_nic::rx, and ena_nic::tx.

                                                    {
        struct ena_nic *ena = netdev->priv;

        /* Dump statistics (for debugging) */
        ena_get_stats ( ena );

        /* Destroy receive queue pair */
        ena_destroy_qp ( ena, &ena->rx );

        /* Discard any unused receive buffers */
        ena_empty_rx ( ena );

        /* Destroy transmit queue pair */
        ena_destroy_qp ( ena, &ena->tx );
}
static int ena_transmit ( struct net_device netdev,
struct io_buffer iobuf 
) [static]

Transmit packet.

Parameters:
netdevNetwork device
iobufI/O buffer
Return values:
rcReturn status code

Definition at line 754 of file ena.c.

References address, ena_tx_sqe::address, cpu_to_le16, cpu_to_le64, io_buffer::data, DBGC, DBGC2, ENA_SQE_CPL, ENA_SQE_FIRST, ENA_SQE_LAST, ENA_SQE_PHASE, ENA_TX_COUNT, ENOBUFS, ena_tx_sqe::flags, ena_tx_sqe::id, index, iob_len(), len, ena_tx_sqe::len, net_device::priv, ena_nic::regs, ena_nic::tx, virt_to_bus(), wmb, and writel().

                                                                               {
        struct ena_nic *ena = netdev->priv;
        struct ena_tx_sqe *sqe;
        unsigned int index;
        physaddr_t address;
        size_t len;

        /* Get next submission queue entry */
        if ( ( ena->tx.sq.prod - ena->tx.cq.cons ) >= ENA_TX_COUNT ) {
                DBGC ( ena, "ENA %p out of transmit descriptors\n", ena );
                return -ENOBUFS;
        }
        index = ( ena->tx.sq.prod % ENA_TX_COUNT );
        sqe = &ena->tx.sq.sqe.tx[index];

        /* Construct submission queue entry */
        address = virt_to_bus ( iobuf->data );
        len = iob_len ( iobuf );
        sqe->len = cpu_to_le16 ( len );
        sqe->id = ena->tx.sq.prod;
        sqe->address = cpu_to_le64 ( address );
        wmb();
        sqe->flags = ( ENA_SQE_FIRST | ENA_SQE_LAST | ENA_SQE_CPL |
                       ena->tx.sq.phase );
        wmb();

        /* Increment producer counter */
        ena->tx.sq.prod++;
        if ( ( ena->tx.sq.prod % ENA_TX_COUNT ) == 0 )
                ena->tx.sq.phase ^= ENA_SQE_PHASE;

        /* Ring doorbell */
        writel ( ena->tx.sq.prod, ( ena->regs + ena->tx.sq.doorbell ) );

        DBGC2 ( ena, "ENA %p TX %d at [%08llx,%08llx)\n", ena, sqe->id,
                ( ( unsigned long long ) address ),
                ( ( unsigned long long ) address + len ) );
        return 0;
}
static void ena_poll_tx ( struct net_device netdev) [static]

Poll for completed transmissions.

Parameters:
netdevNetwork device

Definition at line 799 of file ena.c.

References DBGC2, ENA_CQE_PHASE, ena_tx_cqe::flags, ena_tx_cqe::id, index, le16_to_cpu, netdev_tx_complete_next(), net_device::priv, and ena_nic::tx.

Referenced by ena_poll().

                                                      {
        struct ena_nic *ena = netdev->priv;
        struct ena_tx_cqe *cqe;
        unsigned int index;

        /* Check for completed packets */
        while ( ena->tx.cq.cons != ena->tx.sq.prod ) {

                /* Get next completion queue entry */
                index = ( ena->tx.cq.cons & ena->tx.cq.mask );
                cqe = &ena->tx.cq.cqe.tx[index];

                /* Stop if completion queue entry is empty */
                if ( ( cqe->flags ^ ena->tx.cq.phase ) & ENA_CQE_PHASE )
                        return;
                DBGC2 ( ena, "ENA %p TX %d complete\n", ena,
                        ( le16_to_cpu ( cqe->id ) >> 2 /* Don't ask */ ) );

                /* Increment consumer counter */
                ena->tx.cq.cons++;
                if ( ! ( ena->tx.cq.cons & ena->tx.cq.mask ) )
                        ena->tx.cq.phase ^= ENA_CQE_PHASE;

                /* Complete transmit */
                netdev_tx_complete_next ( netdev );
        }
}
static void ena_poll_rx ( struct net_device netdev) [static]

Poll for received packets.

Parameters:
netdevNetwork device

Definition at line 832 of file ena.c.

References DBGC2, ENA_CQE_PHASE, ENA_RX_COUNT, ena_rx_cqe::flags, ena_rx_cqe::id, index, iob_put, le16_to_cpu, len, ena_rx_cqe::len, netdev_rx(), NULL, net_device::priv, ena_nic::rx, and ena_nic::rx_iobuf.

Referenced by ena_poll().

                                                      {
        struct ena_nic *ena = netdev->priv;
        struct ena_rx_cqe *cqe;
        struct io_buffer *iobuf;
        unsigned int index;
        size_t len;

        /* Check for received packets */
        while ( ena->rx.cq.cons != ena->rx.sq.prod ) {

                /* Get next completion queue entry */
                index = ( ena->rx.cq.cons % ENA_RX_COUNT );
                cqe = &ena->rx.cq.cqe.rx[index];

                /* Stop if completion queue entry is empty */
                if ( ( cqe->flags ^ ena->rx.cq.phase ) & ENA_CQE_PHASE )
                        return;

                /* Increment consumer counter */
                ena->rx.cq.cons++;
                if ( ! ( ena->rx.cq.cons & ena->rx.cq.mask ) )
                        ena->rx.cq.phase ^= ENA_CQE_PHASE;

                /* Populate I/O buffer */
                iobuf = ena->rx_iobuf[index];
                ena->rx_iobuf[index] = NULL;
                len = le16_to_cpu ( cqe->len );
                iob_put ( iobuf, len );

                /* Hand off to network stack */
                DBGC2 ( ena, "ENA %p RX %d complete (length %zd)\n",
                        ena, le16_to_cpu ( cqe->id ), len );
                netdev_rx ( netdev, iobuf );
        }
}
static void ena_poll ( struct net_device netdev) [static]

Poll for completed and received packets.

Parameters:
netdevNetwork device

Definition at line 873 of file ena.c.

References ena_poll_rx(), ena_poll_tx(), and ena_refill_rx().

                                                   {

        /* Poll for transmit completions */
        ena_poll_tx ( netdev );

        /* Poll for receive completions */
        ena_poll_rx ( netdev );

        /* Refill receive ring */
        ena_refill_rx ( netdev );
}
static int ena_probe ( struct pci_device pci) [static]

Probe PCI device.

Parameters:
pciPCI device
Return values:
rcReturn status code

Definition at line 906 of file ena.c.

References ena_nic::acq, adjust_pci_device(), alloc_etherdev(), pci_device::dev, net_device::dev, ENA_ACQ_PHASE, ENA_BAR_SIZE, ena_create_admin(), ena_destroy_admin(), ena_get_device_attributes(), ena_reset(), ENA_RX_COUNT, ENA_SQ_RX, ENA_SQ_TX, ENA_TX_COUNT, ENODEV, ENOMEM, ioremap(), iounmap(), pci_device::membase, memset(), netdev, netdev_init(), netdev_link_up(), netdev_nullify(), netdev_put(), pci_set_drvdata(), ena_acq::phase, net_device::priv, rc, register_netdev(), ena_nic::regs, ena_nic::rx, ena_nic::tx, and unregister_netdev().

                                                {
        struct net_device *netdev;
        struct ena_nic *ena;
        int rc;

        /* Allocate and initialise net device */
        netdev = alloc_etherdev ( sizeof ( *ena ) );
        if ( ! netdev ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        netdev_init ( netdev, &ena_operations );
        ena = netdev->priv;
        pci_set_drvdata ( pci, netdev );
        netdev->dev = &pci->dev;
        memset ( ena, 0, sizeof ( *ena ) );
        ena->acq.phase = ENA_ACQ_PHASE;
        ena_cq_init ( &ena->tx.cq, ENA_TX_COUNT,
                      sizeof ( ena->tx.cq.cqe.tx[0] ) );
        ena_sq_init ( &ena->tx.sq, ENA_SQ_TX, ENA_TX_COUNT,
                      sizeof ( ena->tx.sq.sqe.tx[0] ) );
        ena_cq_init ( &ena->rx.cq, ENA_RX_COUNT,
                      sizeof ( ena->rx.cq.cqe.rx[0] ) );
        ena_sq_init ( &ena->rx.sq, ENA_SQ_RX, ENA_RX_COUNT,
                      sizeof ( ena->rx.sq.sqe.rx[0] ) );

        /* Fix up PCI device */
        adjust_pci_device ( pci );

        /* Map registers */
        ena->regs = ioremap ( pci->membase, ENA_BAR_SIZE );
        if ( ! ena->regs ) {
                rc = -ENODEV;
                goto err_ioremap;
        }

        /* Reset the NIC */
        if ( ( rc = ena_reset ( ena ) ) != 0 )
                goto err_reset;

        /* Create admin queues */
        if ( ( rc = ena_create_admin ( ena ) ) != 0 )
                goto err_create_admin;

        /* Fetch MAC address */
        if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
                goto err_get_device_attributes;

        /* Register network device */
        if ( ( rc = register_netdev ( netdev ) ) != 0 )
                goto err_register_netdev;

        /* Mark as link up, since we have no way to test link state on
         * this hardware.
         */
        netdev_link_up ( netdev );

        return 0;

        unregister_netdev ( netdev );
 err_register_netdev:
 err_get_device_attributes:
        ena_destroy_admin ( ena );
 err_create_admin:
        ena_reset ( ena );
 err_reset:
        iounmap ( ena->regs );
 err_ioremap:
        netdev_nullify ( netdev );
        netdev_put ( netdev );
 err_alloc:
        return rc;
}
static void ena_remove ( struct pci_device pci) [static]

Remove PCI device.

Parameters:
pciPCI device

Definition at line 985 of file ena.c.

References ena_destroy_admin(), ena_reset(), iounmap(), netdev, netdev_nullify(), netdev_put(), pci_get_drvdata(), net_device::priv, ena_nic::regs, and unregister_netdev().

                                                  {
        struct net_device *netdev = pci_get_drvdata ( pci );
        struct ena_nic *ena = netdev->priv;

        /* Unregister network device */
        unregister_netdev ( netdev );

        /* Destroy admin queues */
        ena_destroy_admin ( ena );

        /* Reset card */
        ena_reset ( ena );

        /* Free network device */
        iounmap ( ena->regs );
        netdev_nullify ( netdev );
        netdev_put ( netdev );
}

Variable Documentation

Initial value:
 {
        .open           = ena_open,
        .close          = ena_close,
        .transmit       = ena_transmit,
        .poll           = ena_poll,
}

ENA network device operations.

Definition at line 886 of file ena.c.

struct pci_device_id ena_nics[] [static]
Initial value:
 {
        PCI_ROM ( 0x1d0f, 0xec20, "ena-vf", "ENA VF", 0 ),
        PCI_ROM ( 0x1d0f, 0xec21, "ena-vf-llq", "ENA VF (LLQ)", 0 ),
}

ENA PCI device IDs.

Definition at line 1005 of file ena.c.

struct pci_driver ena_driver __pci_driver
Initial value:
 {
        .ids = ena_nics,
        .id_count = ( sizeof ( ena_nics ) / sizeof ( ena_nics[0] ) ),
        .probe = ena_probe,
        .remove = ena_remove,
}

ENA PCI driver.

Definition at line 1011 of file ena.c.