ena.c File Reference

Amazon ENA network driver. More...

#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include <ipxe/if_ether.h>
#include <ipxe/iobuf.h>
#include <ipxe/malloc.h>
#include <ipxe/pci.h>
#include <ipxe/pcibridge.h>
#include <ipxe/version.h>
#include "ena.h"

Go to the source code of this file.

Functions
	FILE_LICENCE (GPL2_OR_LATER_OR_UBDL)
static const char *	ena_direction (unsigned int direction)
	Get direction name (for debugging) More...
static int	ena_reset_wait (struct ena_nic *ena, uint32_t expected)
	Wait for reset operation to be acknowledged. More...
static int	ena_reset (struct ena_nic *ena)
	Reset hardware. More...
static void	ena_set_base (struct ena_nic *ena, unsigned int offset, void *base)
	Set queue base address. More...
static void	ena_set_caps (struct ena_nic *ena, unsigned int offset, unsigned int count, size_t size)
	Set queue capabilities. More...
static void	ena_clear_caps (struct ena_nic *ena, unsigned int offset)
	Clear queue capabilities. More...
static int	ena_create_admin (struct ena_nic *ena)
	Create admin queues. More...
static void	ena_destroy_admin (struct ena_nic *ena)
	Destroy admin queues. More...
static union ena_aq_req *	ena_admin_req (struct ena_nic *ena)
	Get next available admin queue request. More...
static int	ena_admin (struct ena_nic *ena, union ena_aq_req *req, union ena_acq_rsp **rsp)
	Issue admin queue request. More...
static int	ena_set_aenq_config (struct ena_nic *ena, uint32_t enabled)
	Set async event notification queue config. More...
static int	ena_create_async (struct ena_nic *ena)
	Create async event notification queue. More...
static void	ena_destroy_async (struct ena_nic *ena)
	Destroy async event notification queue. More...
static int	ena_create_sq (struct ena_nic *ena, struct ena_sq *sq, struct ena_cq *cq)
	Create submission queue. More...
static int	ena_destroy_sq (struct ena_nic *ena, struct ena_sq *sq)
	Destroy submission queue. More...
static int	ena_create_cq (struct ena_nic *ena, struct ena_cq *cq)
	Create completion queue. More...
static int	ena_destroy_cq (struct ena_nic *ena, struct ena_cq *cq)
	Destroy completion queue. More...
static int	ena_create_qp (struct ena_nic *ena, struct ena_qp *qp)
	Create queue pair. More...
static int	ena_destroy_qp (struct ena_nic *ena, struct ena_qp *qp)
	Destroy queue pair. More...
static int	ena_get_device_attributes (struct net_device *netdev)
	Get device attributes. More...
static int	ena_set_host_attributes (struct ena_nic *ena)
	Set host attributes. More...
static int	ena_llq_config (struct ena_nic *ena)
	Configure low latency queues. More...
static int	ena_get_stats (struct ena_nic *ena)
	Get statistics (for debugging) More...
static void	ena_refill_rx (struct net_device *netdev)
	Refill receive queue. More...
static void	ena_empty_rx (struct ena_nic *ena)
	Discard unused receive I/O buffers. More...
static void	ena_cancel_tx (struct net_device *netdev)
	Cancel uncompleted transmit I/O buffers. More...
static int	ena_open (struct net_device *netdev)
	Open network device. More...
static void	ena_close (struct net_device *netdev)
	Close network device. More...
static int	ena_transmit (struct net_device *netdev, struct io_buffer *iobuf)
	Transmit packet. More...
static void	ena_poll_tx (struct net_device *netdev)
	Poll for completed transmissions. More...
static void	ena_poll_rx (struct net_device *netdev)
	Poll for received packets. More...
static void	ena_poll (struct net_device *netdev)
	Poll for completed and received packets. More...
static int	ena_membases (struct ena_nic *ena, struct pci_device *pci, unsigned long *prefmembase, unsigned long *prefmemsize)
	Assign memory BARs. More...
static int	ena_probe (struct pci_device *pci)
	Probe PCI device. More...
static void	ena_remove (struct pci_device *pci)
	Remove PCI device. More...

Variables
static struct net_device_operations	ena_operations
	ENA network device operations. More...
static struct pci_device_id	ena_nics []
	ENA PCI device IDs. More...
struct pci_driver ena_driver	__pci_driver
	ENA PCI driver. More...

Detailed Description

Amazon ENA network driver.

Definition in file ena.c.

Function Documentation

FILE_LICENCE()

FILE_LICENCE

(

GPL2_OR_LATER_OR_UBDL

)

ena_direction()

static const char* ena_direction ( unsigned int  direction )

static

Get direction name (for debugging)

Parameters

direction

Direction

Return values

name	Direction name

Definition at line 54 of file ena.c.

                                                             {

        switch ( direction ) {
        case ENA_SQ_TX:         return "TX";
        case ENA_SQ_RX:         return "RX";
        default:                return "<UNKNOWN>";
        }
}

References direction, ENA_SQ_RX, and ENA_SQ_TX.

Referenced by ena_create_sq(), and ena_destroy_sq().

ena_reset_wait()

static int ena_reset_wait ( struct ena_nic *  ena, uint32_t  expected  )

static

Wait for reset operation to be acknowledged.

Parameters

ena	ENA device
expected	Expected reset state

Return values

rc	Return status code

Definition at line 77 of file ena.c.

                                                                     {
        uint32_t stat;
        unsigned int i;

        /* Wait for reset to complete */
        for ( i = 0 ; i < ENA_RESET_MAX_WAIT_MS ; i++ ) {

                /* Check if device is ready */
                stat = readl ( ena->regs + ENA_STAT );
                if ( ( stat & ENA_STAT_RESET ) == expected )
                        return 0;

                /* Delay */
                mdelay ( 1 );
        }

        DBGC ( ena, "ENA %p timed out waiting for reset status %#08x "
               "(got %#08x)\n", ena, expected, stat );
        return -ETIMEDOUT;
}

References DBGC, ENA_RESET_MAX_WAIT_MS, ENA_STAT, ENA_STAT_RESET, ETIMEDOUT, mdelay(), readl(), ena_nic::regs, and stat.

Referenced by ena_reset().

ena_reset()

static int ena_reset ( struct ena_nic *  ena )

static

Reset hardware.

Parameters

ena	ENA device

Return values

rc	Return status code

Definition at line 104 of file ena.c.

                                             {
        int rc;

        /* Trigger reset */
        writel ( ENA_CTRL_RESET, ( ena->regs + ENA_CTRL ) );

        /* Wait for reset to take effect */
        if ( ( rc = ena_reset_wait ( ena, ENA_STAT_RESET ) ) != 0 )
                return rc;

        /* Clear reset */
        writel ( 0, ( ena->regs + ENA_CTRL ) );

        /* Wait for reset to clear */
        if ( ( rc = ena_reset_wait ( ena, 0 ) ) != 0 )
                return rc;

        return 0;
}

References ENA_CTRL, ENA_CTRL_RESET, ena_reset_wait(), ENA_STAT_RESET, rc, ena_nic::regs, and writel().

Referenced by ena_probe(), and ena_remove().

ena_set_base()

static void ena_set_base ( struct ena_nic *  ena, unsigned int  offset, void *  base  )

inlinestatic

Set queue base address.

Parameters

ena	ENA device
offset	Register offset
address	Base address

Definition at line 138 of file ena.c.

                                               {
        physaddr_t phys = virt_to_bus ( base );

        /* Program base address registers */
        writel ( ( phys & 0xffffffffUL ),
                 ( ena->regs + offset + ENA_BASE_LO ) );
        if ( sizeof ( phys ) > sizeof ( uint32_t ) ) {
                writel ( ( ( ( uint64_t ) phys ) >> 32 ),
                         ( ena->regs + offset + ENA_BASE_HI ) );
        } else {
                writel ( 0, ( ena->regs + offset + ENA_BASE_HI ) );
        }
}

References base, ENA_BASE_HI, ENA_BASE_LO, offset, phys, ena_nic::regs, virt_to_bus(), and writel().

Referenced by ena_create_admin(), and ena_create_async().

ena_set_caps()

static void ena_set_caps ( struct ena_nic *  ena, unsigned int  offset, unsigned int  count, size_t  size  )

inlinestatic

Set queue capabilities.

Parameters

ena	ENA device
offset	Register offset
count	Number of entries
size	Size of each entry

Definition at line 162 of file ena.c.

                             {

        /* Program capabilities register */
        writel ( ENA_CAPS ( count, size ), ( ena->regs + offset ) );
}

References count, ENA_CAPS, offset, ena_nic::regs, size, and writel().

Referenced by ena_create_admin(), and ena_create_async().

ena_clear_caps()

static void ena_clear_caps ( struct ena_nic *  ena, unsigned int  offset  )

inlinestatic

Clear queue capabilities.

Parameters

ena	ENA device
offset	Register offset

Definition at line 176 of file ena.c.

                                                            {

        /* Clear capabilities register */
        writel ( 0, ( ena->regs + offset ) );
}

References offset, ena_nic::regs, and writel().

Referenced by ena_create_admin(), ena_create_async(), ena_destroy_admin(), and ena_destroy_async().

ena_create_admin()

static int ena_create_admin ( struct ena_nic *  ena )

static

Create admin queues.

Parameters

ena	ENA device

Return values

rc	Return status code

Definition at line 188 of file ena.c.

                                                    {
        size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
        size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
        int rc;

        /* Allocate admin completion queue */
        ena->acq.rsp = malloc_phys ( acq_len, acq_len );
        if ( ! ena->acq.rsp ) {
                rc = -ENOMEM;
                goto err_alloc_acq;
        }
        memset ( ena->acq.rsp, 0, acq_len );

        /* Allocate admin queue */
        ena->aq.req = malloc_phys ( aq_len, aq_len );
        if ( ! ena->aq.req ) {
                rc = -ENOMEM;
                goto err_alloc_aq;
        }
        memset ( ena->aq.req, 0, aq_len );

        /* Program queue addresses and capabilities */
        ena_set_base ( ena, ENA_ACQ_BASE, ena->acq.rsp );
        ena_set_caps ( ena, ENA_ACQ_CAPS, ENA_ACQ_COUNT,
                       sizeof ( ena->acq.rsp[0] ) );
        ena_set_base ( ena, ENA_AQ_BASE, ena->aq.req );
        ena_set_caps ( ena, ENA_AQ_CAPS, ENA_AQ_COUNT,
                       sizeof ( ena->aq.req[0] ) );

        DBGC ( ena, "ENA %p AQ [%08lx,%08lx) ACQ [%08lx,%08lx)\n",
               ena, virt_to_phys ( ena->aq.req ),
               ( virt_to_phys ( ena->aq.req ) + aq_len ),
               virt_to_phys ( ena->acq.rsp ),
               ( virt_to_phys ( ena->acq.rsp ) + acq_len ) );
        return 0;

        ena_clear_caps ( ena, ENA_AQ_CAPS );
        ena_clear_caps ( ena, ENA_ACQ_CAPS );
        free_phys ( ena->aq.req, aq_len );
 err_alloc_aq:
        free_phys ( ena->acq.rsp, acq_len );
 err_alloc_acq:
        return rc;
}

References ena_nic::acq, ena_nic::aq, DBGC, ENA_ACQ_BASE, ENA_ACQ_CAPS, ENA_ACQ_COUNT, ENA_AQ_BASE, ENA_AQ_CAPS, ENA_AQ_COUNT, ena_clear_caps(), ena_set_base(), ena_set_caps(), ENOMEM, free_phys(), malloc_phys(), memset(), rc, ena_aq::req, and ena_acq::rsp.

Referenced by ena_probe().

ena_destroy_admin()

static void ena_destroy_admin ( struct ena_nic *  ena )

static

Destroy admin queues.

Parameters

ena	ENA device

Definition at line 238 of file ena.c.

                                                      {
        size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
        size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );

        /* Clear queue capabilities */
        ena_clear_caps ( ena, ENA_AQ_CAPS );
        ena_clear_caps ( ena, ENA_ACQ_CAPS );
        wmb();

        /* Free queues */
        free_phys ( ena->aq.req, aq_len );
        free_phys ( ena->acq.rsp, acq_len );
        DBGC ( ena, "ENA %p AQ and ACQ destroyed\n", ena );
}

References ena_nic::acq, ena_nic::aq, DBGC, ENA_ACQ_CAPS, ENA_ACQ_COUNT, ENA_AQ_CAPS, ENA_AQ_COUNT, ena_clear_caps(), free_phys(), ena_aq::req, ena_acq::rsp, and wmb().

Referenced by ena_probe(), and ena_remove().

ena_admin_req()

static union ena_aq_req* ena_admin_req ( struct ena_nic *  ena )

static

Get next available admin queue request.

Parameters

ena	ENA device

Return values

req	Admin queue request

Definition at line 259 of file ena.c.

                                                                {
        union ena_aq_req *req;
        unsigned int index;

        /* Get next request */
        index = ( ena->aq.prod % ENA_AQ_COUNT );
        req = &ena->aq.req[index];

        /* Initialise request */
        memset ( ( ( ( void * ) req ) + sizeof ( req->header ) ), 0,
                 ( sizeof ( *req ) - sizeof ( req->header ) ) );
        req->header.id = ena->aq.prod;

        /* Increment producer counter */
        ena->aq.prod++;

        return req;
}

References ena_nic::aq, ENA_AQ_COUNT, index, memset(), ena_aq::prod, and ena_aq::req.

Referenced by ena_create_cq(), ena_create_sq(), ena_destroy_cq(), ena_destroy_sq(), ena_get_device_attributes(), ena_get_stats(), ena_llq_config(), ena_set_aenq_config(), and ena_set_host_attributes().

ena_admin()

static int ena_admin ( struct ena_nic *  ena, union ena_aq_req *  req, union ena_acq_rsp **  rsp  )

static

Issue admin queue request.

Parameters

ena	ENA device
req	Admin queue request
rsp	Admin queue response to fill in

Return values

rc	Return status code

Definition at line 286 of file ena.c.

                                                 {
        unsigned int index;
        unsigned int i;
        int rc;

        /* Locate response */
        index = ( ena->acq.cons % ENA_ACQ_COUNT );
        *rsp = &ena->acq.rsp[index];

        /* Mark request as ready */
        req->header.flags ^= ENA_AQ_PHASE;
        wmb();
        DBGC2 ( ena, "ENA %p admin request %#x:\n",
                ena, le16_to_cpu ( req->header.id ) );
        DBGC2_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );

        /* Ring doorbell */
        writel ( ena->aq.prod, ( ena->regs + ENA_AQ_DB ) );

        /* Wait for response */
        for ( i = 0 ; i < ENA_ADMIN_MAX_WAIT_MS ; i++ ) {

                /* Check for response */
                if ( ( (*rsp)->header.flags ^ ena->acq.phase ) & ENA_ACQ_PHASE){
                        mdelay ( 1 );
                        continue;
                }
                DBGC2 ( ena, "ENA %p admin response %#x:\n",
                        ena, le16_to_cpu ( (*rsp)->header.id ) );
                DBGC2_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ));

                /* Increment consumer counter */
                ena->acq.cons++;
                if ( ( ena->acq.cons % ENA_ACQ_COUNT ) == 0 )
                        ena->acq.phase ^= ENA_ACQ_PHASE;

                /* Check command identifier */
                if ( (*rsp)->header.id != req->header.id ) {
                        DBGC ( ena, "ENA %p admin response %#x mismatch:\n",
                               ena, le16_to_cpu ( (*rsp)->header.id ) );
                        rc = -EILSEQ;
                        goto err;
                }

                /* Check status */
                if ( (*rsp)->header.status != 0 ) {
                        DBGC ( ena, "ENA %p admin response %#x status %d:\n",
                               ena, le16_to_cpu ( (*rsp)->header.id ),
                               (*rsp)->header.status );
                        rc = -EIO;
                        goto err;
                }

                /* Success */
                return 0;
        }

        rc = -ETIMEDOUT;
        DBGC ( ena, "ENA %p timed out waiting for admin request %#x:\n",
               ena, le16_to_cpu ( req->header.id ) );
 err:
        DBGC_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
        DBGC_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ) );
        return rc;
}

References ena_nic::acq, ena_nic::aq, ena_acq::cons, DBGC, DBGC2, DBGC2_HDA, DBGC_HDA, EILSEQ, EIO, ENA_ACQ_COUNT, ENA_ACQ_PHASE, ENA_ADMIN_MAX_WAIT_MS, ENA_AQ_DB, ENA_AQ_PHASE, ETIMEDOUT, ena_aq_header::flags, ena_aq_req::header, ena_aq_header::id, index, le16_to_cpu, mdelay(), ena_acq::phase, ena_aq::prod, rc, ena_nic::regs, rsp, ena_acq::rsp, wmb(), and writel().

Referenced by ena_create_cq(), ena_create_sq(), ena_destroy_cq(), ena_destroy_sq(), ena_get_device_attributes(), ena_get_stats(), ena_llq_config(), ena_set_aenq_config(), and ena_set_host_attributes().

ena_set_aenq_config()

static int ena_set_aenq_config ( struct ena_nic *  ena, uint32_t  enabled  )

static

Set async event notification queue config.

Parameters

ena	ENA device
enabled	Bitmask of the groups to enable

Return values

rc	Return status code

Definition at line 360 of file ena.c.

                                                                         {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        union ena_feature *feature;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_SET_FEATURE;
        req->set_feature.id = ENA_AENQ_CONFIG;
        feature = &req->set_feature.feature;
        feature->aenq.enabled = cpu_to_le32 ( enabled );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not set AENQ configuration: %s\n",
                       ena, strerror ( rc ) );
                return rc;
        }

        return 0;
}

References cpu_to_le32, DBGC, ena_admin(), ena_admin_req(), ENA_AENQ_CONFIG, ENA_SET_FEATURE, enabled, feature, ena_set_feature_req::feature, ena_aq_req::header, ena_set_feature_req::id, ena_aq_header::opcode, rc, rsp, ena_aq_req::set_feature, and strerror().

Referenced by ena_create_async().

ena_create_async()

static int ena_create_async ( struct ena_nic *  ena )

static

Create async event notification queue.

Parameters

ena	ENA device

Return values

rc	Return status code

Definition at line 389 of file ena.c.

                                                    {
        size_t aenq_len = ( ENA_AENQ_COUNT * sizeof ( ena->aenq.evt[0] ) );
        int rc;

        /* Allocate async event notification queue */
        ena->aenq.evt = malloc_phys ( aenq_len, aenq_len );
        if ( ! ena->aenq.evt ) {
                rc = -ENOMEM;
                goto err_alloc_aenq;
        }
        memset ( ena->aenq.evt, 0, aenq_len );

        /* Program queue address and capabilities */
        ena_set_base ( ena, ENA_AENQ_BASE, ena->aenq.evt );
        ena_set_caps ( ena, ENA_AENQ_CAPS, ENA_AENQ_COUNT,
                       sizeof ( ena->aenq.evt[0] ) );

        DBGC ( ena, "ENA %p AENQ [%08lx,%08lx)\n",
               ena, virt_to_phys ( ena->aenq.evt ),
               ( virt_to_phys ( ena->aenq.evt ) + aenq_len ) );

        /* Disable all events */
        if ( ( rc = ena_set_aenq_config ( ena, 0 ) ) != 0 )
                goto err_set_aenq_config;

        return 0;

 err_set_aenq_config:
        ena_clear_caps ( ena, ENA_AENQ_CAPS );
        free_phys ( ena->aenq.evt, aenq_len );
 err_alloc_aenq:
        return rc;
}

References ena_nic::aenq, DBGC, ENA_AENQ_BASE, ENA_AENQ_CAPS, ENA_AENQ_COUNT, ena_clear_caps(), ena_set_aenq_config(), ena_set_base(), ena_set_caps(), ENOMEM, ena_aenq::evt, free_phys(), malloc_phys(), memset(), and rc.

Referenced by ena_probe().

ena_destroy_async()

static void ena_destroy_async ( struct ena_nic *  ena )

static

Destroy async event notification queue.

Parameters

ena	ENA device

Definition at line 428 of file ena.c.

                                                      {
        size_t aenq_len = ( ENA_AENQ_COUNT * sizeof ( ena->aenq.evt[0] ) );

        /* Clear queue capabilities */
        ena_clear_caps ( ena, ENA_AENQ_CAPS );
        wmb();

        /* Free queue */
        free_phys ( ena->aenq.evt, aenq_len );
        DBGC ( ena, "ENA %p AENQ destroyed\n", ena );
}

References ena_nic::aenq, DBGC, ENA_AENQ_CAPS, ENA_AENQ_COUNT, ena_clear_caps(), ena_aenq::evt, free_phys(), and wmb().

Referenced by ena_probe(), and ena_remove().

ena_create_sq()

static int ena_create_sq ( struct ena_nic *  ena, struct ena_sq *  sq, struct ena_cq *  cq  )

static

Create submission queue.

Parameters

ena	ENA device
sq	Submission queue
cq	Corresponding completion queue

Return values

rc	Return status code

Definition at line 448 of file ena.c.

                                               {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        unsigned int i;
        size_t llqe;
        int rc;

        /* Allocate submission queue entries */
        sq->sqe.raw = malloc_phys ( sq->len, ENA_ALIGN );
        if ( ! sq->sqe.raw ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        memset ( sq->sqe.raw, 0, sq->len );

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_CREATE_SQ;
        req->create_sq.direction = sq->direction;
        req->create_sq.policy = cpu_to_le16 ( sq->policy );
        req->create_sq.cq_id = cpu_to_le16 ( cq->id );
        req->create_sq.count = cpu_to_le16 ( sq->count );
        if ( ! ( sq->policy & ENA_SQ_DEVICE_MEMORY ) ) {
                req->create_sq.address =
                        cpu_to_le64 ( virt_to_bus ( sq->sqe.raw ) );
        }

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not create %s SQ: %s\n",
                       ena, ena_direction ( sq->direction ), strerror ( rc ) );
                goto err_admin;
        }

        /* Parse response */
        sq->id = le16_to_cpu ( rsp->create_sq.id );
        sq->doorbell = le32_to_cpu ( rsp->create_sq.doorbell );
        llqe = le32_to_cpu ( rsp->create_sq.llqe );
        if ( sq->policy & ENA_SQ_DEVICE_MEMORY ) {
                assert ( ena->mem != NULL );
                assert ( sq->len >= sizeof ( *sq->sqe.llq ) );
                sq->llqe = ( ena->mem + llqe );
        } else {
                sq->llqe = NULL;
        }

        /* Reset producer counter and phase */
        sq->prod = 0;
        sq->phase = ENA_SQE_PHASE;

        /* Calculate fill level */
        sq->fill = sq->count;
        if ( sq->fill > cq->actual )
                sq->fill = cq->actual;

        /* Initialise buffer ID ring */
        for ( i = 0 ; i < sq->count ; i++ )
                sq->ids[i] = i;

        DBGC ( ena, "ENA %p %s SQ%d at ",
               ena, ena_direction ( sq->direction ), sq->id );
        if ( sq->policy & ENA_SQ_DEVICE_MEMORY ) {
                DBGC ( ena, "LLQ [+%08zx,+%08zx)", llqe,
                       ( llqe + ( sq->count * sizeof ( sq->sqe.llq[0] ) ) ) );
        } else {
                DBGC ( ena, "[%08lx,%08lx)", virt_to_phys ( sq->sqe.raw ),
                       ( virt_to_phys ( sq->sqe.raw ) + sq->len ) );
        }
        DBGC ( ena, " fill %d db +%04x CQ%d\n",
               sq->fill, sq->doorbell, cq->id );
        return 0;

 err_admin:
        free_phys ( sq->sqe.raw, sq->len );
 err_alloc:
        return rc;
}

References ena_cq::actual, ena_create_sq_req::address, assert(), ena_create_sq_req::count, ena_sq::count, cpu_to_le16, cpu_to_le64, ena_create_sq_req::cq_id, ena_aq_req::create_sq, DBGC, ena_create_sq_req::direction, ena_sq::direction, ena_sq::doorbell, ena_admin(), ena_admin_req(), ENA_ALIGN, ENA_CREATE_SQ, ena_direction(), ENA_SQ_DEVICE_MEMORY, ENA_SQE_PHASE, ENOMEM, ena_sq::fill, free_phys(), ena_aq_req::header, ena_sq::id, ena_cq::id, ena_sq::ids, le16_to_cpu, le32_to_cpu, ena_sq::len, ena_sq::llq, llqe, ena_sq::llqe, malloc_phys(), ena_nic::mem, memset(), NULL, ena_aq_header::opcode, ena_sq::phase, ena_create_sq_req::policy, ena_sq::policy, ena_sq::prod, ena_sq::raw, rc, rsp, ena_sq::sqe, strerror(), and virt_to_bus().

Referenced by ena_create_qp().

ena_destroy_sq()

static int ena_destroy_sq ( struct ena_nic *  ena, struct ena_sq *  sq  )

static

Destroy submission queue.

Parameters

ena	ENA device
sq	Submission queue

Return values

rc	Return status code

Definition at line 534 of file ena.c.

                                                                     {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_DESTROY_SQ;
        req->destroy_sq.id = cpu_to_le16 ( sq->id );
        req->destroy_sq.direction = sq->direction;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not destroy %s SQ%d: %s\n",
                       ena, ena_direction ( sq->direction ), sq->id,
                       strerror ( rc ) );
                return rc;
        }

        /* Free submission queue entries */
        free_phys ( sq->sqe.raw, sq->len );

        DBGC ( ena, "ENA %p %s SQ%d destroyed\n",
               ena, ena_direction ( sq->direction ), sq->id );
        return 0;
}

References cpu_to_le16, DBGC, ena_aq_req::destroy_sq, ena_destroy_sq_req::direction, ena_sq::direction, ena_admin(), ena_admin_req(), ENA_DESTROY_SQ, ena_direction(), free_phys(), ena_aq_req::header, ena_destroy_sq_req::id, ena_sq::id, ena_sq::len, ena_aq_header::opcode, ena_sq::raw, rc, rsp, ena_sq::sqe, and strerror().

Referenced by ena_create_qp(), and ena_destroy_qp().

ena_create_cq()

static int ena_create_cq ( struct ena_nic *  ena, struct ena_cq *  cq  )

static

Create completion queue.

Parameters

ena	ENA device
cq	Completion queue

Return values

rc	Return status code

Definition at line 568 of file ena.c.

                                                                    {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Allocate completion queue entries */
        cq->cqe.raw = malloc_phys ( cq->len, ENA_ALIGN );
        if ( ! cq->cqe.raw ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        memset ( cq->cqe.raw, 0, cq->len );

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_CREATE_CQ;
        req->create_cq.size = cq->size;
        req->create_cq.count = cpu_to_le16 ( cq->requested );
        req->create_cq.vector = cpu_to_le32 ( ENA_MSIX_NONE );
        req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not create CQ (broken firmware?): "
                       "%s\n", ena, strerror ( rc ) );
                goto err_admin;
        }

        /* Parse response */
        cq->id = le16_to_cpu ( rsp->create_cq.id );
        cq->actual = le16_to_cpu ( rsp->create_cq.count );
        cq->doorbell = le32_to_cpu ( rsp->create_cq.doorbell );
        cq->mask = ( cq->actual - 1 );
        if ( cq->actual != cq->requested ) {
                DBGC ( ena, "ENA %p CQ%d requested %d actual %d\n",
                       ena, cq->id, cq->requested, cq->actual );
        }

        /* Reset consumer counter and phase */
        cq->cons = 0;
        cq->phase = ENA_CQE_PHASE;

        DBGC ( ena, "ENA %p CQ%d at [%08lx,%08lx) db +%04x\n",
               ena, cq->id, virt_to_phys ( cq->cqe.raw ),
               ( virt_to_phys ( cq->cqe.raw ) + cq->len ), cq->doorbell );
        return 0;

 err_admin:
        free_phys ( cq->cqe.raw, cq->len );
 err_alloc:
        return rc;
}

References ena_cq::actual, ena_create_cq_req::address, ena_cq::cons, ena_create_cq_req::count, cpu_to_le16, cpu_to_le32, cpu_to_le64, ena_cq::cqe, ena_aq_req::create_cq, DBGC, ena_cq::doorbell, ena_admin(), ena_admin_req(), ENA_ALIGN, ENA_CQE_PHASE, ENA_CREATE_CQ, ENA_MSIX_NONE, ENOMEM, free_phys(), ena_aq_req::header, ena_cq::id, le16_to_cpu, le32_to_cpu, ena_cq::len, malloc_phys(), ena_cq::mask, memset(), ena_aq_header::opcode, ena_cq::phase, ena_cq::raw, rc, ena_cq::requested, rsp, ena_create_cq_req::size, ena_cq::size, strerror(), ena_create_cq_req::vector, and virt_to_bus().

Referenced by ena_create_qp().

ena_destroy_cq()

static int ena_destroy_cq ( struct ena_nic *  ena, struct ena_cq *  cq  )

static

Destroy completion queue.

Parameters

ena	ENA device
cq	Completion queue

Return values

rc	Return status code

Definition at line 628 of file ena.c.

                                                                     {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_DESTROY_CQ;
        req->destroy_cq.id = cpu_to_le16 ( cq->id );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not destroy CQ%d: %s\n",
                       ena, cq->id, strerror ( rc ) );
                return rc;
        }

        /* Free completion queue entries */
        free_phys ( cq->cqe.raw, cq->len );

        DBGC ( ena, "ENA %p CQ%d destroyed\n", ena, cq->id );
        return 0;
}

References cpu_to_le16, ena_cq::cqe, DBGC, ena_aq_req::destroy_cq, ena_admin(), ena_admin_req(), ENA_DESTROY_CQ, free_phys(), ena_aq_req::header, ena_destroy_cq_req::id, ena_cq::id, ena_cq::len, ena_aq_header::opcode, ena_cq::raw, rc, rsp, and strerror().

Referenced by ena_create_qp(), and ena_destroy_qp().

ena_create_qp()

static int ena_create_qp ( struct ena_nic *  ena, struct ena_qp *  qp  )

static

Create queue pair.

Parameters

ena	ENA device
qp	Queue pair

Return values

rc	Return status code

Definition at line 659 of file ena.c.

                                                                    {
        int rc;

        /* Create completion queue */
        if ( ( rc = ena_create_cq ( ena, &qp->cq ) ) != 0 )
                goto err_create_cq;

        /* Create submission queue */
        if ( ( rc = ena_create_sq ( ena, &qp->sq, &qp->cq ) ) != 0 )
                goto err_create_sq;

        return 0;

        ena_destroy_sq ( ena, &qp->sq );
 err_create_sq:
        ena_destroy_cq ( ena, &qp->cq );
 err_create_cq:
        return rc;
}

References ena_create_cq(), ena_create_sq(), ena_destroy_cq(), ena_destroy_sq(), qp, and rc.

Referenced by ena_open().

ena_destroy_qp()

static int ena_destroy_qp ( struct ena_nic *  ena, struct ena_qp *  qp  )

static

Destroy queue pair.

Parameters

ena	ENA device
qp	Queue pair

Return values

rc	Return status code

Definition at line 686 of file ena.c.

                                                                     {

        /* Destroy submission queue */
        ena_destroy_sq ( ena,  &qp->sq );

        /* Destroy completion queue */
        ena_destroy_cq ( ena, &qp->cq );

        return 0;
}

References ena_destroy_cq(), ena_destroy_sq(), and qp.

Referenced by ena_close(), and ena_open().

ena_get_device_attributes()

static int ena_get_device_attributes ( struct net_device *  netdev )

static

Get device attributes.

Parameters

netdev

Network device

Return values

rc	Return status code

Definition at line 703 of file ena.c.

                                                                   {
        struct ena_nic *ena = netdev->priv;
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        union ena_feature *feature;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_GET_FEATURE;
        req->get_feature.id = ENA_DEVICE_ATTRIBUTES;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not get device attributes: %s\n",
                       ena, strerror ( rc ) );
                return rc;
        }

        /* Parse response */
        feature = &rsp->get_feature.feature;
        memcpy ( netdev->hw_addr, feature->device.mac, ETH_ALEN );
        netdev->max_pkt_len = le32_to_cpu ( feature->device.mtu );
        netdev->mtu = ( netdev->max_pkt_len - ETH_HLEN );
        ena->features = le32_to_cpu ( feature->device.features );

        DBGC ( ena, "ENA %p MAC %s MTU %zd features %#08x\n",
               ena, eth_ntoa ( netdev->hw_addr ), netdev->max_pkt_len,
               ena->features );
        return 0;
}

References DBGC, ena_admin(), ena_admin_req(), ENA_DEVICE_ATTRIBUTES, ENA_GET_FEATURE, ETH_ALEN, ETH_HLEN, eth_ntoa(), feature, ena_nic::features, ena_aq_req::get_feature, ena_aq_req::header, net_device::hw_addr, ena_get_feature_req::id, le32_to_cpu, net_device::max_pkt_len, memcpy(), net_device::mtu, netdev, ena_aq_header::opcode, net_device::priv, rc, rsp, and strerror().

Referenced by ena_probe().

ena_set_host_attributes()

static int ena_set_host_attributes ( struct ena_nic *  ena )

static

Set host attributes.

Parameters

ena	ENA device

Return values

rc	Return status code

Definition at line 741 of file ena.c.

                                                           {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        union ena_feature *feature;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_SET_FEATURE;
        req->set_feature.id = ENA_HOST_ATTRIBUTES;
        feature = &req->set_feature.feature;
        feature->host.info = cpu_to_le64 ( virt_to_bus ( ena->info ) );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not set host attributes: %s\n",
                       ena, strerror ( rc ) );
                return rc;
        }

        return 0;
}

References cpu_to_le64, DBGC, ena_admin(), ena_admin_req(), ENA_HOST_ATTRIBUTES, ENA_SET_FEATURE, feature, ena_set_feature_req::feature, ena_aq_req::header, ena_set_feature_req::id, ena_nic::info, ena_aq_header::opcode, rc, rsp, ena_aq_req::set_feature, strerror(), and virt_to_bus().

Referenced by ena_probe().

ena_llq_config()

static int ena_llq_config ( struct ena_nic *  ena )

static

Configure low latency queues.

Parameters

ena	ENA device

Return values

rc	Return status code

Definition at line 770 of file ena.c.

                                                  {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        union ena_feature *feature;
        uint16_t header;
        uint16_t size;
        uint16_t desc;
        uint16_t stride;
        uint16_t mode;
        int rc;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_GET_FEATURE;
        req->get_feature.id = ENA_LLQ_CONFIG;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not get LLQ configuration: %s\n",
                       ena, strerror ( rc ) );
                return rc;
        }

        /* Parse response */
        feature = &rsp->get_feature.feature;
        header = le16_to_cpu ( feature->llq.header.supported );
        size = le16_to_cpu ( feature->llq.size.supported );
        desc = le16_to_cpu ( feature->llq.desc.supported );
        stride = le16_to_cpu ( feature->llq.stride.supported );
        mode = le16_to_cpu ( feature->llq.mode );
        DBGC ( ena, "ENA %p LLQ supports %02x:%02x:%02x:%02x:%02x with %dx%d "
               "entries\n", ena, header, size, desc, stride, mode,
               le32_to_cpu ( feature->llq.queues ),
               le32_to_cpu ( feature->llq.count ) );

        /* Check for a supported configuration */
        if ( ! feature->llq.queues ) {
                DBGC ( ena, "ENA %p LLQ has no queues\n", ena );
                return -ENOTSUP;
        }
        if ( ! ( header & ENA_LLQ_HEADER_INLINE ) ) {
                DBGC ( ena, "ENA %p LLQ does not support inline headers\n",
                       ena );
                return -ENOTSUP;
        }
        if ( ! ( size & ENA_LLQ_SIZE_128 ) ) {
                DBGC ( ena, "ENA %p LLQ does not support 128-byte entries\n",
                       ena );
                return -ENOTSUP;
        }
        if ( ! ( desc & ENA_LLQ_DESC_2 ) ) {
                DBGC ( ena, "ENA %p LLQ does not support two-descriptor "
                       "entries\n", ena );
                return -ENOTSUP;
        }

        /* Enable a minimal configuration */
        header = ENA_LLQ_HEADER_INLINE;
        size = ENA_LLQ_SIZE_128;
        desc = ENA_LLQ_DESC_2;
        stride &= ( -stride ); /* Don't care: use first supported option */
        DBGC ( ena, "ENA %p LLQ enabling %02x:%02x:%02x:%02x:%02x\n",
               ena, header, size, desc, stride, mode );

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_SET_FEATURE;
        req->set_feature.id = ENA_LLQ_CONFIG;
        feature = &req->set_feature.feature;
        feature->llq.header.enabled = cpu_to_le16 ( header );
        feature->llq.size.enabled = cpu_to_le16 ( size );
        feature->llq.desc.enabled = cpu_to_le16 ( desc );
        feature->llq.stride.enabled = cpu_to_le16 ( stride );
        feature->llq.mode = cpu_to_le16 ( mode );

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not set LLQ configuration: %s\n",
                       ena, strerror ( rc ) );
                return rc;
        }

        /* Use on-device memory for transmit queue */
        ena->tx.sq.policy |= ENA_SQ_DEVICE_MEMORY;
        ena->tx.sq.inlined = sizeof ( ena->tx.sq.sqe.llq->inlined );

        return 0;
}

References cpu_to_le16, DBGC, desc, ena_admin(), ena_admin_req(), ENA_GET_FEATURE, ENA_LLQ_CONFIG, ENA_LLQ_DESC_2, ENA_LLQ_HEADER_INLINE, ENA_LLQ_SIZE_128, ENA_SET_FEATURE, ENA_SQ_DEVICE_MEMORY, ENOTSUP, feature, ena_set_feature_req::feature, ena_aq_req::get_feature, header, ena_aq_req::header, ena_get_feature_req::id, ena_set_feature_req::id, ena_tx_llqe::inlined, ena_sq::inlined, le16_to_cpu, le32_to_cpu, ena_sq::llq, mode, ena_aq_header::opcode, ena_sq::policy, rc, rsp, ena_aq_req::set_feature, size, ena_qp::sq, ena_sq::sqe, strerror(), stride, and ena_nic::tx.

Referenced by ena_probe().

ena_get_stats()

static int ena_get_stats ( struct ena_nic *  ena )

static

Get statistics (for debugging)

Parameters

ena	ENA device

Return values

rc	Return status code

Definition at line 865 of file ena.c.

                                                 {
        union ena_aq_req *req;
        union ena_acq_rsp *rsp;
        struct ena_get_stats_rsp *stats;
        int rc;

        /* Do nothing unless debug messages are enabled */
        if ( ! DBG_LOG )
                return 0;

        /* Construct request */
        req = ena_admin_req ( ena );
        req->header.opcode = ENA_GET_STATS;
        req->get_stats.type = ENA_STATS_TYPE_BASIC;
        req->get_stats.scope = ENA_STATS_SCOPE_ETH;
        req->get_stats.device = ENA_DEVICE_MINE;

        /* Issue request */
        if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 ) {
                DBGC ( ena, "ENA %p could not get statistics: %s\n",
                       ena, strerror ( rc ) );
                return rc;
        }

        /* Parse response */
        stats = &rsp->get_stats;
        DBGC ( ena, "ENA %p TX bytes %#llx packets %#llx\n", ena,
               ( ( unsigned long long ) le64_to_cpu ( stats->tx_bytes ) ),
               ( ( unsigned long long ) le64_to_cpu ( stats->tx_packets ) ) );
        DBGC ( ena, "ENA %p RX bytes %#llx packets %#llx drops %#llx\n", ena,
               ( ( unsigned long long ) le64_to_cpu ( stats->rx_bytes ) ),
               ( ( unsigned long long ) le64_to_cpu ( stats->rx_packets ) ),
               ( ( unsigned long long ) le64_to_cpu ( stats->rx_drops ) ) );

        return 0;
}

References DBG_LOG, DBGC, ena_get_stats_req::device, ena_admin(), ena_admin_req(), ENA_DEVICE_MINE, ENA_GET_STATS, ENA_STATS_SCOPE_ETH, ENA_STATS_TYPE_BASIC, ena_aq_req::get_stats, ena_aq_req::header, le64_to_cpu, ena_aq_header::opcode, rc, rsp, ena_get_stats_rsp::rx_bytes, ena_get_stats_rsp::rx_drops, ena_get_stats_rsp::rx_packets, ena_get_stats_req::scope, strerror(), ena_get_stats_rsp::tx_bytes, ena_get_stats_rsp::tx_packets, and ena_get_stats_req::type.

Referenced by ena_close().

ena_refill_rx()

static void ena_refill_rx ( struct net_device *  netdev )

static

Refill receive queue.

Parameters

netdev

Network device

Definition at line 914 of file ena.c.

                                                        {
        struct ena_nic *ena = netdev->priv;
        struct io_buffer *iobuf;
        struct ena_rx_sqe *sqe;
        physaddr_t address;
        size_t len = netdev->max_pkt_len;
        unsigned int refilled = 0;
        unsigned int index;
        unsigned int id;

        /* Refill queue */
        while ( ( ena->rx.sq.prod - ena->rx.cq.cons ) < ena->rx.sq.fill ) {

                /* Allocate I/O buffer */
                iobuf = alloc_iob ( len );
                if ( ! iobuf ) {
                        /* Wait for next refill */
                        break;
                }

                /* Get next submission queue entry and buffer ID */
                index = ( ena->rx.sq.prod % ENA_RX_COUNT );
                sqe = &ena->rx.sq.sqe.rx[index];
                id = ena->rx_ids[index];

                /* Construct submission queue entry */
                address = virt_to_bus ( iobuf->data );
                sqe->len = cpu_to_le16 ( len );
                sqe->id = cpu_to_le16 ( id );
                sqe->address = cpu_to_le64 ( address );
                wmb();
                sqe->flags = ( ENA_SQE_FIRST | ENA_SQE_LAST | ENA_SQE_CPL |
                               ena->rx.sq.phase );

                /* Increment producer counter */
                ena->rx.sq.prod++;
                if ( ( ena->rx.sq.prod % ENA_RX_COUNT ) == 0 )
                        ena->rx.sq.phase ^= ENA_SQE_PHASE;

                /* Record I/O buffer */
                assert ( ena->rx_iobuf[id] == NULL );
                ena->rx_iobuf[id] = iobuf;

                DBGC2 ( ena, "ENA %p RX %d at [%08llx,%08llx)\n", ena, id,
                        ( ( unsigned long long ) address ),
                        ( ( unsigned long long ) address + len ) );
                refilled++;
        }

        /* Ring doorbell, if applicable */
        if ( refilled ) {
                wmb();
                writel ( ena->rx.sq.prod, ( ena->regs + ena->rx.sq.doorbell ) );
        }
}

References address, ena_tx_sqe::address, alloc_iob(), assert(), ena_cq::cons, cpu_to_le16, cpu_to_le64, ena_qp::cq, io_buffer::data, DBGC2, ena_sq::doorbell, ENA_RX_COUNT, ENA_SQE_CPL, ENA_SQE_FIRST, ENA_SQE_LAST, ENA_SQE_PHASE, ena_sq::fill, ena_tx_sqe::flags, id, ena_tx_sqe::id, index, len, ena_tx_sqe::len, net_device::max_pkt_len, netdev, NULL, ena_sq::phase, net_device::priv, ena_sq::prod, ena_nic::regs, ena_sq::rx, ena_nic::rx, ena_nic::rx_ids, ena_nic::rx_iobuf, ena_qp::sq, sqe, ena_sq::sqe, virt_to_bus(), wmb(), and writel().

Referenced by ena_open(), and ena_poll().

ena_empty_rx()

static void ena_empty_rx ( struct ena_nic *  ena )

static

Discard unused receive I/O buffers.

Parameters

ena	ENA device

Definition at line 975 of file ena.c.

                                                 {
        struct io_buffer *iobuf;
        unsigned int i;

        for ( i = 0 ; i < ENA_RX_COUNT ; i++ ) {
                iobuf = ena->rx_iobuf[i];
                ena->rx_iobuf[i] = NULL;
                if ( iobuf )
                        free_iob ( iobuf );
        }
}

References ENA_RX_COUNT, free_iob(), NULL, and ena_nic::rx_iobuf.

Referenced by ena_close().

ena_cancel_tx()

static void ena_cancel_tx ( struct net_device *  netdev )

static

Cancel uncompleted transmit I/O buffers.

Parameters

netdev

Network device

Definition at line 992 of file ena.c.

                                                        {
        struct ena_nic *ena = netdev->priv;
        struct io_buffer *iobuf;
        unsigned int i;

        for ( i = 0 ; i < ENA_TX_COUNT ; i++ ) {
                iobuf = ena->tx_iobuf[i];
                ena->tx_iobuf[i] = NULL;
                if ( iobuf )
                        netdev_tx_complete_err ( netdev, iobuf, -ECANCELED );
        }
}

References ECANCELED, ENA_TX_COUNT, netdev, netdev_tx_complete_err(), NULL, net_device::priv, and ena_nic::tx_iobuf.

Referenced by ena_close().

ena_open()

static int ena_open ( struct net_device *  netdev )

static

Open network device.

Parameters

netdev

Network device

Return values

rc	Return status code

Definition at line 1011 of file ena.c.

                                                  {
        struct ena_nic *ena = netdev->priv;
        int rc;

        /* Create transmit queue pair */
        if ( ( rc = ena_create_qp ( ena, &ena->tx ) ) != 0 )
                goto err_create_tx;

        /* Create receive queue pair */
        if ( ( rc = ena_create_qp ( ena, &ena->rx ) ) != 0 )
                goto err_create_rx;

        /* Refill receive queue */
        ena_refill_rx ( netdev );

        return 0;

        ena_destroy_qp ( ena, &ena->rx );
 err_create_rx:
        ena_destroy_qp ( ena, &ena->tx );
 err_create_tx:
        return rc;
}

References ena_create_qp(), ena_destroy_qp(), ena_refill_rx(), netdev, net_device::priv, rc, ena_nic::rx, and ena_nic::tx.

ena_close()

static void ena_close ( struct net_device *  netdev )

static

Close network device.

Parameters

netdev

Network device

Definition at line 1040 of file ena.c.

                                                    {
        struct ena_nic *ena = netdev->priv;

        /* Dump statistics (for debugging) */
        ena_get_stats ( ena );

        /* Destroy receive queue pair */
        ena_destroy_qp ( ena, &ena->rx );

        /* Discard any unused receive buffers */
        ena_empty_rx ( ena );

        /* Destroy transmit queue pair */
        ena_destroy_qp ( ena, &ena->tx );

        /* Cancel any uncompleted transmit buffers */
        ena_cancel_tx ( netdev );
}

References ena_cancel_tx(), ena_destroy_qp(), ena_empty_rx(), ena_get_stats(), netdev, net_device::priv, ena_nic::rx, and ena_nic::tx.

ena_transmit()

static int ena_transmit ( struct net_device *  netdev, struct io_buffer *  iobuf  )

static

Transmit packet.

Parameters

netdev	Network device
iobuf	I/O buffer

Return values

rc	Return status code

Definition at line 1066 of file ena.c.

                                                                               {
        struct ena_nic *ena = netdev->priv;
        struct ena_tx_sqe *sqe;
        struct ena_tx_llqe *llqe;
        const uint64_t *src;
        uint64_t *dest;
        physaddr_t address;
        unsigned int index;
        unsigned int id;
        unsigned int i;
        uint8_t flags;
        size_t inlined;
        size_t len;

        /* Get next submission queue entry */
        if ( ( ena->tx.sq.prod - ena->tx.cq.cons ) >= ena->tx.sq.fill ) {
                DBGC ( ena, "ENA %p out of transmit descriptors\n", ena );
                return -ENOBUFS;
        }
        index = ( ena->tx.sq.prod % ENA_TX_COUNT );
        sqe = &ena->tx.sq.sqe.tx[index];
        id = ena->tx_ids[index];

        /* Construct submission queue entry values */
        address = virt_to_bus ( iobuf->data );
        len = iob_len ( iobuf );
        inlined = ena->tx.sq.inlined;
        if ( inlined > len )
                inlined = len;
        len -= inlined;
        address += inlined;
        flags = ( ENA_SQE_FIRST | ENA_SQE_LAST | ENA_SQE_CPL |
                  ena->tx.sq.phase );

        /* Prepare low-latency queue bounce buffer, if applicable */
        llqe = ena->tx.sq.sqe.llq;
        if ( ena->tx.sq.llqe ) {

                /* Construct zero-information metadata queue entry */
                llqe->meta.meta = ENA_TX_SQE_META;
                llqe->meta.flags = ( flags & ~( ENA_SQE_LAST | ENA_SQE_CPL ) );

                /* Copy inlined data */
                memcpy ( llqe->inlined, iobuf->data, inlined );

                /* Place submission queue entry within bounce buffer */
                sqe = &llqe->sqe;
                flags &= ~ENA_SQE_FIRST;
        }

        /* Construct submission queue entry */
        sqe->len = cpu_to_le16 ( len );
        sqe->id = cpu_to_le16 ( id );
        sqe->address = cpu_to_le64 ( address );
        sqe->inlined = inlined;
        wmb();
        sqe->flags = flags;
        wmb();

        /* Copy bounce buffer to on-device memory, if applicable */
        if ( ena->tx.sq.llqe ) {
                src = ( ( const void * ) llqe );
                dest = ( ena->tx.sq.llqe + ( index * sizeof ( *llqe ) ) );
                for ( i = 0 ; i < ( sizeof ( *llqe ) / sizeof ( *src ) ); i++ )
                        writeq ( *(src++), dest++ );
                wmb();
        }

        /* Increment producer counter */
        ena->tx.sq.prod++;
        if ( ( ena->tx.sq.prod % ENA_TX_COUNT ) == 0 )
                ena->tx.sq.phase ^= ENA_SQE_PHASE;

        /* Record I/O buffer */
        assert ( ena->tx_iobuf[id] == NULL );
        ena->tx_iobuf[id] = iobuf;

        /* Ring doorbell */
        writel ( ena->tx.sq.prod, ( ena->regs + ena->tx.sq.doorbell ) );

        DBGC2 ( ena, "ENA %p TX %d at [%08llx,%08llx)\n", ena, id,
                ( ( unsigned long long ) address ),
                ( ( unsigned long long ) address + len ) );
        return 0;
}

References address, ena_tx_sqe::address, assert(), ena_cq::cons, cpu_to_le16, cpu_to_le64, ena_qp::cq, io_buffer::data, DBGC, DBGC2, dest, ena_sq::doorbell, ENA_SQE_CPL, ENA_SQE_FIRST, ENA_SQE_LAST, ENA_SQE_PHASE, ENA_TX_COUNT, ENA_TX_SQE_META, ENOBUFS, ena_sq::fill, flags, ena_tx_sqe::flags, id, ena_tx_sqe::id, index, inlined, ena_tx_sqe::inlined, ena_sq::inlined, iob_len(), len, ena_tx_sqe::len, ena_sq::llq, llqe, ena_sq::llqe, memcpy(), netdev, NULL, ena_sq::phase, net_device::priv, ena_sq::prod, ena_nic::regs, ena_qp::sq, sqe, ena_sq::sqe, src, ena_sq::tx, ena_nic::tx, ena_nic::tx_ids, ena_nic::tx_iobuf, virt_to_bus(), wmb(), writel(), and writeq().

ena_poll_tx()

static void ena_poll_tx ( struct net_device *  netdev )

static

Poll for completed transmissions.

Parameters

netdev

Network device

Definition at line 1157 of file ena.c.

                                                      {
        struct ena_nic *ena = netdev->priv;
        struct ena_tx_cqe *cqe;
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int id;

        /* Check for completed packets */
        while ( ena->tx.cq.cons != ena->tx.sq.prod ) {

                /* Get next completion queue entry */
                index = ( ena->tx.cq.cons & ena->tx.cq.mask );
                cqe = &ena->tx.cq.cqe.tx[index];

                /* Stop if completion queue entry is empty */
                if ( ( cqe->flags ^ ena->tx.cq.phase ) & ENA_CQE_PHASE )
                        return;

                /* Increment consumer counter */
                ena->tx.cq.cons++;
                if ( ! ( ena->tx.cq.cons & ena->tx.cq.mask ) )
                        ena->tx.cq.phase ^= ENA_CQE_PHASE;

                /* Identify and free buffer ID */
                id = ENA_TX_CQE_ID ( le16_to_cpu ( cqe->id ) );
                ena->tx_ids[index] = id;

                /* Identify I/O buffer */
                iobuf = ena->tx_iobuf[id];
                assert ( iobuf != NULL );
                ena->tx_iobuf[id] = NULL;

                /* Complete transmit */
                DBGC2 ( ena, "ENA %p TX %d complete\n", ena, id );
                netdev_tx_complete ( netdev, iobuf );
        }
}

References assert(), ena_cq::cons, ena_qp::cq, ena_cq::cqe, DBGC2, ENA_CQE_PHASE, ENA_TX_CQE_ID, ena_tx_cqe::flags, id, ena_tx_cqe::id, index, le16_to_cpu, ena_cq::mask, netdev, netdev_tx_complete(), NULL, ena_cq::phase, net_device::priv, ena_sq::prod, ena_qp::sq, ena_cq::tx, ena_nic::tx, ena_nic::tx_ids, and ena_nic::tx_iobuf.

Referenced by ena_poll().

ena_poll_rx()

static void ena_poll_rx ( struct net_device *  netdev )

static

Poll for received packets.

Parameters

netdev

Network device

Definition at line 1200 of file ena.c.

                                                      {
        struct ena_nic *ena = netdev->priv;
        struct ena_rx_cqe *cqe;
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int id;
        size_t len;

        /* Check for received packets */
        while ( ena->rx.cq.cons != ena->rx.sq.prod ) {

                /* Get next completion queue entry */
                index = ( ena->rx.cq.cons & ena->rx.cq.mask );
                cqe = &ena->rx.cq.cqe.rx[index];

                /* Stop if completion queue entry is empty */
                if ( ( cqe->flags ^ ena->rx.cq.phase ) & ENA_CQE_PHASE )
                        return;

                /* Increment consumer counter */
                ena->rx.cq.cons++;
                if ( ! ( ena->rx.cq.cons & ena->rx.cq.mask ) )
                        ena->rx.cq.phase ^= ENA_CQE_PHASE;

                /* Identify and free buffer ID */
                id = le16_to_cpu ( cqe->id );
                ena->rx_ids[index] = id;

                /* Populate I/O buffer */
                iobuf = ena->rx_iobuf[id];
                assert ( iobuf != NULL );
                ena->rx_iobuf[id] = NULL;
                len = le16_to_cpu ( cqe->len );
                iob_put ( iobuf, len );

                /* Hand off to network stack */
                DBGC2 ( ena, "ENA %p RX %d complete (length %zd)\n",
                        ena, id, len );
                netdev_rx ( netdev, iobuf );
        }
}

References assert(), ena_cq::cons, ena_qp::cq, ena_cq::cqe, DBGC2, ENA_CQE_PHASE, ena_rx_cqe::flags, id, ena_rx_cqe::id, index, iob_put, le16_to_cpu, len, ena_rx_cqe::len, ena_cq::mask, netdev, netdev_rx(), NULL, ena_cq::phase, net_device::priv, ena_sq::prod, ena_cq::rx, ena_nic::rx, ena_nic::rx_ids, ena_nic::rx_iobuf, and ena_qp::sq.

Referenced by ena_poll().

ena_poll()

static void ena_poll ( struct net_device *  netdev )

static

Poll for completed and received packets.

Parameters

netdev

Network device

Definition at line 1247 of file ena.c.

                                                   {

        /* Poll for transmit completions */
        ena_poll_tx ( netdev );

        /* Poll for receive completions */
        ena_poll_rx ( netdev );

        /* Refill receive ring */
        ena_refill_rx ( netdev );
}

References ena_poll_rx(), ena_poll_tx(), ena_refill_rx(), and netdev.

ena_membases()

static int ena_membases ( struct ena_nic *  ena, struct pci_device *  pci, unsigned long *  prefmembase, unsigned long *  prefmemsize  )

static

Assign memory BARs.

Parameters

ena	ENA device
pci	PCI device
prefmembase	On-device memory base address to fill in
prefmemsize	On-device memory size to fill in

Return values

rc	Return status code

Some BIOSes in AWS EC2 are observed to fail to assign a base address to the ENA device. The device is the only device behind its bridge, and the BIOS does assign a memory window to the bridge. We therefore place the device at the start of the memory window.

Definition at line 1288 of file ena.c.

                                                       {
        struct pci_bridge *bridge;

        /* Get on-device memory base address and size */
        *prefmembase = pci_bar_start ( pci, ENA_MEM_BAR );
        *prefmemsize = pci_bar_size ( pci, ENA_MEM_BAR );

        /* Do nothing if addresses are already assigned */
        if ( pci->membase && ( *prefmembase || ( ! *prefmemsize ) ) )
                return 0;

        /* Locate PCI bridge */
        bridge = pcibridge_find ( pci );
        if ( ! bridge ) {
                DBGC ( ena, "ENA %p found no PCI bridge\n", ena );
                return -ENOTCONN;
        }
        DBGC ( ena, "ENA %p at " PCI_FMT " claiming bridge " PCI_FMT "\n",
               ena, PCI_ARGS ( pci ), PCI_ARGS ( bridge->pci ) );

        /* Sanity check */
        if ( PCI_SLOT ( pci->busdevfn ) || PCI_FUNC ( pci->busdevfn ) ) {
                DBGC ( ena, "ENA %p at " PCI_FMT " may not be only device "
                       "on bus\n", ena, PCI_ARGS ( pci ) );
                return -ENOTSUP;
        }

        /* Place register BAR at start of memory window, if applicable */
        if ( ! pci->membase ) {
                pci_bar_set ( pci, ENA_REGS_BAR, bridge->membase );
                pci->membase = bridge->membase;
                DBGC ( ena, "ENA %p at " PCI_FMT " claiming mem %08lx\n",
                       ena, PCI_ARGS ( pci ), pci->membase );
        }

        /* Place memory BAR at start of prefetchable window, if applicable */
        if ( *prefmemsize && ( ! *prefmembase ) ) {
                pci_bar_set ( pci, ENA_MEM_BAR, bridge->prefmembase );
                *prefmembase = bridge->prefmembase;
                DBGC ( ena, "ENA %p at " PCI_FMT " claiming prefmem %08lx\n",
                       ena, PCI_ARGS ( pci ), *prefmembase );
        }

        return 0;
}

References pci_device::busdevfn, DBGC, ENA_MEM_BAR, ENA_REGS_BAR, ENOTCONN, ENOTSUP, pci_bridge::membase, pci_device::membase, pci_bridge::pci, PCI_ARGS, pci_bar_set(), pci_bar_size(), pci_bar_start(), PCI_FMT, PCI_FUNC, PCI_SLOT, pcibridge_find(), and pci_bridge::prefmembase.

Referenced by ena_probe().

ena_probe()

static int ena_probe ( struct pci_device *  pci )

static

Probe PCI device.

Parameters

pci	PCI device

Return values

rc	Return status code

Definition at line 1342 of file ena.c.

                                                {
        struct net_device *netdev;
        struct ena_nic *ena;
        struct ena_host_info *info;
        unsigned long prefmembase;
        unsigned long prefmemsize;
        int rc;

        /* Allocate and initialise net device */
        netdev = alloc_etherdev ( sizeof ( *ena ) );
        if ( ! netdev ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        netdev_init ( netdev, &ena_operations );
        ena = netdev->priv;
        pci_set_drvdata ( pci, netdev );
        netdev->dev = &pci->dev;
        memset ( ena, 0, sizeof ( *ena ) );
        ena->acq.phase = ENA_ACQ_PHASE;
        ena_cq_init ( &ena->tx.cq, ENA_TX_COUNT,
                      sizeof ( ena->tx.cq.cqe.tx[0] ) );
        ena_sq_init ( &ena->tx.sq, ENA_SQ_TX, ENA_TX_COUNT,
                      sizeof ( ena->tx.sq.sqe.tx[0] ), ena->tx_ids );
        ena_cq_init ( &ena->rx.cq, ENA_RX_COUNT,
                      sizeof ( ena->rx.cq.cqe.rx[0] ) );
        ena_sq_init ( &ena->rx.sq, ENA_SQ_RX, ENA_RX_COUNT,
                      sizeof ( ena->rx.sq.sqe.rx[0] ), ena->rx_ids );

        /* Fix up PCI device */
        adjust_pci_device ( pci );

        /* Fix up PCI BAR if left unassigned by BIOS */
        if ( ( rc = ena_membases ( ena, pci, &prefmembase,
                                   &prefmemsize ) ) != 0 ) {
                goto err_membases;
        }

        /* Map registers */
        ena->regs = pci_ioremap ( pci, pci->membase, ENA_REGS_SIZE );
        if ( ! ena->regs ) {
                rc = -ENODEV;
                goto err_regs;
        }

        /* Map device memory */
        if ( prefmemsize ) {
                ena->mem = pci_ioremap ( pci, prefmembase, prefmemsize );
                if ( ! ena->mem ) {
                        rc = -ENODEV;
                        goto err_mem;
                }
                DBGC ( ena, "ENA %p has %ldkB of on-device memory\n",
                       ena, ( prefmemsize >> 10 ) );
        }

        /* Allocate and initialise host info */
        info = malloc_phys ( PAGE_SIZE, PAGE_SIZE );
        if ( ! info ) {
                rc = -ENOMEM;
                goto err_info;
        }
        ena->info = info;
        memset ( info, 0, PAGE_SIZE );
        info->type = cpu_to_le32 ( ENA_HOST_INFO_TYPE_IPXE );
        snprintf ( info->dist_str, sizeof ( info->dist_str ), "%s",
                   ( product_name[0] ? product_name : product_short_name ) );
        snprintf ( info->kernel_str, sizeof ( info->kernel_str ), "%s",
                   product_version );
        info->version = cpu_to_le32 ( ENA_HOST_INFO_VERSION_WTF );
        info->spec = cpu_to_le16 ( ENA_HOST_INFO_SPEC_2_0 );
        info->busdevfn = cpu_to_le16 ( pci->busdevfn );
        DBGC2 ( ena, "ENA %p host info:\n", ena );
        DBGC2_HDA ( ena, virt_to_phys ( info ), info, sizeof ( *info ) );

        /* Reset the NIC */
        if ( ( rc = ena_reset ( ena ) ) != 0 )
                goto err_reset;

        /* Create admin queues */
        if ( ( rc = ena_create_admin ( ena ) ) != 0 )
                goto err_create_admin;

        /* Create async event notification queue */
        if ( ( rc = ena_create_async ( ena ) ) != 0 )
                goto err_create_async;

        /* Set host attributes */
        if ( ( rc = ena_set_host_attributes ( ena ) ) != 0 )
                goto err_set_host_attributes;

        /* Fetch MAC address */
        if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
                goto err_get_device_attributes;

        /* Attempt to configure low latency queues, if applicable.
         * Ignore any errors and continue without using LLQs.
         */
        if ( ena->mem && ( ena->features & ENA_FEATURE_LLQ ) )
                ena_llq_config ( ena );

        /* Register network device */
        if ( ( rc = register_netdev ( netdev ) ) != 0 )
                goto err_register_netdev;

        /* Mark as link up, since we have no way to test link state on
         * this hardware.
         */
        netdev_link_up ( netdev );

        return 0;

        unregister_netdev ( netdev );
 err_register_netdev:
 err_get_device_attributes:
 err_set_host_attributes:
        ena_destroy_async ( ena );
 err_create_async:
        ena_destroy_admin ( ena );
 err_create_admin:
        ena_reset ( ena );
 err_reset:
        free_phys ( ena->info, PAGE_SIZE );
 err_info:
        if ( ena->mem )
                iounmap ( ena->mem );
 err_mem:
        iounmap ( ena->regs );
 err_regs:
 err_membases:
        netdev_nullify ( netdev );
        netdev_put ( netdev );
 err_alloc:
        return rc;
}

References ena_nic::acq, adjust_pci_device(), alloc_etherdev(), pci_device::busdevfn, cpu_to_le16, cpu_to_le32, ena_qp::cq, ena_cq::cqe, DBGC, DBGC2, DBGC2_HDA, pci_device::dev, net_device::dev, ENA_ACQ_PHASE, ena_create_admin(), ena_create_async(), ena_destroy_admin(), ena_destroy_async(), ENA_FEATURE_LLQ, ena_get_device_attributes(), ENA_HOST_INFO_SPEC_2_0, ENA_HOST_INFO_TYPE_IPXE, ENA_HOST_INFO_VERSION_WTF, ena_llq_config(), ena_membases(), ena_operations, ENA_REGS_SIZE, ena_reset(), ENA_RX_COUNT, ena_set_host_attributes(), ENA_SQ_RX, ENA_SQ_TX, ENA_TX_COUNT, ENODEV, ENOMEM, ena_nic::features, free_phys(), info, ena_nic::info, iounmap(), malloc_phys(), ena_nic::mem, pci_device::membase, memset(), netdev, netdev_init(), netdev_link_up(), netdev_nullify(), netdev_put(), PAGE_SIZE, pci_ioremap(), pci_set_drvdata(), ena_acq::phase, net_device::priv, product_name, product_short_name, product_version, rc, register_netdev(), ena_nic::regs, ena_sq::rx, ena_cq::rx, ena_nic::rx, ena_nic::rx_ids, snprintf(), ena_qp::sq, ena_sq::sqe, ena_sq::tx, ena_cq::tx, ena_nic::tx, ena_nic::tx_ids, and unregister_netdev().

ena_remove()

static void ena_remove ( struct pci_device *  pci )

static

Remove PCI device.

Parameters

pci	PCI device

Definition at line 1483 of file ena.c.

                                                  {
        struct net_device *netdev = pci_get_drvdata ( pci );
        struct ena_nic *ena = netdev->priv;

        /* Unregister network device */
        unregister_netdev ( netdev );

        /* Destroy async event notification queue */
        ena_destroy_async ( ena );

        /* Destroy admin queues */
        ena_destroy_admin ( ena );

        /* Reset card */
        ena_reset ( ena );

        /* Free host info */
        free_phys ( ena->info, PAGE_SIZE );

        /* Unmap registers and on-device memory */
        if ( ena->mem )
                iounmap ( ena->mem );
        iounmap ( ena->regs );

        /* Free network device */
        netdev_nullify ( netdev );
        netdev_put ( netdev );
}

References ena_destroy_admin(), ena_destroy_async(), ena_reset(), free_phys(), ena_nic::info, iounmap(), ena_nic::mem, netdev, netdev_nullify(), netdev_put(), PAGE_SIZE, pci_get_drvdata(), net_device::priv, ena_nic::regs, and unregister_netdev().

Variable Documentation

ena_operations

struct net_device_operations ena_operations

static

Initial value:

= {
        .open           = ena_open,
        .close          = ena_close,
        .transmit       = ena_transmit,
        .poll           = ena_poll,
}

ENA network device operations.

Definition at line 1260 of file ena.c.

Referenced by ena_probe().

ena_nics

struct pci_device_id ena_nics[]

static

Initial value:

= {
        PCI_ROM ( 0x1d0f, 0xec20, "ena-vf", "ENA VF", 0 ),
        PCI_ROM ( 0x1d0f, 0xec21, "ena-vf-llq", "ENA VF (LLQ)", 0 ),
}

ENA PCI device IDs.

Definition at line 1513 of file ena.c.

__pci_driver

struct pci_driver ena_driver __pci_driver

Initial value:

= {
        .ids = ena_nics,
        .id_count = ( sizeof ( ena_nics ) / sizeof ( ena_nics[0] ) ),
        .probe = ena_probe,
        .remove = ena_remove,
}

ENA PCI driver.

Definition at line 1519 of file ena.c.