gve.c File Reference

Google Virtual Ethernet network driver. More...

#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>
#include <byteswap.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include <ipxe/if_ether.h>
#include <ipxe/iobuf.h>
#include <ipxe/dma.h>
#include <ipxe/pci.h>
#include <ipxe/fault.h>
#include "gve.h"

Go to the source code of this file.

Macros
#define	EINFO_EIO_ADMIN_UNSET   __einfo_uniqify ( EINFO_EIO, 0x00, "Uncompleted" )
#define	EIO_ADMIN_UNSET   __einfo_error ( EINFO_EIO_ADMIN_UNSET )
#define	EINFO_EIO_ADMIN_ABORTED   __einfo_uniqify ( EINFO_EIO, 0x10, "Aborted" )
#define	EIO_ADMIN_ABORTED   __einfo_error ( EINFO_EIO_ADMIN_ABORTED )
#define	EINFO_EIO_ADMIN_EXISTS   __einfo_uniqify ( EINFO_EIO, 0x11, "Already exists" )
#define	EIO_ADMIN_EXISTS   __einfo_error ( EINFO_EIO_ADMIN_EXISTS )
#define	EINFO_EIO_ADMIN_CANCELLED   __einfo_uniqify ( EINFO_EIO, 0x12, "Cancelled" )
#define	EIO_ADMIN_CANCELLED   __einfo_error ( EINFO_EIO_ADMIN_CANCELLED )
#define	EINFO_EIO_ADMIN_DATALOSS   __einfo_uniqify ( EINFO_EIO, 0x13, "Data loss" )
#define	EIO_ADMIN_DATALOSS   __einfo_error ( EINFO_EIO_ADMIN_DATALOSS )
#define	EINFO_EIO_ADMIN_DEADLINE   __einfo_uniqify ( EINFO_EIO, 0x14, "Deadline exceeded" )
#define	EIO_ADMIN_DEADLINE   __einfo_error ( EINFO_EIO_ADMIN_DEADLINE )
#define	EINFO_EIO_ADMIN_PRECONDITION   __einfo_uniqify ( EINFO_EIO, 0x15, "Failed precondition" )
#define	EIO_ADMIN_PRECONDITION   __einfo_error ( EINFO_EIO_ADMIN_PRECONDITION )
#define	EINFO_EIO_ADMIN_INTERNAL   __einfo_uniqify ( EINFO_EIO, 0x16, "Internal error" )
#define	EIO_ADMIN_INTERNAL   __einfo_error ( EINFO_EIO_ADMIN_INTERNAL )
#define	EINFO_EIO_ADMIN_INVAL   __einfo_uniqify ( EINFO_EIO, 0x17, "Invalid argument" )
#define	EIO_ADMIN_INVAL   __einfo_error ( EINFO_EIO_ADMIN_INVAL )
#define	EINFO_EIO_ADMIN_NOT_FOUND   __einfo_uniqify ( EINFO_EIO, 0x18, "Not found" )
#define	EIO_ADMIN_NOT_FOUND   __einfo_error ( EINFO_EIO_ADMIN_NOT_FOUND )
#define	EINFO_EIO_ADMIN_RANGE   __einfo_uniqify ( EINFO_EIO, 0x19, "Out of range" )
#define	EIO_ADMIN_RANGE   __einfo_error ( EINFO_EIO_ADMIN_RANGE )
#define	EINFO_EIO_ADMIN_PERM   __einfo_uniqify ( EINFO_EIO, 0x1a, "Permission denied" )
#define	EIO_ADMIN_PERM   __einfo_error ( EINFO_EIO_ADMIN_PERM )
#define	EINFO_EIO_ADMIN_UNAUTH   __einfo_uniqify ( EINFO_EIO, 0x1b, "Unauthenticated" )
#define	EIO_ADMIN_UNAUTH   __einfo_error ( EINFO_EIO_ADMIN_UNAUTH )
#define	EINFO_EIO_ADMIN_RESOURCE   __einfo_uniqify ( EINFO_EIO, 0x1c, "Resource exhausted" )
#define	EIO_ADMIN_RESOURCE   __einfo_error ( EINFO_EIO_ADMIN_RESOURCE )
#define	EINFO_EIO_ADMIN_UNAVAIL   __einfo_uniqify ( EINFO_EIO, 0x1d, "Unavailable" )
#define	EIO_ADMIN_UNAVAIL   __einfo_error ( EINFO_EIO_ADMIN_UNAVAIL )
#define	EINFO_EIO_ADMIN_NOTSUP   __einfo_uniqify ( EINFO_EIO, 0x1e, "Unimplemented" )
#define	EIO_ADMIN_NOTSUP   __einfo_error ( EINFO_EIO_ADMIN_NOTSUP )
#define	EINFO_EIO_ADMIN_UNKNOWN   __einfo_uniqify ( EINFO_EIO, 0x1f, "Unknown error" )
#define	EIO_ADMIN_UNKNOWN   __einfo_error ( EINFO_EIO_ADMIN_UNKNOWN )
#define	EIO_ADMIN(status)

Functions
	FILE_LICENCE (GPL2_OR_LATER_OR_UBDL)
static size_t	gve_offset (struct gve_queue *queue, unsigned int tag)
	Get buffer offset (within queue page list allocation) More...
static physaddr_t	gve_address (struct gve_queue *queue, unsigned int tag)
	Get buffer address (within queue page list address space) More...
static void *	gve_buffer (struct gve_queue *queue, unsigned int tag)
	Get buffer address. More...
static int	gve_reset (struct gve_nic *gve)
	Reset hardware. More...
static const char *	gve_mode_name (unsigned int mode)
	Get operating mode name (for debugging) More...
static int	gve_admin_alloc (struct gve_nic *gve)
	Allocate admin queue. More...
static void	gve_admin_free (struct gve_nic *gve)
	Free admin queue. More...
static void	gve_admin_enable (struct gve_nic *gve)
	Enable admin queue. More...
static union gve_admin_command *	gve_admin_command (struct gve_nic *gve)
	Get next available admin queue command slot. More...
static int	gve_admin_wait (struct gve_nic *gve)
	Wait for admin queue command to complete. More...
static int	gve_admin (struct gve_nic *gve)
	Issue admin queue command. More...
static int	gve_admin_simple (struct gve_nic *gve, unsigned int opcode, unsigned int id)
	Issue simple admin queue command. More...
static int	gve_describe (struct gve_nic *gve)
	Get device descriptor. More...
static int	gve_configure (struct gve_nic *gve)
	Configure device resources. More...
static int	gve_deconfigure (struct gve_nic *gve)
	Deconfigure device resources. More...
static int	gve_register (struct gve_nic *gve, struct gve_qpl *qpl)
	Register queue page list. More...
static int	gve_unregister (struct gve_nic *gve, struct gve_qpl *qpl)
	Unregister page list. More...
static void	gve_create_tx_param (struct gve_queue *queue, uint32_t qpl, union gve_admin_command *cmd)
	Construct command to create transmit queue. More...
static void	gve_create_rx_param (struct gve_queue *queue, uint32_t qpl, union gve_admin_command *cmd)
	Construct command to create receive queue. More...
static int	gve_create_queue (struct gve_nic *gve, struct gve_queue *queue)
	Create transmit or receive queue. More...
static int	gve_destroy_queue (struct gve_nic *gve, struct gve_queue *queue)
	Destroy transmit or receive queue. More...
static int	gve_alloc_shared (struct gve_nic *gve)
	Allocate shared queue resources. More...
static void	gve_free_shared (struct gve_nic *gve)
	Free shared queue resources. More...
static int	gve_alloc_qpl (struct gve_nic *gve, struct gve_qpl *qpl, uint32_t id, unsigned int buffers)
	Allocate queue page list. More...
static void	gve_free_qpl (struct gve_nic *nic __unused, struct gve_qpl *qpl)
	Free queue page list. More...
static unsigned int	gve_next (unsigned int seq)
	Calculate next receive sequence number. More...
static int	gve_alloc_queue (struct gve_nic *gve, struct gve_queue *queue)
	Allocate descriptor queue. More...
static void	gve_free_queue (struct gve_nic *gve, struct gve_queue *queue)
	Free descriptor queue. More...
static void	gve_cancel_tx (struct gve_nic *gve)
	Cancel any pending transmissions. More...
static int	gve_start (struct gve_nic *gve)
	Start up device. More...
static void	gve_stop (struct gve_nic *gve)
	Stop device. More...
static void	gve_startup (struct gve_nic *gve)
	Device startup process. More...
static void	gve_restart (struct gve_nic *gve)
	Trigger startup process. More...
static void	gve_watchdog (struct retry_timer *timer, int over __unused)
	Reset recovery watchdog. More...
static int	gve_open (struct net_device *netdev)
	Open network device. More...
static void	gve_close (struct net_device *netdev)
	Close network device. More...
static int	gve_transmit (struct net_device *netdev, struct io_buffer *iobuf)
	Transmit packet. More...
static void	gve_poll_tx (struct net_device *netdev)
	Poll for completed transmissions. More...
static void	gve_poll_rx (struct net_device *netdev)
	Poll for received packets. More...
static void	gve_refill_rx (struct net_device *netdev)
	Refill receive queue. More...
static void	gve_poll (struct net_device *netdev)
	Poll for completed and received packets. More...
static int	gve_setup (struct gve_nic *gve)
	Set up admin queue and get device description. More...
static int	gve_probe (struct pci_device *pci)
	Probe PCI device. More...
static void	gve_remove (struct pci_device *pci)
	Remove PCI device. More...

Variables
static struct net_device_operations	gve_operations
	GVE network device operations. More...
static const struct gve_queue_type	gve_tx_type
	Transmit descriptor queue type. More...
static const struct gve_queue_type	gve_rx_type
	Receive descriptor queue type. More...
static struct process_descriptor	gve_startup_desc
	Device startup process descriptor. More...
static struct pci_device_id	gve_nics []
	GVE PCI device IDs. More...
struct pci_driver gve_driver	__pci_driver
	GVE PCI driver. More...

Detailed Description

Google Virtual Ethernet network driver.

Definition in file gve.c.

Macro Definition Documentation

EINFO_EIO_ADMIN_UNSET

#define EINFO_EIO_ADMIN_UNSET   __einfo_uniqify ( EINFO_EIO, 0x00, "Uncompleted" )

Definition at line 49 of file gve.c.

EIO_ADMIN_UNSET

#define EIO_ADMIN_UNSET   __einfo_error ( EINFO_EIO_ADMIN_UNSET )

Definition at line 51 of file gve.c.

EINFO_EIO_ADMIN_ABORTED

#define EINFO_EIO_ADMIN_ABORTED   __einfo_uniqify ( EINFO_EIO, 0x10, "Aborted" )

Definition at line 53 of file gve.c.

EIO_ADMIN_ABORTED

#define EIO_ADMIN_ABORTED   __einfo_error ( EINFO_EIO_ADMIN_ABORTED )

Definition at line 55 of file gve.c.

EINFO_EIO_ADMIN_EXISTS

#define EINFO_EIO_ADMIN_EXISTS   __einfo_uniqify ( EINFO_EIO, 0x11, "Already exists" )

Definition at line 57 of file gve.c.

EIO_ADMIN_EXISTS

#define EIO_ADMIN_EXISTS   __einfo_error ( EINFO_EIO_ADMIN_EXISTS )

Definition at line 59 of file gve.c.

EINFO_EIO_ADMIN_CANCELLED

#define EINFO_EIO_ADMIN_CANCELLED   __einfo_uniqify ( EINFO_EIO, 0x12, "Cancelled" )

Definition at line 61 of file gve.c.

EIO_ADMIN_CANCELLED

#define EIO_ADMIN_CANCELLED   __einfo_error ( EINFO_EIO_ADMIN_CANCELLED )

Definition at line 63 of file gve.c.

EINFO_EIO_ADMIN_DATALOSS

#define EINFO_EIO_ADMIN_DATALOSS   __einfo_uniqify ( EINFO_EIO, 0x13, "Data loss" )

Definition at line 65 of file gve.c.

EIO_ADMIN_DATALOSS

#define EIO_ADMIN_DATALOSS   __einfo_error ( EINFO_EIO_ADMIN_DATALOSS )

Definition at line 67 of file gve.c.

EINFO_EIO_ADMIN_DEADLINE

#define EINFO_EIO_ADMIN_DEADLINE   __einfo_uniqify ( EINFO_EIO, 0x14, "Deadline exceeded" )

Definition at line 69 of file gve.c.

EIO_ADMIN_DEADLINE

#define EIO_ADMIN_DEADLINE   __einfo_error ( EINFO_EIO_ADMIN_DEADLINE )

Definition at line 71 of file gve.c.

EINFO_EIO_ADMIN_PRECONDITION

#define EINFO_EIO_ADMIN_PRECONDITION   __einfo_uniqify ( EINFO_EIO, 0x15, "Failed precondition" )

Definition at line 73 of file gve.c.

EIO_ADMIN_PRECONDITION

#define EIO_ADMIN_PRECONDITION   __einfo_error ( EINFO_EIO_ADMIN_PRECONDITION )

Definition at line 75 of file gve.c.

EINFO_EIO_ADMIN_INTERNAL

#define EINFO_EIO_ADMIN_INTERNAL   __einfo_uniqify ( EINFO_EIO, 0x16, "Internal error" )

Definition at line 77 of file gve.c.

EIO_ADMIN_INTERNAL

#define EIO_ADMIN_INTERNAL   __einfo_error ( EINFO_EIO_ADMIN_INTERNAL )

Definition at line 79 of file gve.c.

EINFO_EIO_ADMIN_INVAL

#define EINFO_EIO_ADMIN_INVAL   __einfo_uniqify ( EINFO_EIO, 0x17, "Invalid argument" )

Definition at line 81 of file gve.c.

EIO_ADMIN_INVAL

#define EIO_ADMIN_INVAL   __einfo_error ( EINFO_EIO_ADMIN_INVAL )

Definition at line 83 of file gve.c.

EINFO_EIO_ADMIN_NOT_FOUND

#define EINFO_EIO_ADMIN_NOT_FOUND   __einfo_uniqify ( EINFO_EIO, 0x18, "Not found" )

Definition at line 85 of file gve.c.

EIO_ADMIN_NOT_FOUND

#define EIO_ADMIN_NOT_FOUND   __einfo_error ( EINFO_EIO_ADMIN_NOT_FOUND )

Definition at line 87 of file gve.c.

EINFO_EIO_ADMIN_RANGE

#define EINFO_EIO_ADMIN_RANGE   __einfo_uniqify ( EINFO_EIO, 0x19, "Out of range" )

Definition at line 89 of file gve.c.

EIO_ADMIN_RANGE

#define EIO_ADMIN_RANGE   __einfo_error ( EINFO_EIO_ADMIN_RANGE )

Definition at line 91 of file gve.c.

EINFO_EIO_ADMIN_PERM

#define EINFO_EIO_ADMIN_PERM   __einfo_uniqify ( EINFO_EIO, 0x1a, "Permission denied" )

Definition at line 93 of file gve.c.

EIO_ADMIN_PERM

#define EIO_ADMIN_PERM   __einfo_error ( EINFO_EIO_ADMIN_PERM )

Definition at line 95 of file gve.c.

EINFO_EIO_ADMIN_UNAUTH

#define EINFO_EIO_ADMIN_UNAUTH   __einfo_uniqify ( EINFO_EIO, 0x1b, "Unauthenticated" )

Definition at line 97 of file gve.c.

EIO_ADMIN_UNAUTH

#define EIO_ADMIN_UNAUTH   __einfo_error ( EINFO_EIO_ADMIN_UNAUTH )

Definition at line 99 of file gve.c.

EINFO_EIO_ADMIN_RESOURCE

#define EINFO_EIO_ADMIN_RESOURCE   __einfo_uniqify ( EINFO_EIO, 0x1c, "Resource exhausted" )

Definition at line 101 of file gve.c.

EIO_ADMIN_RESOURCE

#define EIO_ADMIN_RESOURCE   __einfo_error ( EINFO_EIO_ADMIN_RESOURCE )

Definition at line 103 of file gve.c.

EINFO_EIO_ADMIN_UNAVAIL

#define EINFO_EIO_ADMIN_UNAVAIL   __einfo_uniqify ( EINFO_EIO, 0x1d, "Unavailable" )

Definition at line 105 of file gve.c.

EIO_ADMIN_UNAVAIL

#define EIO_ADMIN_UNAVAIL   __einfo_error ( EINFO_EIO_ADMIN_UNAVAIL )

Definition at line 107 of file gve.c.

EINFO_EIO_ADMIN_NOTSUP

#define EINFO_EIO_ADMIN_NOTSUP   __einfo_uniqify ( EINFO_EIO, 0x1e, "Unimplemented" )

Definition at line 109 of file gve.c.

EIO_ADMIN_NOTSUP

#define EIO_ADMIN_NOTSUP   __einfo_error ( EINFO_EIO_ADMIN_NOTSUP )

Definition at line 111 of file gve.c.

EINFO_EIO_ADMIN_UNKNOWN

#define EINFO_EIO_ADMIN_UNKNOWN   __einfo_uniqify ( EINFO_EIO, 0x1f, "Unknown error" )

Definition at line 113 of file gve.c.

EIO_ADMIN_UNKNOWN

#define EIO_ADMIN_UNKNOWN   __einfo_error ( EINFO_EIO_ADMIN_UNKNOWN )

Definition at line 115 of file gve.c.

EIO_ADMIN

#define EIO_ADMIN

(

status

)

Value:

EUNIQ ( EINFO_EIO, ( (status) & 0x1f ),                         \
                EIO_ADMIN_UNSET, EIO_ADMIN_ABORTED, EIO_ADMIN_EXISTS,   \
                EIO_ADMIN_CANCELLED, EIO_ADMIN_DATALOSS,                \
                EIO_ADMIN_DEADLINE, EIO_ADMIN_PRECONDITION,             \
                EIO_ADMIN_INTERNAL, EIO_ADMIN_INVAL,                    \
                EIO_ADMIN_NOT_FOUND, EIO_ADMIN_RANGE, EIO_ADMIN_PERM,   \
                EIO_ADMIN_UNAUTH, EIO_ADMIN_RESOURCE,                   \
                EIO_ADMIN_UNAVAIL, EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN )

Definition at line 117 of file gve.c.

Function Documentation

FILE_LICENCE()

FILE_LICENCE

(

GPL2_OR_LATER_OR_UBDL

)

gve_offset()

static size_t gve_offset ( struct gve_queue *  queue, unsigned int  tag  )

inlinestatic

Get buffer offset (within queue page list allocation)

Parameters

queue	Descriptor queue
tag	Buffer tag

Return values

addr	Buffer address within queue page list address space

Definition at line 142 of file gve.c.

                                                         {

        /* We allocate sufficient pages for the maximum fill level of
         * buffers, and reuse the buffers in strict rotation as they
         * are released by the hardware.
         */
        assert ( tag < queue->fill );
        return ( tag * GVE_BUF_SIZE );
}

References assert(), fill, GVE_BUF_SIZE, and tag.

Referenced by gve_address(), gve_buffer(), and gve_poll_rx().

gve_address()

static physaddr_t gve_address ( struct gve_queue *  queue, unsigned int  tag  )

inlinestatic

Get buffer address (within queue page list address space)

Parameters

queue	Descriptor queue
tag	Buffer tag

Return values

addr	Buffer address within queue page list address space

Definition at line 160 of file gve.c.

                                                          {

        /* Pages are allocated as a single contiguous block */
        return ( queue->qpl.base + gve_offset ( queue, tag ) );
}

References gve_offset(), queue, and tag.

Referenced by gve_create_queue(), gve_refill_rx(), and gve_transmit().

gve_buffer()

static void* gve_buffer ( struct gve_queue *  queue, unsigned int  tag  )

inlinestatic

Get buffer address.

Parameters

queue	Descriptor queue
tag	Buffer tag

Return values

addr	Buffer address

Definition at line 174 of file gve.c.

                                                         {

        /* Pages are allocated as a single contiguous block */
        return ( queue->qpl.data + gve_offset ( queue, tag ) );
}

References gve_offset(), queue, and tag.

gve_reset()

static int gve_reset ( struct gve_nic *  gve )

static

Reset hardware.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 193 of file gve.c.

                                             {
        uint32_t pfn;
        unsigned int i;

        /* Skip reset if admin queue page frame number is already
         * clear.  Triggering a reset on an already-reset device seems
         * to cause a delayed reset to be scheduled.  This can cause
         * the device to end up in a reset loop, where each attempt to
         * recover from reset triggers another reset a few seconds
         * later.
         */
        pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
        if ( ! pfn ) {
                DBGC ( gve, "GVE %p skipping reset\n", gve );
                return 0;
        }

        /* Clear admin queue page frame number */
        writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN );
        wmb();

        /* Wait for device to reset */
        for ( i = 0 ; i < GVE_RESET_MAX_WAIT_MS ; i++ ) {

                /* Delay */
                mdelay ( 1 );

                /* Check for reset completion */
                pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
                if ( ! pfn )
                        return 0;
        }

        DBGC ( gve, "GVE %p reset timed out (PFN %#08x devstat %#08x)\n",
               gve, bswap_32 ( pfn ),
               bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) );
        return -ETIMEDOUT;
}

References bswap_32, gve_nic::cfg, DBGC, ETIMEDOUT, GVE_CFG_ADMIN_PFN, GVE_CFG_DEVSTAT, GVE_RESET_MAX_WAIT_MS, mdelay(), readl(), wmb(), and writel().

Referenced by gve_close(), gve_probe(), gve_remove(), gve_setup(), and gve_startup().

gve_mode_name()

static const char* gve_mode_name ( unsigned int  mode )

inlinestatic

Get operating mode name (for debugging)

Parameters

mode	Operating mode

Return values

name

Mode name

Definition at line 245 of file gve.c.

                                                               {
        static char buf[ 8 /* "XXX-XXX" + NUL */ ];

        snprintf ( buf, sizeof ( buf ), "%s-%s",
                   ( ( mode & GVE_MODE_DQO ) ? "DQO" : "GQI" ),
                   ( ( mode & GVE_MODE_QPL ) ? "QPL" : "RDA" ) );
        return buf;
}

References GVE_MODE_DQO, GVE_MODE_QPL, mode, and snprintf().

Referenced by gve_describe().

gve_admin_alloc()

static int gve_admin_alloc ( struct gve_nic *  gve )

static

Allocate admin queue.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 260 of file gve.c.

                                                   {
        struct dma_device *dma = gve->dma;
        struct gve_admin *admin = &gve->admin;
        struct gve_scratch *scratch = &gve->scratch;
        size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
        size_t scratch_len = sizeof ( *scratch->buf );
        int rc;

        /* Allocate admin queue */
        admin->cmd = dma_alloc ( dma, &admin->map, admin_len, GVE_ALIGN );
        if ( ! admin->cmd ) {
                rc = -ENOMEM;
                goto err_admin;
        }

        /* Allocate scratch buffer */
        scratch->buf = dma_alloc ( dma, &scratch->map, scratch_len, GVE_ALIGN );
        if ( ! scratch->buf ) {
                rc = -ENOMEM;
                goto err_scratch;
        }

        DBGC ( gve, "GVE %p AQ at [%08lx,%08lx) scratch [%08lx,%08lx)\n",
               gve, virt_to_phys ( admin->cmd ),
               ( virt_to_phys ( admin->cmd ) + admin_len ),
               virt_to_phys ( scratch->buf ),
               ( virt_to_phys ( scratch->buf ) + scratch_len ) );
        return 0;

        dma_free ( &scratch->map, scratch->buf, scratch_len );
 err_scratch:
        dma_free ( &admin->map, admin->cmd, admin_len );
 err_admin:
        return rc;
}

References gve_nic::admin, gve_scratch::buf, gve_admin::cmd, DBGC, dma(), gve_nic::dma, dma_alloc(), dma_free(), ENOMEM, GVE_ADMIN_COUNT, GVE_ALIGN, gve_admin::map, gve_scratch::map, rc, and gve_nic::scratch.

Referenced by gve_probe().

gve_admin_free()

static void gve_admin_free ( struct gve_nic *  gve )

static

Free admin queue.

Parameters

gve	GVE device

Definition at line 301 of file gve.c.

                                                   {
        struct gve_admin *admin = &gve->admin;
        struct gve_scratch *scratch = &gve->scratch;
        size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
        size_t scratch_len = sizeof ( *scratch->buf );

        /* Free scratch buffer */
        dma_free ( &scratch->map, scratch->buf, scratch_len );

        /* Free admin queue */
        dma_free ( &admin->map, admin->cmd, admin_len );
}

References gve_nic::admin, gve_scratch::buf, gve_admin::cmd, dma_free(), GVE_ADMIN_COUNT, gve_admin::map, gve_scratch::map, and gve_nic::scratch.

Referenced by gve_probe(), and gve_remove().

gve_admin_enable()

static void gve_admin_enable ( struct gve_nic *  gve )

static

Enable admin queue.

Parameters

gve	GVE device

Definition at line 319 of file gve.c.

                                                     {
        struct gve_admin *admin = &gve->admin;
        size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
        physaddr_t base;

        /* Reset queue */
        admin->prod = 0;

        /* Program queue addresses and capabilities */
        base = dma ( &admin->map, admin->cmd );
        writel ( bswap_32 ( base / GVE_PAGE_SIZE ),
                 gve->cfg + GVE_CFG_ADMIN_PFN );
        writel ( bswap_32 ( base & 0xffffffffUL ),
                 gve->cfg + GVE_CFG_ADMIN_BASE_LO );
        if ( sizeof ( base ) > sizeof ( uint32_t ) ) {
                writel ( bswap_32 ( ( ( uint64_t ) base ) >> 32 ),
                         gve->cfg + GVE_CFG_ADMIN_BASE_HI );
        } else {
                writel ( 0, gve->cfg + GVE_CFG_ADMIN_BASE_HI );
        }
        writel ( bswap_16 ( admin_len ), gve->cfg + GVE_CFG_ADMIN_LEN );
        writel ( bswap_32 ( GVE_CFG_DRVSTAT_RUN ), gve->cfg + GVE_CFG_DRVSTAT );
}

References gve_nic::admin, base, bswap_16, bswap_32, gve_nic::cfg, gve_admin::cmd, dma(), GVE_ADMIN_COUNT, GVE_CFG_ADMIN_BASE_HI, GVE_CFG_ADMIN_BASE_LO, GVE_CFG_ADMIN_LEN, GVE_CFG_ADMIN_PFN, GVE_CFG_DRVSTAT, GVE_CFG_DRVSTAT_RUN, GVE_PAGE_SIZE, gve_admin::map, gve_admin::prod, and writel().

Referenced by gve_setup(), and gve_startup().

gve_admin_command()

static union gve_admin_command* gve_admin_command ( struct gve_nic *  gve )

static

Get next available admin queue command slot.

Parameters

gve	GVE device

Return values

cmd	Admin queue command

Definition at line 349 of file gve.c.

                                                                           {
        struct gve_admin *admin = &gve->admin;
        union gve_admin_command *cmd;
        unsigned int index;

        /* Get next command slot */
        index = admin->prod;
        cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ];

        /* Initialise request */
        memset ( cmd, 0, sizeof ( *cmd ) );

        return cmd;
}

References gve_nic::admin, cmd, gve_admin::cmd, GVE_ADMIN_COUNT, index, memset(), and gve_admin::prod.

Referenced by gve_admin_simple(), gve_configure(), gve_create_queue(), gve_describe(), and gve_register().

gve_admin_wait()

static int gve_admin_wait ( struct gve_nic *  gve )

static

Wait for admin queue command to complete.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 370 of file gve.c.

                                                  {
        struct gve_admin *admin = &gve->admin;
        uint32_t evt;
        uint32_t pfn;
        unsigned int i;

        /* Wait for any outstanding commands to complete */
        for ( i = 0 ; i < GVE_ADMIN_MAX_WAIT_MS ; i++ ) {

                /* Check event counter */
                rmb();
                evt = bswap_32 ( readl ( gve->cfg + GVE_CFG_ADMIN_EVT ) );
                if ( evt == admin->prod )
                        return 0;

                /* Check for device reset */
                pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
                if ( ! pfn )
                        break;

                /* Delay */
                mdelay ( 1 );
        }

        DBGC ( gve, "GVE %p AQ %#02x %s (completed %#02x, status %#08x)\n",
               gve, admin->prod, ( pfn ? "timed out" : "saw reset" ), evt,
               bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) );
        return ( pfn ? -ETIMEDOUT : -ECONNRESET );
}

References gve_nic::admin, bswap_32, gve_nic::cfg, DBGC, ECONNRESET, ETIMEDOUT, GVE_ADMIN_MAX_WAIT_MS, GVE_CFG_ADMIN_EVT, GVE_CFG_ADMIN_PFN, GVE_CFG_DEVSTAT, mdelay(), gve_admin::prod, readl(), and rmb.

Referenced by gve_admin().

gve_admin()

static int gve_admin ( struct gve_nic *  gve )

static

Issue admin queue command.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 406 of file gve.c.

                                             {
        struct gve_admin *admin = &gve->admin;
        union gve_admin_command *cmd;
        unsigned int index;
        uint32_t opcode;
        uint32_t status;
        int rc;

        /* Ensure admin queue is idle */
        if ( ( rc = gve_admin_wait ( gve ) ) != 0 )
                return rc;

        /* Get next command slot */
        index = admin->prod;
        cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ];
        opcode = cmd->hdr.opcode;
        DBGC2 ( gve, "GVE %p AQ %#02x command %#04x request:\n",
                gve, index, opcode );
        DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) );

        /* Increment producer counter */
        admin->prod++;

        /* Ring doorbell */
        wmb();
        writel ( bswap_32 ( admin->prod ), gve->cfg + GVE_CFG_ADMIN_DB );

        /* Wait for command to complete */
        if ( ( rc = gve_admin_wait ( gve ) ) != 0 )
                return rc;

        /* Check command status */
        status = be32_to_cpu ( cmd->hdr.status );
        if ( status != GVE_ADMIN_STATUS_OK ) {
                rc = -EIO_ADMIN ( status );
                DBGC ( gve, "GVE %p AQ %#02x command %#04x failed: %#08x\n",
                       gve, index, opcode, status );
                DBGC_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
                DBGC ( gve, "GVE %p AQ error: %s\n", gve, strerror ( rc ) );
                return rc;
        }

        DBGC2 ( gve, "GVE %p AQ %#02x command %#04x result:\n",
                gve, index, opcode );
        DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
        return 0;
}

References gve_nic::admin, be32_to_cpu, bswap_32, gve_nic::cfg, cmd, gve_admin::cmd, DBGC, DBGC2, DBGC2_HDA, DBGC_HDA, EIO_ADMIN, GVE_ADMIN_COUNT, GVE_ADMIN_STATUS_OK, gve_admin_wait(), GVE_CFG_ADMIN_DB, index, opcode, gve_admin::prod, rc, status, strerror(), wmb(), and writel().

gve_admin_simple()

static int gve_admin_simple ( struct gve_nic *  gve, unsigned int  opcode, unsigned int  id  )

static

Issue simple admin queue command.

Parameters

gve	GVE device
opcode	Operation code
id	ID parameter (or zero if not applicable)

Return values

rc	Return status code

Several admin queue commands take either an empty parameter list or a single 32-bit ID parameter.

Definition at line 465 of file gve.c.

                                                {
        union gve_admin_command *cmd;
        int rc;

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = opcode;
        cmd->simple.id = cpu_to_be32 ( id );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        return 0;
}

References cmd, cpu_to_be32, gve_admin_command(), opcode, and rc.

gve_describe()

static int gve_describe ( struct gve_nic *  gve )

static

Get device descriptor.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 488 of file gve.c.

                                                {
        struct net_device *netdev = gve->netdev;
        struct gve_device_descriptor *desc = &gve->scratch.buf->desc;
        union gve_admin_command *cmd;
        struct gve_option *opt;
        unsigned int count;
        unsigned int id;
        size_t offset;
        size_t max;
        size_t len;
        int rc;

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = GVE_ADMIN_DESCRIBE;
        cmd->desc.addr = cpu_to_be64 ( dma ( &gve->scratch.map, desc ) );
        cmd->desc.ver = cpu_to_be32 ( GVE_ADMIN_DESCRIBE_VER );
        cmd->desc.len = cpu_to_be32 ( sizeof ( *desc ) );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;
        DBGC2 ( gve, "GVE %p device descriptor:\n", gve );
        DBGC2_HDA ( gve, 0, desc, sizeof ( *desc ) );

        /* Extract queue parameters */
        gve->events.count = be16_to_cpu ( desc->counters );
        gve->tx.count = be16_to_cpu ( desc->tx_count );
        gve->rx.count = be16_to_cpu ( desc->rx_count );
        DBGC ( gve, "GVE %p using %d TX, %d RX, %d events\n",
               gve, gve->tx.count, gve->rx.count, gve->events.count );

        /* Extract network parameters */
        build_assert ( sizeof ( desc->mac ) == ETH_ALEN );
        memcpy ( netdev->hw_addr, &desc->mac, sizeof ( desc->mac ) );
        netdev->mtu = be16_to_cpu ( desc->mtu );
        netdev->max_pkt_len = ( netdev->mtu + ETH_HLEN );
        DBGC ( gve, "GVE %p MAC %s (\"%s\") MTU %zd\n",
               gve, eth_ntoa ( netdev->hw_addr ),
               inet_ntoa ( desc->mac.in ), netdev->mtu );

        /* Parse options */
        count = be16_to_cpu ( desc->opt_count );
        max = be16_to_cpu ( desc->len );
        gve->options = 0;
        for ( offset = offsetof ( typeof ( *desc ), opts ) ; count ;
              count--, offset += len ) {

                /* Check space for option header */
                if ( ( offset + sizeof ( *opt ) ) > max ) {
                        DBGC ( gve, "GVE %p underlength option at +%#02zx:\n",
                               gve, offset );
                        DBGC_HDA ( gve, 0, desc, sizeof ( *desc ) );
                        return -EINVAL;
                }
                opt = ( ( ( void * ) desc ) + offset );

                /* Check space for option body */
                len = ( sizeof ( *opt ) + be16_to_cpu ( opt->len ) );
                if ( ( offset + len ) > max ) {
                        DBGC ( gve, "GVE %p malformed option at +%#02zx:\n",
                               gve, offset );
                        DBGC_HDA ( gve, 0, desc, sizeof ( *desc ) );
                        return -EINVAL;
                }

                /* Record option as supported */
                id = be16_to_cpu ( opt->id );
                if ( id < ( 8 * sizeof ( gve->options ) ) )
                        gve->options |= ( 1 << id );
        }
        DBGC ( gve, "GVE %p supports options %#08x\n", gve, gve->options );

        /* Select preferred operating mode */
        if ( gve->options & ( 1 << GVE_OPT_GQI_QPL ) ) {
                /* GQI-QPL: in-order queues, queue page list addressing */
                gve->mode = GVE_MODE_QPL;
        } else if ( gve->options & ( 1 << GVE_OPT_GQI_RDA ) ) {
                /* GQI-RDA: in-order queues, raw DMA addressing */
                gve->mode = 0;
        } else if ( gve->options & ( 1 << GVE_OPT_DQO_QPL ) ) {
                /* DQO-QPL: out-of-order queues, queue page list addressing */
                gve->mode = ( GVE_MODE_DQO | GVE_MODE_QPL );
        } else if ( gve->options & ( 1 << GVE_OPT_DQO_RDA ) ) {
                /* DQO-RDA: out-of-order queues, raw DMA addressing */
                gve->mode = GVE_MODE_DQO;
        } else {
                /* No options matched: assume the original GQI-QPL mode */
                gve->mode = GVE_MODE_QPL;
        }
        DBGC ( gve, "GVE %p using %s mode\n",
               gve, gve_mode_name ( gve->mode ) );

        return 0;
}

References be16_to_cpu, gve_scratch::buf, build_assert, cmd, count, gve_events::count, gve_queue::count, cpu_to_be32, cpu_to_be64, DBGC, DBGC2, DBGC2_HDA, DBGC_HDA, desc, gve_scratch::desc, dma(), EINVAL, ETH_ALEN, ETH_HLEN, eth_ntoa(), gve_nic::events, gve_admin_command(), GVE_ADMIN_DESCRIBE, GVE_ADMIN_DESCRIBE_VER, GVE_MODE_DQO, gve_mode_name(), GVE_MODE_QPL, GVE_OPT_DQO_QPL, GVE_OPT_DQO_RDA, GVE_OPT_GQI_QPL, GVE_OPT_GQI_RDA, net_device::hw_addr, id, gve_option::id, inet_ntoa(), gve_option::len, len, gve_scratch::map, max, net_device::max_pkt_len, memcpy(), gve_nic::mode, net_device::mtu, netdev, gve_nic::netdev, offset, offsetof, gve_nic::options, opts, rc, gve_nic::rx, gve_nic::scratch, gve_nic::tx, and typeof().

Referenced by gve_setup().

gve_configure()

static int gve_configure ( struct gve_nic *  gve )

static

Configure device resources.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 590 of file gve.c.

                                                 {
        struct gve_events *events = &gve->events;
        struct gve_irqs *irqs = &gve->irqs;
        union gve_admin_command *cmd;
        uint32_t doorbell;
        unsigned int db_off;
        unsigned int i;
        int rc;

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = GVE_ADMIN_CONFIGURE;
        cmd->conf.events =
                cpu_to_be64 ( dma ( &events->map, events->event ) );
        cmd->conf.irqs =
                cpu_to_be64 ( dma ( &irqs->map, irqs->irq ) );
        cmd->conf.num_events = cpu_to_be32 ( events->count );
        cmd->conf.num_irqs = cpu_to_be32 ( GVE_IRQ_COUNT );
        cmd->conf.irq_stride = cpu_to_be32 ( sizeof ( irqs->irq[0] ) );
        cmd->conf.format = GVE_FORMAT ( gve->mode );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        /* Disable all interrupts */
        doorbell = ( ( gve->mode & GVE_MODE_DQO ) ?
                     0 : bswap_32 ( GVE_GQI_IRQ_DISABLE ) );
        for ( i = 0 ; i < GVE_IRQ_COUNT ; i++ ) {
                db_off = ( be32_to_cpu ( irqs->irq[i].db_idx ) *
                           sizeof ( uint32_t ) );
                DBGC ( gve, "GVE %p IRQ %d doorbell +%#04x\n", gve, i, db_off );
                irqs->db[i] = ( gve->db + db_off );
                writel ( doorbell, irqs->db[i] );
        }

        return 0;
}

References be32_to_cpu, bswap_32, cmd, gve_events::count, cpu_to_be32, cpu_to_be64, gve_irqs::db, gve_nic::db, gve_irq::db_idx, DBGC, dma(), doorbell, gve_events::event, gve_nic::events, gve_admin_command(), GVE_ADMIN_CONFIGURE, GVE_FORMAT, GVE_GQI_IRQ_DISABLE, GVE_IRQ_COUNT, GVE_MODE_DQO, gve_irqs::irq, gve_nic::irqs, gve_events::map, gve_irqs::map, gve_nic::mode, rc, and writel().

Referenced by gve_start().

gve_deconfigure()

static int gve_deconfigure ( struct gve_nic *  gve )

static

Deconfigure device resources.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 635 of file gve.c.

                                                   {
        int rc;

        /* Issue command (with meaningless ID) */
        if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_DECONFIGURE, 0 ) ) != 0 )
                return rc;

        return 0;
}

References GVE_ADMIN_DECONFIGURE, and rc.

Referenced by gve_start(), and gve_stop().

gve_register()

static int gve_register ( struct gve_nic *  gve, struct gve_qpl *  qpl  )

static

Register queue page list.

Parameters

gve	GVE device
qpl	Queue page list

Return values

rc	Return status code

Definition at line 652 of file gve.c.

                                                                     {
        struct gve_pages *pages = &gve->scratch.buf->pages;
        union gve_admin_command *cmd;
        void *addr;
        unsigned int i;
        int rc;

        /* Do nothing if using raw DMA addressing */
        if ( ! ( gve->mode & GVE_MODE_QPL ) )
                return 0;

        /* Build page address list */
        for ( i = 0 ; i < qpl->count ; i++ ) {
                addr = ( qpl->data + ( i * GVE_PAGE_SIZE ) );
                pages->addr[i] = cpu_to_be64 ( dma ( &qpl->map, addr ) );
        }

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = GVE_ADMIN_REGISTER;
        cmd->reg.id = cpu_to_be32 ( qpl->id );
        cmd->reg.count = cpu_to_be32 ( qpl->count );
        cmd->reg.addr = cpu_to_be64 ( dma ( &gve->scratch.map, pages ) );
        cmd->reg.size = cpu_to_be64 ( GVE_PAGE_SIZE );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        return 0;
}

References addr, gve_pages::addr, gve_scratch::buf, cmd, gve_qpl::count, cpu_to_be32, cpu_to_be64, gve_qpl::data, dma(), gve_admin_command(), GVE_ADMIN_REGISTER, GVE_MODE_QPL, GVE_PAGE_SIZE, gve_qpl::id, gve_scratch::map, gve_qpl::map, gve_nic::mode, gve_scratch::pages, rc, and gve_nic::scratch.

Referenced by gve_start().

gve_unregister()

static int gve_unregister ( struct gve_nic *  gve, struct gve_qpl *  qpl  )

static

Unregister page list.

Parameters

gve	GVE device
qpl	Queue page list

Return values

rc	Return status code

Definition at line 691 of file gve.c.

                                                                       {
        int rc;

        /* Do nothing if using raw DMA addressing */
        if ( ! ( gve->mode & GVE_MODE_QPL ) )
                return 0;

        /* Issue command */
        if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_UNREGISTER,
                                       qpl->id ) ) != 0 ) {
                return rc;
        }

        return 0;
}

References GVE_ADMIN_UNREGISTER, GVE_MODE_QPL, gve_qpl::id, gve_nic::mode, and rc.

Referenced by gve_start(), and gve_stop().

gve_create_tx_param()

static void gve_create_tx_param ( struct gve_queue *  queue, uint32_t  qpl, union gve_admin_command *  cmd  )

static

Construct command to create transmit queue.

Parameters

queue	Transmit queue
qpl	Queue page list ID
cmd	Admin queue command

Definition at line 714 of file gve.c.

                                                                 {
        struct gve_admin_create_tx *create = &cmd->create_tx;
        const struct gve_queue_type *type = queue->type;

        /* Construct request parameters */
        create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) );
        create->desc =
                cpu_to_be64 ( dma ( &queue->desc_map, queue->desc.raw ) );
        create->qpl_id = cpu_to_be32 ( qpl );
        create->notify_id = cpu_to_be32 ( type->irq );
        create->desc_count = cpu_to_be16 ( queue->count );
        if ( queue->cmplt.raw ) {
                create->cmplt = cpu_to_be64 ( dma ( &queue->cmplt_map,
                                                    queue->cmplt.raw ) );
                create->cmplt_count = cpu_to_be16 ( queue->count );
        }
}

References cmd, cpu_to_be16, cpu_to_be32, cpu_to_be64, gve_queue_type::create, dma(), gve_queue_type::qpl, queue, and type.

gve_create_rx_param()

static void gve_create_rx_param ( struct gve_queue *  queue, uint32_t  qpl, union gve_admin_command *  cmd  )

static

Construct command to create receive queue.

Parameters

queue	Receive queue
qpl	Queue page list ID
cmd	Admin queue command

Definition at line 740 of file gve.c.

                                                                 {
        struct gve_admin_create_rx *create = &cmd->create_rx;
        const struct gve_queue_type *type = queue->type;

        /* Construct request parameters */
        create->notify_id = cpu_to_be32 ( type->irq );
        create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) );
        create->desc =
                cpu_to_be64 ( dma ( &queue->desc_map, queue->desc.raw ) );
        create->cmplt =
                cpu_to_be64 ( dma ( &queue->cmplt_map, queue->cmplt.raw ) );
        create->qpl_id = cpu_to_be32 ( qpl );
        create->desc_count = cpu_to_be16 ( queue->count );
        create->bufsz = cpu_to_be16 ( GVE_BUF_SIZE );
        create->cmplt_count = cpu_to_be16 ( queue->count );
}

References cmd, cpu_to_be16, cpu_to_be32, cpu_to_be64, gve_queue_type::create, dma(), GVE_BUF_SIZE, gve_queue_type::qpl, queue, and type.

gve_create_queue()

static int gve_create_queue ( struct gve_nic *  gve, struct gve_queue *  queue  )

static

Create transmit or receive queue.

Parameters

gve	GVE device
queue	Descriptor queue

Return values

rc	Return status code

Definition at line 765 of file gve.c.

                                                                             {
        const struct gve_queue_type *type = queue->type;
        const struct gve_queue_stride *stride = &queue->stride;
        union gve_admin_command *cmd;
        struct gve_buffer *buf;
        unsigned int db_off;
        unsigned int evt_idx;
        unsigned int tag;
        unsigned int i;
        uint32_t qpl;
        int rc;

        /* Reset queue */
        queue->prod = 0;
        queue->cons = 0;
        queue->done = 0;
        memset ( queue->desc.raw, 0, ( queue->count * stride->desc ) );
        memset ( queue->cmplt.raw, 0, ( queue->count * stride->cmplt ) );
        for ( i = 0 ; i < queue->fill ; i++ )
                queue->tag[i] = i;

        /* Pre-populate descriptor offsets for in-order queues */
        if ( ! ( gve->mode & GVE_MODE_DQO ) ) {
                buf = ( queue->desc.raw + stride->desc - sizeof ( *buf ) );
                for ( i = 0 ; i < queue->count ; i++ ) {
                        tag = ( i & ( queue->fill - 1 ) );
                        buf->addr = cpu_to_be64 ( gve_address ( queue, tag ) );
                        buf = ( ( ( void * ) buf ) + stride->desc );
                }
        }

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = type->create;
        qpl = ( ( gve->mode & GVE_MODE_QPL ) ? type->qpl : GVE_RAW_QPL );
        type->param ( queue, qpl, cmd );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        /* Record indices */
        db_off = ( be32_to_cpu ( queue->res->db_idx ) * sizeof ( uint32_t ) );
        evt_idx = be32_to_cpu ( queue->res->evt_idx );
        DBGC ( gve, "GVE %p %s doorbell +%#04x event counter %d\n",
               gve, type->name, db_off, evt_idx );
        queue->db = ( gve->db + db_off );
        assert ( evt_idx < gve->events.count );
        queue->event = &gve->events.event[evt_idx];
        assert ( queue->event->count == 0 );

        /* Unmask dummy interrupt */
        pci_msix_unmask ( &gve->msix, type->irq );

        /* Rearm queue interrupt if applicable */
        if ( gve->mode & GVE_MODE_DQO )
                writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[type->irq] );

        return 0;
}

References gve_buffer::addr, assert(), be32_to_cpu, cmd, cpu_to_be64, gve_irqs::db, gve_nic::db, DBGC, gve_events::event, gve_nic::events, gve_address(), gve_admin_command(), GVE_DQO_IRQ_REARM, GVE_MODE_DQO, GVE_MODE_QPL, GVE_RAW_QPL, if(), gve_nic::irqs, memset(), gve_nic::mode, gve_nic::msix, pci_msix_unmask(), queue, rc, stride, tag, type, and writel().

Referenced by gve_start().

gve_destroy_queue()

static int gve_destroy_queue ( struct gve_nic *  gve, struct gve_queue *  queue  )

static

Destroy transmit or receive queue.

Parameters

gve	GVE device
queue	Descriptor queue

Return values

rc	Return status code

Definition at line 833 of file gve.c.

                                                                              {
        const struct gve_queue_type *type = queue->type;
        int rc;

        /* Mask dummy interrupt */
        pci_msix_mask ( &gve->msix, type->irq );

        /* Issue command */
        if ( ( rc = gve_admin_simple ( gve, type->destroy, 0 ) ) != 0 )
                return rc;

        return 0;
}

References gve_nic::msix, pci_msix_mask(), queue, rc, and type.

Referenced by gve_start(), and gve_stop().

gve_alloc_shared()

static int gve_alloc_shared ( struct gve_nic *  gve )

static

Allocate shared queue resources.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 860 of file gve.c.

                                                    {
        struct dma_device *dma = gve->dma;
        struct gve_irqs *irqs = &gve->irqs;
        struct gve_events *events = &gve->events;
        size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
        size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) );
        int rc;

        /* Allocate interrupt channels */
        irqs->irq = dma_alloc ( dma, &irqs->map, irqs_len, GVE_ALIGN );
        if ( ! irqs->irq ) {
                rc = -ENOMEM;
                goto err_irqs;
        }
        DBGC ( gve, "GVE %p IRQs at [%08lx,%08lx)\n",
               gve, virt_to_phys ( irqs->irq ),
               ( virt_to_phys ( irqs->irq ) + irqs_len ) );

        /* Allocate event counters */
        events->event = dma_alloc ( dma, &events->map, events_len, GVE_ALIGN );
        if ( ! events->event ) {
                rc = -ENOMEM;
                goto err_events;
        }
        DBGC ( gve, "GVE %p events at [%08lx,%08lx)\n",
               gve, virt_to_phys ( events->event ),
               ( virt_to_phys ( events->event ) + events_len ) );

        return 0;

        dma_free ( &events->map, events->event, events_len );
 err_events:
        dma_free ( &irqs->map, irqs->irq, irqs_len );
 err_irqs:
        return rc;
}

References gve_events::count, DBGC, dma(), gve_nic::dma, dma_alloc(), dma_free(), ENOMEM, gve_events::event, gve_nic::events, GVE_ALIGN, GVE_IRQ_COUNT, gve_irqs::irq, gve_nic::irqs, gve_events::map, gve_irqs::map, and rc.

Referenced by gve_open().

gve_free_shared()

static void gve_free_shared ( struct gve_nic *  gve )

static

Free shared queue resources.

Parameters

gve	GVE device

Definition at line 902 of file gve.c.

                                                    {
        struct gve_irqs *irqs = &gve->irqs;
        struct gve_events *events = &gve->events;
        size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
        size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) );

        /* Free event counters */
        dma_free ( &events->map, events->event, events_len );

        /* Free interrupt channels */
        dma_free ( &irqs->map, irqs->irq, irqs_len );
}

References gve_events::count, dma_free(), gve_events::event, gve_nic::events, GVE_IRQ_COUNT, gve_irqs::irq, gve_nic::irqs, gve_events::map, and gve_irqs::map.

Referenced by gve_close(), and gve_open().

gve_alloc_qpl()

static int gve_alloc_qpl ( struct gve_nic *  gve, struct gve_qpl *  qpl, uint32_t  id, unsigned int  buffers  )

static

Allocate queue page list.

Parameters

gve	GVE device
qpl	Queue page list
id	Queue page list ID
buffers	Number of data buffers

Return values

rc	Return status code

Definition at line 924 of file gve.c.

                                                               {
        size_t len;

        /* Record ID */
        qpl->id = id;

        /* Calculate number of pages required */
        build_assert ( GVE_BUF_SIZE <= GVE_PAGE_SIZE );
        qpl->count = ( ( buffers + GVE_BUF_PER_PAGE - 1 ) / GVE_BUF_PER_PAGE );
        assert ( qpl->count <= GVE_QPL_MAX );

        /* Allocate pages (as a single block) */
        len = ( qpl->count * GVE_PAGE_SIZE );
        qpl->data = dma_umalloc ( gve->dma, &qpl->map, len, GVE_ALIGN );
        if ( ! qpl->data )
                return -ENOMEM;
        qpl->base = ( ( gve->mode == GVE_MODE_QPL ) ?
                      0 : dma ( &qpl->map, qpl->data ) );

        DBGC ( gve, "GVE %p QPL %#08x at [%08lx,%08lx)\n",
               gve, qpl->id, virt_to_phys ( qpl->data ),
               ( virt_to_phys ( qpl->data ) + len ) );
        return 0;
}

References assert(), gve_qpl::base, build_assert, gve_qpl::count, gve_qpl::data, DBGC, dma(), gve_nic::dma, dma_umalloc(), ENOMEM, GVE_ALIGN, GVE_BUF_PER_PAGE, GVE_BUF_SIZE, GVE_MODE_QPL, GVE_PAGE_SIZE, GVE_QPL_MAX, id, gve_qpl::id, len, gve_qpl::map, and gve_nic::mode.

Referenced by gve_alloc_queue().

gve_free_qpl()

static void gve_free_qpl ( struct gve_nic *nic  __unused, struct gve_qpl *  qpl  )

static

Free queue page list.

Parameters

gve	GVE device
qpl	Queue page list

Definition at line 956 of file gve.c.

                                                 {
        size_t len = ( qpl->count * GVE_PAGE_SIZE );

        /* Free pages */
        dma_ufree ( &qpl->map, qpl->data, len );
}

References gve_qpl::count, gve_qpl::data, dma_ufree(), GVE_PAGE_SIZE, len, and gve_qpl::map.

Referenced by gve_alloc_queue(), and gve_free_queue().

gve_next()

static unsigned int gve_next ( unsigned int  seq )

inlinestatic

Calculate next receive sequence number.

Parameters

seq	Current sequence number, or zero to start sequence

Return values

next	Next sequence number

Definition at line 971 of file gve.c.

                              {

        /* The receive completion sequence number is a modulo 7
         * counter that cycles through the non-zero three-bit values 1
         * to 7 inclusive.
         *
         * Since 7 is coprime to 2^n, this ensures that the sequence
         * number changes each time that a new completion is written
         * to memory.
         *
         * Since the counter takes only non-zero values, this ensures
         * that the sequence number changes whenever a new completion
         * is first written to a zero-initialised completion ring.
         */
        seq = ( ( seq + 1 ) & GVE_GQI_RX_SEQ_MASK );
        return ( seq ? seq : 1 );
}

References GVE_GQI_RX_SEQ_MASK, and seq.

Referenced by gve_poll_rx(), and gve_start().

gve_alloc_queue()

static int gve_alloc_queue ( struct gve_nic *  gve, struct gve_queue *  queue  )

static

Allocate descriptor queue.

Parameters

gve	GVE device
queue	Descriptor queue

Return values

rc	Return status code

Definition at line 996 of file gve.c.

                                                                            {
        const struct gve_queue_type *type = queue->type;
        struct gve_queue_stride *stride = &queue->stride;
        struct dma_device *dma = gve->dma;
        size_t desc_len;
        size_t cmplt_len;
        size_t res_len;
        int rc;

        /* Sanity checks */
        if ( ( queue->count == 0 ) ||
             ( queue->count & ( queue->count - 1 ) ) ) {
                DBGC ( gve, "GVE %p %s invalid queue size %d\n",
                       gve, type->name, queue->count );
                rc = -EINVAL;
                goto err_sanity;
        }

        /* Set queue strides and calculate total lengths */
        *stride = ( ( gve->mode & GVE_MODE_DQO ) ?
                    type->stride.dqo : type->stride.gqi );
        desc_len = ( queue->count * stride->desc );
        cmplt_len = ( queue->count * stride->cmplt );
        res_len = sizeof ( *queue->res );

        /* Calculate maximum fill level */
        assert ( ( type->fill & ( type->fill - 1 ) ) == 0 );
        queue->fill = type->fill;
        if ( queue->fill > queue->count )
                queue->fill = queue->count;
        DBGC ( gve, "GVE %p %s using QPL %#08x with %d/%d descriptors\n",
               gve, type->name, type->qpl, queue->fill, queue->count );

        /* Allocate queue page list */
        if ( ( rc = gve_alloc_qpl ( gve, &queue->qpl, type->qpl,
                                    queue->fill ) ) != 0 )
                goto err_qpl;

        /* Allocate descriptors */
        queue->desc.raw = dma_umalloc ( dma, &queue->desc_map, desc_len,
                                        GVE_ALIGN );
        if ( ! queue->desc.raw ) {
                rc = -ENOMEM;
                goto err_desc;
        }
        DBGC ( gve, "GVE %p %s descriptors at [%08lx,%08lx)\n",
               gve, type->name, virt_to_phys ( queue->desc.raw ),
               ( virt_to_phys ( queue->desc.raw ) + desc_len ) );

        /* Allocate completions */
        if ( cmplt_len ) {
                queue->cmplt.raw = dma_umalloc ( dma, &queue->cmplt_map,
                                                 cmplt_len, GVE_ALIGN );
                if ( ! queue->cmplt.raw ) {
                        rc = -ENOMEM;
                        goto err_cmplt;
                }
                DBGC ( gve, "GVE %p %s completions at [%08lx,%08lx)\n",
                       gve, type->name, virt_to_phys ( queue->cmplt.raw ),
                       ( virt_to_phys ( queue->cmplt.raw ) + cmplt_len ) );
        }

        /* Allocate queue resources */
        queue->res = dma_alloc ( dma, &queue->res_map, res_len, GVE_ALIGN );
        if ( ! queue->res ) {
                rc = -ENOMEM;
                goto err_res;
        }
        memset ( queue->res, 0, res_len );

        return 0;

        dma_free ( &queue->res_map, queue->res, res_len );
 err_res:
        if ( cmplt_len )
                dma_ufree ( &queue->cmplt_map, queue->cmplt.raw, cmplt_len );
 err_cmplt:
        dma_ufree ( &queue->desc_map, queue->desc.raw, desc_len );
 err_desc:
        gve_free_qpl ( gve, &queue->qpl );
 err_qpl:
 err_sanity:
        return rc;
}

References assert(), DBGC, dma(), gve_nic::dma, dma_alloc(), dma_free(), dma_ufree(), dma_umalloc(), EINVAL, ENOMEM, GVE_ALIGN, gve_alloc_qpl(), gve_free_qpl(), GVE_MODE_DQO, memset(), gve_nic::mode, queue, rc, stride, and type.

Referenced by gve_open().

gve_free_queue()

static void gve_free_queue ( struct gve_nic *  gve, struct gve_queue *  queue  )

static

Free descriptor queue.

Parameters

gve	GVE device
queue	Descriptor queue

Definition at line 1087 of file gve.c.

                                                                            {
        const struct gve_queue_stride *stride = &queue->stride;
        size_t desc_len = ( queue->count * stride->desc );
        size_t cmplt_len = ( queue->count * stride->cmplt );
        size_t res_len = sizeof ( *queue->res );

        /* Free queue resources */
        dma_free ( &queue->res_map, queue->res, res_len );

        /* Free completions, if applicable */
        if ( cmplt_len )
                dma_ufree ( &queue->cmplt_map, queue->cmplt.raw, cmplt_len );

        /* Free descriptors */
        dma_ufree ( &queue->desc_map, queue->desc.raw, desc_len );

        /* Free queue page list */
        gve_free_qpl ( gve, &queue->qpl );
}

References dma_free(), dma_ufree(), gve_free_qpl(), queue, and stride.

Referenced by gve_close(), and gve_open().

gve_cancel_tx()

static void gve_cancel_tx ( struct gve_nic *  gve )

static

Cancel any pending transmissions.

Parameters

gve	GVE device

Definition at line 1112 of file gve.c.

                                                  {
        struct net_device *netdev = gve->netdev;
        struct io_buffer *iobuf;
        unsigned int i;

        /* Cancel any pending transmissions */
        for ( i = 0 ; i < ( sizeof ( gve->tx_iobuf ) /
                            sizeof ( gve->tx_iobuf[0] ) ) ; i++ ) {
                iobuf = gve->tx_iobuf[i];
                gve->tx_iobuf[i] = NULL;
                if ( iobuf )
                        netdev_tx_complete_err ( netdev, iobuf, -ECANCELED );
        }
}

References ECANCELED, netdev, gve_nic::netdev, netdev_tx_complete_err(), NULL, and gve_nic::tx_iobuf.

Referenced by gve_close(), and gve_start().

gve_start()

static int gve_start ( struct gve_nic *  gve )

static

Start up device.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 1133 of file gve.c.

                                             {
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;
        int rc;

        /* Cancel any pending transmissions */
        gve_cancel_tx ( gve );

        /* Reset receive sequence */
        gve->seq = gve_next ( 0 );

        /* Configure device resources */
        if ( ( rc = gve_configure ( gve ) ) != 0 )
                goto err_configure;

        /* Register transmit queue page list */
        if ( ( rc = gve_register ( gve, &tx->qpl ) ) != 0 )
                goto err_register_tx;

        /* Register receive queue page list */
        if ( ( rc = gve_register ( gve, &rx->qpl ) ) != 0 )
                goto err_register_rx;

        /* Create transmit queue */
        if ( ( rc = gve_create_queue ( gve, tx ) ) != 0 )
                goto err_create_tx;

        /* Create receive queue */
        if ( ( rc = gve_create_queue ( gve, rx ) ) != 0 )
                goto err_create_rx;

        return 0;

        gve_destroy_queue ( gve, rx );
 err_create_rx:
        gve_destroy_queue ( gve, tx );
 err_create_tx:
        gve_unregister ( gve, &rx->qpl );
 err_register_rx:
        gve_unregister ( gve, &tx->qpl );
 err_register_tx:
        gve_deconfigure ( gve );
 err_configure:
        return rc;
}

References gve_cancel_tx(), gve_configure(), gve_create_queue(), gve_deconfigure(), gve_destroy_queue(), gve_next(), gve_register(), gve_unregister(), rc, rx, gve_nic::rx, gve_nic::seq, tx, and gve_nic::tx.

Referenced by gve_startup().

gve_stop()

static void gve_stop ( struct gve_nic *  gve )

static

Stop device.

Parameters

gve	GVE device

Definition at line 1184 of file gve.c.

                                             {
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;

        /* Destroy queues */
        gve_destroy_queue ( gve, rx );
        gve_destroy_queue ( gve, tx );

        /* Unregister page lists */
        gve_unregister ( gve, &rx->qpl );
        gve_unregister ( gve, &tx->qpl );

        /* Deconfigure device */
        gve_deconfigure ( gve );
}

References gve_deconfigure(), gve_destroy_queue(), gve_unregister(), rx, gve_nic::rx, tx, and gve_nic::tx.

Referenced by gve_close(), and gve_startup().

gve_startup()

static void gve_startup ( struct gve_nic *  gve )

static

Device startup process.

Parameters

gve	GVE device

Definition at line 1205 of file gve.c.

                                                {
        struct net_device *netdev = gve->netdev;
        int rc;

        /* Reset device */
        if ( ( rc = gve_reset ( gve ) ) != 0 )
                goto err_reset;

        /* Enable admin queue */
        gve_admin_enable ( gve );

        /* Start device */
        if ( ( rc = gve_start ( gve ) ) != 0 )
                goto err_start;

        /* Reset retry count */
        gve->retries = 0;

        /* (Ab)use link status to report startup status */
        netdev_link_up ( netdev );

        return;

        gve_stop ( gve );
 err_start:
 err_reset:
        DBGC ( gve, "GVE %p startup failed: %s\n", gve, strerror ( rc ) );
        netdev_link_err ( netdev, rc );
        if ( gve->retries++ < GVE_RESET_MAX_RETRY )
                process_add ( &gve->startup );
}

References DBGC, gve_admin_enable(), gve_reset(), GVE_RESET_MAX_RETRY, gve_start(), gve_stop(), netdev, gve_nic::netdev, netdev_link_err(), netdev_link_up(), process_add(), rc, gve_nic::retries, gve_nic::startup, and strerror().

gve_restart()

static void gve_restart ( struct gve_nic *  gve )

static

Trigger startup process.

Parameters

gve	GVE device

Definition at line 1242 of file gve.c.

                                                {
        struct net_device *netdev = gve->netdev;

        /* Mark link down to inhibit polling and transmit activity */
        netdev_link_down ( netdev );

        /* Schedule startup process */
        process_add ( &gve->startup );
}

References netdev, gve_nic::netdev, netdev_link_down(), process_add(), and gve_nic::startup.

Referenced by gve_open(), and gve_watchdog().

gve_watchdog()

static void gve_watchdog ( struct retry_timer *  timer, int over  __unused  )

static

Reset recovery watchdog.

Parameters

timer	Reset recovery watchdog timer
over	Failure indicator

Definition at line 1258 of file gve.c.

                                                                          {
        struct gve_nic *gve = container_of ( timer, struct gve_nic, watchdog );
        uint32_t activity;
        uint32_t pfn;
        int rc;

        /* Reschedule watchdog */
        start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT );

        /* Reset device (for test purposes) if applicable */
        if ( ( rc = inject_fault ( VM_MIGRATED_RATE ) ) != 0 ) {
                DBGC ( gve, "GVE %p synthesising host reset\n", gve );
                writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN );
        }

        /* Check for activity since last timer invocation */
        activity = ( gve->tx.cons + gve->rx.cons );
        if ( activity != gve->activity ) {
                gve->activity = activity;
                return;
        }

        /* Check for reset */
        pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
        if ( pfn ) {
                DBGC2 ( gve, "GVE %p idle but not in reset\n", gve );
                return;
        }

        /* Schedule restart */
        DBGC ( gve, "GVE %p watchdog detected reset by host\n", gve );
        gve_restart ( gve );
}

References gve_nic::activity, gve_nic::cfg, gve_queue::cons, container_of, DBGC, DBGC2, GVE_CFG_ADMIN_PFN, gve_restart(), GVE_WATCHDOG_TIMEOUT, rc, readl(), gve_nic::rx, start_timer_fixed(), gve_nic::tx, VM_MIGRATED_RATE, gve_nic::watchdog, and writel().

Referenced by gve_probe().

gve_open()

static int gve_open ( struct net_device *  netdev )

static

Open network device.

Parameters

netdev

Network device

Return values

rc	Return status code

Definition at line 1298 of file gve.c.

                                                  {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;
        int rc;

        /* Allocate shared queue resources */
        if ( ( rc = gve_alloc_shared ( gve ) ) != 0 )
                goto err_alloc_shared;

        /* Allocate and prepopulate transmit queue */
        if ( ( rc = gve_alloc_queue ( gve, tx ) ) != 0 )
                goto err_alloc_tx;

        /* Allocate and prepopulate receive queue */
        if ( ( rc = gve_alloc_queue ( gve, rx ) ) != 0 )
                goto err_alloc_rx;

        /* Trigger startup */
        gve_restart ( gve );

        /* Start reset recovery watchdog timer */
        start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT );

        return 0;

        gve_free_queue ( gve, rx );
 err_alloc_rx:
        gve_free_queue ( gve, tx );
 err_alloc_tx:
        gve_free_shared ( gve );
 err_alloc_shared:
        return rc;
}

References gve_alloc_queue(), gve_alloc_shared(), gve_free_queue(), gve_free_shared(), gve_restart(), GVE_WATCHDOG_TIMEOUT, netdev, net_device::priv, rc, rx, gve_nic::rx, start_timer_fixed(), tx, gve_nic::tx, and gve_nic::watchdog.

gve_close()

static void gve_close ( struct net_device *  netdev )

static

Close network device.

Parameters

netdev

Network device

Definition at line 1338 of file gve.c.

                                                    {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;

        /* Stop reset recovery timer */
        stop_timer ( &gve->watchdog );

        /* Terminate startup process */
        process_del ( &gve->startup );

        /* Stop and reset device */
        gve_stop ( gve );
        gve_reset ( gve );

        /* Cancel any pending transmissions */
        gve_cancel_tx ( gve );

        /* Free queues */
        gve_free_queue ( gve, rx );
        gve_free_queue ( gve, tx );

        /* Free shared queue resources */
        gve_free_shared ( gve );
}

References gve_cancel_tx(), gve_free_queue(), gve_free_shared(), gve_reset(), gve_stop(), netdev, net_device::priv, process_del(), rx, gve_nic::rx, gve_nic::startup, stop_timer(), tx, gve_nic::tx, and gve_nic::watchdog.

gve_transmit()

static int gve_transmit ( struct net_device *  netdev, struct io_buffer *  iobuf  )

static

Transmit packet.

Parameters

netdev	Network device
iobuf	I/O buffer

Return values

rc	Return status code

Definition at line 1371 of file gve.c.

                                                                               {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_gqi_tx_descriptor *gqi;
        struct gve_dqo_tx_descriptor *dqo;
        unsigned int count;
        unsigned int index;
        unsigned int tag;
        unsigned int chain;
        uint32_t doorbell;
        size_t frag_len;
        size_t offset;
        size_t next;
        size_t len;

        /* Do nothing if queues are not yet set up */
        if ( ! netdev_link_ok ( netdev ) )
                return -ENETDOWN;

        /* Defer packet if there is no space in the transmit ring */
        len = iob_len ( iobuf );
        count = ( ( len + GVE_BUF_SIZE - 1 ) / GVE_BUF_SIZE );
        if ( ( ( tx->prod - tx->cons ) + count ) > tx->fill ) {
                netdev_tx_defer ( netdev, iobuf );
                return 0;
        }

        /* Copy packet to queue pages and populate descriptors */
        for ( offset = 0, chain = 0 ; ; offset = next, chain = tag ) {

                /* Identify next available buffer */
                index = ( tx->prod++ & ( tx->count - 1 ) );
                tag = tx->tag[ index % GVE_TX_FILL ];

                /* Sanity check */
                assert ( gve->tx_iobuf[tag] == NULL );

                /* Copy packet fragment */
                frag_len = ( len - offset );
                if ( frag_len > GVE_BUF_SIZE )
                        frag_len = GVE_BUF_SIZE;
                memcpy ( gve_buffer ( tx, tag ),
                         ( iobuf->data + offset ), frag_len );
                next = ( offset + frag_len );

                /* Populate descriptor */
                if ( gve->mode & GVE_MODE_DQO ) {

                        /* Out-of-order descriptor */
                        dqo = &tx->desc.tx.dqo[index];
                        dqo->buf.addr =
                                cpu_to_le64 ( gve_address ( tx, tag ) );
                        if ( next == len ) {
                                dqo->type = ( GVE_DQO_TX_TYPE_PACKET |
                                              GVE_DQO_TX_TYPE_LAST );
                                dqo->tag.id = tag;
                                dqo->tag.count = count;
                        } else {
                                dqo->type = GVE_DQO_TX_TYPE_PACKET;
                                dqo->tag.id = 0;
                                dqo->tag.count = 0;
                        }
                        dqo->len = cpu_to_le16 ( frag_len );
                        gve->tx_chain[tag] = chain;

                } else {

                        /* In-order descriptor */
                        gqi = &tx->desc.tx.gqi[index];
                        if ( offset ) {
                                gqi->type = GVE_GQI_TX_TYPE_CONT;
                                gqi->count = 0;
                                gqi->total = 0;
                        } else {
                                gqi->type = GVE_GQI_TX_TYPE_START;
                                gqi->count = count;
                                gqi->total = cpu_to_be16 ( len );
                        }
                        gqi->len = cpu_to_be16 ( frag_len );

                }
                DBGC2 ( gve, "GVE %p TXD %#04x %#02x:%#02x len %#04zx/%#04zx "
                        "at %#08lx\n", gve, index, tag, count, frag_len, len,
                        gve_address ( tx, tag ) );

                /* Record I/O buffer against final descriptor */
                if ( next == len ) {
                        gve->tx_iobuf[tag] = iobuf;
                        break;
                }
        }
        assert ( ( tx->prod - tx->cons ) <= tx->fill );

        /* Ring doorbell */
        doorbell = tx->prod;
        if ( gve->mode & GVE_MODE_DQO ) {
                doorbell &= ( tx->count - 1 );
        } else {
                doorbell = bswap_32 ( doorbell );
        }
        wmb();
        writel ( doorbell, tx->db );

        return 0;
}

References gve_buffer::addr, assert(), bswap_32, gve_dqo_tx_descriptor::buf, count, gve_gqi_tx_descriptor::count, gve_dqo_tx_tag::count, cpu_to_be16, cpu_to_le16, cpu_to_le64, io_buffer::data, DBGC2, doorbell, ENETDOWN, gve_address(), GVE_BUF_SIZE, GVE_DQO_TX_TYPE_LAST, GVE_DQO_TX_TYPE_PACKET, GVE_GQI_TX_TYPE_CONT, GVE_GQI_TX_TYPE_START, GVE_MODE_DQO, GVE_TX_FILL, gve_dqo_tx_tag::id, index, iob_len(), len, gve_gqi_tx_descriptor::len, gve_dqo_tx_descriptor::len, memcpy(), gve_nic::mode, netdev, netdev_link_ok(), netdev_tx_defer(), next, NULL, offset, net_device::priv, tag, gve_dqo_tx_descriptor::tag, gve_gqi_tx_descriptor::total, tx, gve_nic::tx, gve_nic::tx_chain, gve_nic::tx_iobuf, gve_gqi_tx_descriptor::type, gve_dqo_tx_descriptor::type, wmb(), and writel().

gve_poll_tx()

static void gve_poll_tx ( struct net_device *  netdev )

static

Poll for completed transmissions.

Parameters

netdev

Network device

Definition at line 1482 of file gve.c.

                                                      {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_dqo_tx_completion *dqo;
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int gen;
        unsigned int bit;
        unsigned int tag;
        uint32_t count;

        /* Process transmit completions */
        if ( gve->mode & GVE_MODE_DQO ) {

                /* Out-of-order completions */
                while ( 1 ) {

                        /* Read next possible completion */
                        gen = ( tx->done & tx->count );
                        index = ( tx->done & ( tx->count - 1 ) );
                        dqo = &tx->cmplt.tx.dqo[index];

                        /* Check generation bit */
                        bit = ( dqo->flags & GVE_DQO_TXF_GEN );
                        if ( ( !! bit ) == ( !! gen ) )
                                break;
                        rmb();
                        tx->done++;

                        /* Ignore non-packet completions */
                        if ( ( ! ( dqo->flags & GVE_DQO_TXF_PKT ) ) ||
                             ( dqo->tag.count < 0 ) ) {
                                DBGC2 ( gve, "GVE %p TXC %#04x flags %#02x "
                                        "ignored\n", gve, index, dqo->flags );
                                continue;
                        }

                        /* Parse completion */
                        tag = dqo->tag.id;
                        count = dqo->tag.count;
                        iobuf = gve->tx_iobuf[tag];
                        gve->tx_iobuf[tag] = NULL;
                        assert ( iobuf != NULL );

                        /* Return completed descriptors to ring */
                        while ( count-- ) {
                                DBGC2 ( gve, "GVE %p TXC %#04x %#02x:%#02x "
                                        "complete\n", gve, index, tag,
                                        dqo->tag.count );
                                tx->tag[ tx->cons++ % GVE_TX_FILL ] = tag;
                                tag = gve->tx_chain[tag];
                        }

                        /* Hand off to network stack */
                        if ( iobuf )
                                netdev_tx_complete ( netdev, iobuf );
                }

        } else {

                /* Read event counter */
                count = be32_to_cpu ( tx->event->count );

                /* Process transmit completions */
                while ( count != tx->cons ) {
                        DBGC2 ( gve, "GVE %p TXC %#04x complete\n",
                                gve, tx->cons );
                        tag = ( tx->cons % GVE_TX_FILL );
                        iobuf = gve->tx_iobuf[tag];
                        gve->tx_iobuf[tag] = NULL;
                        tx->cons++;
                        if ( iobuf )
                                netdev_tx_complete ( netdev, iobuf );
                }
        }
}

References assert(), be32_to_cpu, bit, count, gve_dqo_tx_tag::count, DBGC2, gve_dqo_tx_completion::flags, GVE_DQO_TXF_GEN, GVE_DQO_TXF_PKT, GVE_MODE_DQO, GVE_TX_FILL, gve_dqo_tx_tag::id, index, gve_nic::mode, netdev, netdev_tx_complete(), NULL, net_device::priv, rmb, tag, gve_dqo_tx_completion::tag, tx, gve_nic::tx, gve_nic::tx_chain, and gve_nic::tx_iobuf.

Referenced by gve_poll().

gve_poll_rx()

static void gve_poll_rx ( struct net_device *  netdev )

static

Poll for received packets.

Parameters

netdev

Network device

Definition at line 1564 of file gve.c.

                                                      {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *rx = &gve->rx;
        struct gve_gqi_rx_completion *gqi;
        struct gve_dqo_rx_completion *dqo;
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int gen;
        unsigned int bit;
        unsigned int seq;
        unsigned int tag;
        uint32_t done;
        size_t total;
        size_t len;
        int rc;

        /* Process receive completions */
        done = rx->done;
        seq = gve->seq;
        total = 0;
        while ( 1 ) {

                /* Read next possible completion */
                rc = 0;
                gen = ( done & rx->count );
                index = ( done++ & ( rx->count - 1 ) );
                if ( gve->mode & GVE_MODE_DQO ) {

                        /* Out-of-order completion */
                        dqo = &rx->cmplt.rx.dqo[index];

                        /* Check generation bit */
                        bit = ( dqo->len & cpu_to_le16 ( GVE_DQO_RXL_GEN ) );
                        if ( ( !! bit ) == ( !! gen ) )
                                break;
                        rmb();

                        /* Parse completion */
                        len = ( le16_to_cpu ( dqo->len ) &
                                ( GVE_BUF_SIZE - 1 ) );
                        tag = dqo->tag;
                        DBGC2 ( gve, "GVE %p RXC %#04x %#02x:%#02x len %#04zx "
                                "at %#08zx\n", gve, index, tag, dqo->flags,
                                len, gve_offset ( rx, tag ) );

                        /* Accumulate a complete packet */
                        if ( dqo->status & GVE_DQO_RXS_ERROR ) {
                                rc = -EIO;
                                total = 0;
                        } else {
                                total += len;
                                if ( ! ( dqo->flags & GVE_DQO_RXF_LAST ) )
                                        continue;
                        }

                } else {

                        /* In-order completion */
                        gqi = &rx->cmplt.rx.gqi[index];

                        /* Check sequence number */
                        if ( ( gqi->seq & GVE_GQI_RX_SEQ_MASK ) != seq )
                                break;
                        rmb();
                        seq = gve_next ( seq );

                        /* Parse completion */
                        len = be16_to_cpu ( gqi->len );
                        tag = ( index % GVE_RX_FILL );
                        DBGC2 ( gve, "GVE %p RXC %#04x %#02x:%#02x len %#04zx "
                                "at %#08zx\n", gve, index, gqi->seq,
                                gqi->flags, len, gve_offset ( rx, tag ) );

                        /* Accumulate a complete packet */
                        if ( gqi->flags & GVE_GQI_RXF_ERROR ) {
                                rc = -EIO;
                                total = 0;
                        } else {
                                total += len;
                                if ( gqi->flags & GVE_GQI_RXF_MORE )
                                        continue;
                        }
                        gve->seq = seq;
                }

                /* Allocate and populate I/O buffer */
                iobuf = ( total ? alloc_iob ( total ) : NULL );
                for ( ; rx->done != done ; rx->done++ ) {

                        /* Re-read completion and return tag to ring */
                        index = ( rx->done & ( rx->count - 1 ) );
                        if ( gve->mode & GVE_MODE_DQO ) {
                                dqo = &rx->cmplt.rx.dqo[index];
                                tag = dqo->tag;
                                len = ( le16_to_cpu ( dqo->len ) &
                                        ( GVE_BUF_SIZE - 1 ) );
                                rx->tag[ rx->cons++ % GVE_RX_FILL ] = tag;
                        } else {
                                gqi = &rx->cmplt.rx.gqi[index];
                                tag = ( index % GVE_RX_FILL );
                                len = be16_to_cpu ( gqi->len );
                                assert ( rx->cons == rx->done );
                                rx->cons++;
                        }

                        /* Copy data */
                        if ( iobuf ) {
                                memcpy ( iob_put ( iobuf, len ),
                                         gve_buffer ( rx, tag ), len );
                        }
                }
                assert ( ( iobuf == NULL ) || ( iob_len ( iobuf ) == total ) );
                total = 0;

                /* Hand off packet to network stack */
                if ( iobuf ) {
                        if ( ! ( gve->mode & GVE_MODE_DQO ) )
                                iob_pull ( iobuf, GVE_GQI_RX_PAD );
                        netdev_rx ( netdev, iobuf );
                } else {
                        netdev_rx_err ( netdev, NULL, ( rc ? rc : -ENOMEM ) );
                }
        }
}

References alloc_iob(), assert(), be16_to_cpu, bit, cpu_to_le16, DBGC2, done, EIO, ENOMEM, gve_gqi_rx_completion::flags, gve_dqo_rx_completion::flags, GVE_BUF_SIZE, GVE_DQO_RXF_LAST, GVE_DQO_RXL_GEN, GVE_DQO_RXS_ERROR, GVE_GQI_RX_PAD, GVE_GQI_RX_SEQ_MASK, GVE_GQI_RXF_ERROR, GVE_GQI_RXF_MORE, GVE_MODE_DQO, gve_next(), gve_offset(), GVE_RX_FILL, index, iob_len(), iob_pull, iob_put, le16_to_cpu, len, gve_gqi_rx_completion::len, gve_dqo_rx_completion::len, memcpy(), gve_nic::mode, netdev, netdev_rx(), netdev_rx_err(), NULL, net_device::priv, rc, rmb, rx, gve_nic::rx, seq, gve_gqi_rx_completion::seq, gve_nic::seq, gve_dqo_rx_completion::status, tag, and gve_dqo_rx_completion::tag.

Referenced by gve_poll().

gve_refill_rx()

static void gve_refill_rx ( struct net_device *  netdev )

static

Refill receive queue.

Parameters

netdev

Network device

Definition at line 1694 of file gve.c.

                                                        {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *rx = &gve->rx;
        struct gve_dqo_rx_descriptor *dqo;
        unsigned int refill;
        unsigned int index;
        unsigned int tag;
        uint32_t doorbell;

        /* Calculate refill quantity */
        doorbell = ( rx->cons + rx->fill );
        refill = ( doorbell - rx->prod );
        if ( ! refill )
                return;

        /* Refill ring */
        if ( gve->mode & GVE_MODE_DQO ) {

                /* Out-of-order descriptors */
                while ( refill-- ) {

                        /* Identify next available buffer */
                        index = ( rx->prod++ & ( rx->count - 1 ) );
                        tag = rx->tag[ index % GVE_RX_FILL ];

                        /* Populate descriptor */
                        dqo = &rx->desc.rx.dqo[index];
                        dqo->tag = tag;
                        dqo->buf.addr =
                                cpu_to_le64 ( gve_address ( rx, tag ) );
                        DBGC2 ( gve, "GVE %p RXD %#04x:%#02x at %#08llx\n",
                                gve, index, dqo->tag,
                                ( ( unsigned long long )
                                  le64_to_cpu ( dqo->buf.addr ) ) );
                }
                wmb();
                assert ( rx->prod == doorbell );

        } else {

                /* The in-order receive descriptors are prepopulated
                 * at the time of creating the receive queue (pointing
                 * to the preallocated queue pages).  Refilling is
                 * therefore just a case of ringing the doorbell if
                 * the device is not yet aware of any available
                 * descriptors.
                 */
                rx->prod += refill;
                assert ( rx->prod == doorbell );
                DBGC2 ( gve, "GVE %p RXD %#04x ready\n", gve, rx->prod );

                /* Doorbell is big-endian */
                doorbell = bswap_32 ( doorbell );
        }

        /* Ring doorbell */
        writel ( doorbell, rx->db );
}

References gve_buffer::addr, assert(), bswap_32, gve_dqo_rx_descriptor::buf, cpu_to_le64, DBGC2, doorbell, gve_address(), GVE_MODE_DQO, GVE_RX_FILL, index, le64_to_cpu, gve_nic::mode, netdev, net_device::priv, rx, gve_nic::rx, tag, gve_dqo_rx_descriptor::tag, wmb(), and writel().

Referenced by gve_poll().

gve_poll()

static void gve_poll ( struct net_device *  netdev )

static

Poll for completed and received packets.

Parameters

netdev

Network device

Definition at line 1758 of file gve.c.

                                                   {
        struct gve_nic *gve = netdev->priv;

        /* Do nothing if queues are not yet set up */
        if ( ! netdev_link_ok ( netdev ) )
                return;

        /* Poll for transmit completions */
        gve_poll_tx ( netdev );

        /* Poll for receive completions */
        gve_poll_rx ( netdev );

        /* Refill receive queue */
        gve_refill_rx ( netdev );

        /* Rearm queue interrupts if applicable */
        if ( gve->mode & GVE_MODE_DQO ) {
                writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[GVE_TX_IRQ] );
                writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[GVE_RX_IRQ] );
        }
}

References gve_irqs::db, GVE_DQO_IRQ_REARM, GVE_MODE_DQO, gve_poll_rx(), gve_poll_tx(), gve_refill_rx(), GVE_RX_IRQ, GVE_TX_IRQ, gve_nic::irqs, gve_nic::mode, netdev, netdev_link_ok(), net_device::priv, and writel().

gve_setup()

static int gve_setup ( struct gve_nic *  gve )

static

Set up admin queue and get device description.

Parameters

gve	GVE device

Return values

rc	Return status code

Definition at line 1843 of file gve.c.

                                             {
        unsigned int i;
        int rc;

        /* Attempt several times, since the device may decide to add
         * in a few spurious resets.
         */
        for ( i = 0 ; i < GVE_RESET_MAX_RETRY ; i++ ) {

                /* Reset device */
                if ( ( rc = gve_reset ( gve ) ) != 0 )
                        continue;

                /* Enable admin queue */
                gve_admin_enable ( gve );

                /* Fetch MAC address */
                if ( ( rc = gve_describe ( gve ) ) != 0 )
                        continue;

                /* Success */
                return 0;
        }

        DBGC ( gve, "GVE %p failed to get device description: %s\n",
               gve, strerror ( rc ) );
        return rc;
}

References DBGC, gve_admin_enable(), gve_describe(), gve_reset(), GVE_RESET_MAX_RETRY, rc, and strerror().

Referenced by gve_probe().

gve_probe()

static int gve_probe ( struct pci_device *  pci )

static

Probe PCI device.

Parameters

pci	PCI device

Return values

rc	Return status code

Definition at line 1882 of file gve.c.

                                                {
        struct net_device *netdev;
        struct gve_nic *gve;
        unsigned long cfg_start;
        unsigned long db_start;
        unsigned long db_size;
        int rc;

        /* Allocate and initialise net device */
        netdev = alloc_etherdev ( sizeof ( *gve ) );
        if ( ! netdev ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        netdev_init ( netdev, &gve_operations );
        gve = netdev->priv;
        pci_set_drvdata ( pci, netdev );
        netdev->dev = &pci->dev;
        memset ( gve, 0, sizeof ( *gve ) );
        gve->netdev = netdev;
        gve->tx.type = &gve_tx_type;
        gve->rx.type = &gve_rx_type;
        gve->tx.tag = gve->tx_tag;
        gve->rx.tag = gve->rx_tag;
        process_init_stopped ( &gve->startup, &gve_startup_desc,
                               &netdev->refcnt );
        timer_init ( &gve->watchdog, gve_watchdog, &netdev->refcnt );

        /* Fix up PCI device */
        adjust_pci_device ( pci );

        /* Check PCI revision */
        pci_read_config_byte ( pci, PCI_REVISION, &gve->revision );
        DBGC ( gve, "GVE %p is revision %#02x\n", gve, gve->revision );

        /* Map configuration registers */
        cfg_start = pci_bar_start ( pci, GVE_CFG_BAR );
        gve->cfg = pci_ioremap ( pci, cfg_start, GVE_CFG_SIZE );
        if ( ! gve->cfg ) {
                rc = -ENODEV;
                goto err_cfg;
        }

        /* Map doorbell registers */
        db_start = pci_bar_start ( pci, GVE_DB_BAR );
        db_size = pci_bar_size ( pci, GVE_DB_BAR );
        gve->db = pci_ioremap ( pci, db_start, db_size );
        if ( ! gve->db ) {
                rc = -ENODEV;
                goto err_db;
        }

        /* Configure DMA */
        gve->dma = &pci->dma;
        dma_set_mask_64bit ( gve->dma );
        assert ( netdev->dma == NULL );

        /* Configure dummy MSI-X interrupt */
        if ( ( rc = pci_msix_enable ( pci, &gve->msix ) ) != 0 )
                goto err_msix;

        /* Allocate admin queue */
        if ( ( rc = gve_admin_alloc ( gve ) ) != 0 )
                goto err_admin;

        /* Set up the device */
        if ( ( rc = gve_setup ( gve ) ) != 0 )
                goto err_setup;

        /* Register network device */
        if ( ( rc = register_netdev ( netdev ) ) != 0 )
                goto err_register_netdev;

        return 0;

        unregister_netdev ( netdev );
 err_register_netdev:
 err_setup:
        gve_reset ( gve );
        gve_admin_free ( gve );
 err_admin:
        pci_msix_disable ( pci, &gve->msix );
 err_msix:
        iounmap ( gve->db );
 err_db:
        iounmap ( gve->cfg );
 err_cfg:
        netdev_nullify ( netdev );
        netdev_put ( netdev );
 err_alloc:
        return rc;
}

References adjust_pci_device(), alloc_etherdev(), assert(), gve_nic::cfg, gve_nic::db, DBGC, pci_device::dev, net_device::dev, pci_device::dma, net_device::dma, gve_nic::dma, dma_set_mask_64bit(), ENODEV, ENOMEM, gve_admin_alloc(), gve_admin_free(), GVE_CFG_BAR, GVE_CFG_SIZE, GVE_DB_BAR, gve_operations, gve_reset(), gve_rx_type, gve_setup(), gve_startup_desc, gve_tx_type, gve_watchdog(), iounmap(), memset(), gve_nic::msix, netdev, gve_nic::netdev, netdev_init(), netdev_nullify(), netdev_put(), NULL, pci_bar_size(), pci_bar_start(), pci_ioremap(), pci_msix_disable(), pci_msix_enable(), pci_read_config_byte(), PCI_REVISION, pci_set_drvdata(), net_device::priv, process_init_stopped(), rc, net_device::refcnt, register_netdev(), gve_nic::revision, gve_nic::rx, gve_nic::rx_tag, gve_nic::startup, gve_queue::tag, gve_nic::tx, gve_nic::tx_tag, gve_queue::type, unregister_netdev(), and gve_nic::watchdog.

gve_remove()

static void gve_remove ( struct pci_device *  pci )

static

Remove PCI device.

Parameters

pci	PCI device

Definition at line 1980 of file gve.c.

                                                  {
        struct net_device *netdev = pci_get_drvdata ( pci );
        struct gve_nic *gve = netdev->priv;

        /* Unregister network device */
        unregister_netdev ( netdev );

        /* Reset device */
        gve_reset ( gve );

        /* Free admin queue */
        gve_admin_free ( gve );

        /* Disable dummy MSI-X interrupt */
        pci_msix_disable ( pci, &gve->msix );

        /* Unmap registers */
        iounmap ( gve->db );
        iounmap ( gve->cfg );

        /* Free network device */
        netdev_nullify ( netdev );
        netdev_put ( netdev );
}

References gve_nic::cfg, gve_nic::db, gve_admin_free(), gve_reset(), iounmap(), gve_nic::msix, netdev, netdev_nullify(), netdev_put(), pci_get_drvdata(), pci_msix_disable(), net_device::priv, and unregister_netdev().

Variable Documentation

gve_operations

struct net_device_operations gve_operations

static

Initial value:

= {
        .open           = gve_open,
        .close          = gve_close,
        .transmit       = gve_transmit,
        .poll           = gve_poll,
}

GVE network device operations.

Definition at line 1782 of file gve.c.

Referenced by gve_probe().

gve_tx_type

const struct gve_queue_type gve_tx_type

static

Initial value:

= {
        .name = "TX",
        .param = gve_create_tx_param,
        .qpl = GVE_TX_QPL,
        .irq = GVE_TX_IRQ,
        .fill = GVE_TX_FILL,
        .stride = {
                .gqi = {
                        .desc = sizeof ( struct gve_gqi_tx_descriptor ),
                },
                .dqo = {
                        .desc = sizeof ( struct gve_dqo_tx_descriptor ),
                        .cmplt = sizeof ( struct gve_dqo_tx_completion ),
                },
        },
        .create = GVE_ADMIN_CREATE_TX,
        .destroy = GVE_ADMIN_DESTROY_TX,
}

Transmit descriptor queue type.

Definition at line 1797 of file gve.c.

Referenced by gve_probe().

gve_rx_type

const struct gve_queue_type gve_rx_type

static

Initial value:

= {
        .name = "RX",
        .param = gve_create_rx_param,
        .qpl = GVE_RX_QPL,
        .irq = GVE_RX_IRQ,
        .fill = GVE_RX_FILL,
        .stride = {
                .gqi = {
                        .desc = sizeof ( struct gve_gqi_rx_descriptor ),
                        .cmplt = sizeof ( struct gve_gqi_rx_completion ),
                },
                .dqo = {
                        .desc = sizeof ( struct gve_dqo_rx_descriptor ),
                        .cmplt = sizeof ( struct gve_dqo_rx_completion ),
                },
        },
        .create = GVE_ADMIN_CREATE_RX,
        .destroy = GVE_ADMIN_DESTROY_RX,
}

Receive descriptor queue type.

Definition at line 1817 of file gve.c.

Referenced by gve_probe().

gve_startup_desc

struct process_descriptor gve_startup_desc

static

Initial value:

=
        PROC_DESC_ONCE ( struct gve_nic, startup, gve_startup )

Device startup process descriptor.

Definition at line 1873 of file gve.c.

Referenced by gve_probe().

gve_nics

struct pci_device_id gve_nics[]

static

Initial value:

= {
        PCI_ROM ( 0x1ae0, 0x0042, "gve", "gVNIC", 0 ),
}

GVE PCI device IDs.

Definition at line 2006 of file gve.c.

__pci_driver

struct pci_driver gve_driver __pci_driver

Initial value:

= {
        .ids = gve_nics,
        .id_count = ( sizeof ( gve_nics ) / sizeof ( gve_nics[0] ) ),
        .probe = gve_probe,
        .remove = gve_remove,
}

GVE PCI driver.

Definition at line 2011 of file gve.c.