gve.c

Go to the documentation of this file.

/*
 * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 *
 * You can also choose to distribute this program under the terms of
 * the Unmodified Binary Distribution Licence (as given in the file
 * COPYING.UBDL), provided that you have satisfied its requirements.
 */

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>
#include <byteswap.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include <ipxe/if_ether.h>
#include <ipxe/iobuf.h>
#include <ipxe/dma.h>
#include <ipxe/pci.h>
#include <ipxe/fault.h>
#include "gve.h"

/** @file
 *
 * Google Virtual Ethernet network driver
 *
 */

/* Disambiguate the various error causes */
#define EINFO_EIO_ADMIN_UNSET                                           \
        __einfo_uniqify ( EINFO_EIO, 0x00, "Uncompleted" )
#define EIO_ADMIN_UNSET                                                 \
        __einfo_error ( EINFO_EIO_ADMIN_UNSET )
#define EINFO_EIO_ADMIN_ABORTED                                         \
        __einfo_uniqify ( EINFO_EIO, 0x10, "Aborted" )
#define EIO_ADMIN_ABORTED                                               \
        __einfo_error ( EINFO_EIO_ADMIN_ABORTED )
#define EINFO_EIO_ADMIN_EXISTS                                          \
        __einfo_uniqify ( EINFO_EIO, 0x11, "Already exists" )
#define EIO_ADMIN_EXISTS                                                \
        __einfo_error ( EINFO_EIO_ADMIN_EXISTS )
#define EINFO_EIO_ADMIN_CANCELLED                                       \
        __einfo_uniqify ( EINFO_EIO, 0x12, "Cancelled" )
#define EIO_ADMIN_CANCELLED                                             \
        __einfo_error ( EINFO_EIO_ADMIN_CANCELLED )
#define EINFO_EIO_ADMIN_DATALOSS                                        \
        __einfo_uniqify ( EINFO_EIO, 0x13, "Data loss" )
#define EIO_ADMIN_DATALOSS                                              \
        __einfo_error ( EINFO_EIO_ADMIN_DATALOSS )
#define EINFO_EIO_ADMIN_DEADLINE                                        \
        __einfo_uniqify ( EINFO_EIO, 0x14, "Deadline exceeded" )
#define EIO_ADMIN_DEADLINE                                              \
        __einfo_error ( EINFO_EIO_ADMIN_DEADLINE )
#define EINFO_EIO_ADMIN_PRECONDITION                                    \
        __einfo_uniqify ( EINFO_EIO, 0x15, "Failed precondition" )
#define EIO_ADMIN_PRECONDITION                                          \
        __einfo_error ( EINFO_EIO_ADMIN_PRECONDITION )
#define EINFO_EIO_ADMIN_INTERNAL                                        \
        __einfo_uniqify ( EINFO_EIO, 0x16, "Internal error" )
#define EIO_ADMIN_INTERNAL                                              \
        __einfo_error ( EINFO_EIO_ADMIN_INTERNAL )
#define EINFO_EIO_ADMIN_INVAL                                           \
        __einfo_uniqify ( EINFO_EIO, 0x17, "Invalid argument" )
#define EIO_ADMIN_INVAL                                                 \
        __einfo_error ( EINFO_EIO_ADMIN_INVAL )
#define EINFO_EIO_ADMIN_NOT_FOUND                                       \
        __einfo_uniqify ( EINFO_EIO, 0x18, "Not found" )
#define EIO_ADMIN_NOT_FOUND                                             \
        __einfo_error ( EINFO_EIO_ADMIN_NOT_FOUND )
#define EINFO_EIO_ADMIN_RANGE                                           \
        __einfo_uniqify ( EINFO_EIO, 0x19, "Out of range" )
#define EIO_ADMIN_RANGE                                                 \
        __einfo_error ( EINFO_EIO_ADMIN_RANGE )
#define EINFO_EIO_ADMIN_PERM                                            \
        __einfo_uniqify ( EINFO_EIO, 0x1a, "Permission denied" )
#define EIO_ADMIN_PERM                                                  \
        __einfo_error ( EINFO_EIO_ADMIN_PERM )
#define EINFO_EIO_ADMIN_UNAUTH                                          \
        __einfo_uniqify ( EINFO_EIO, 0x1b, "Unauthenticated" )
#define EIO_ADMIN_UNAUTH                                                \
        __einfo_error ( EINFO_EIO_ADMIN_UNAUTH )
#define EINFO_EIO_ADMIN_RESOURCE                                        \
        __einfo_uniqify ( EINFO_EIO, 0x1c, "Resource exhausted" )
#define EIO_ADMIN_RESOURCE                                              \
        __einfo_error ( EINFO_EIO_ADMIN_RESOURCE )
#define EINFO_EIO_ADMIN_UNAVAIL                                         \
        __einfo_uniqify ( EINFO_EIO, 0x1d, "Unavailable" )
#define EIO_ADMIN_UNAVAIL                                               \
        __einfo_error ( EINFO_EIO_ADMIN_UNAVAIL )
#define EINFO_EIO_ADMIN_NOTSUP                                          \
        __einfo_uniqify ( EINFO_EIO, 0x1e, "Unimplemented" )
#define EIO_ADMIN_NOTSUP                                                \
        __einfo_error ( EINFO_EIO_ADMIN_NOTSUP )
#define EINFO_EIO_ADMIN_UNKNOWN                                         \
        __einfo_uniqify ( EINFO_EIO, 0x1f, "Unknown error" )
#define EIO_ADMIN_UNKNOWN                                               \
        __einfo_error ( EINFO_EIO_ADMIN_UNKNOWN )
#define EIO_ADMIN( status )                                             \
        EUNIQ ( EINFO_EIO, ( (status) & 0x1f ),                         \
                EIO_ADMIN_UNSET, EIO_ADMIN_ABORTED, EIO_ADMIN_EXISTS,   \
                EIO_ADMIN_CANCELLED, EIO_ADMIN_DATALOSS,                \
                EIO_ADMIN_DEADLINE, EIO_ADMIN_PRECONDITION,             \
                EIO_ADMIN_INTERNAL, EIO_ADMIN_INVAL,                    \
                EIO_ADMIN_NOT_FOUND, EIO_ADMIN_RANGE, EIO_ADMIN_PERM,   \
                EIO_ADMIN_UNAUTH, EIO_ADMIN_RESOURCE,                   \
                EIO_ADMIN_UNAVAIL, EIO_ADMIN_NOTSUP, EIO_ADMIN_UNKNOWN )

/******************************************************************************
 *
 * Buffer layout
 *
 ******************************************************************************
 */

/**
 * Get buffer offset (within queue page list allocation)
 *
 * @v queue             Descriptor queue
 * @v tag               Buffer tag
 * @ret addr            Buffer address within queue page list address space
 */
static inline __attribute__ (( always_inline)) size_t
gve_offset ( struct gve_queue *queue, unsigned int tag ) {

        /* We allocate sufficient pages for the maximum fill level of
         * buffers, and reuse the buffers in strict rotation as they
         * are released by the hardware.
         */
        assert ( tag < queue->fill );
        return ( tag * GVE_BUF_SIZE );
}

/**
 * Get buffer address (within queue page list address space)
 *
 * @v queue             Descriptor queue
 * @v tag               Buffer tag
 * @ret addr            Buffer address within queue page list address space
 */
static inline __attribute__ (( always_inline)) physaddr_t
gve_address ( struct gve_queue *queue, unsigned int tag ) {

        /* Pages are allocated as a single contiguous block */
        return ( queue->qpl.base + gve_offset ( queue, tag ) );
}

/**
 * Get buffer address
 *
 * @v queue             Descriptor queue
 * @v tag               Buffer tag
 * @ret addr            Buffer address
 */
static inline __attribute__ (( always_inline )) void *
gve_buffer ( struct gve_queue *queue, unsigned int tag ) {

        /* Pages are allocated as a single contiguous block */
        return ( queue->qpl.data + gve_offset ( queue, tag ) );
}

/******************************************************************************
 *
 * Device reset
 *
 ******************************************************************************
 */

/**
 * Reset hardware
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_reset ( struct gve_nic *gve ) {
        uint32_t pfn;
        unsigned int i;

        /* Skip reset if admin queue page frame number is already
         * clear.  Triggering a reset on an already-reset device seems
         * to cause a delayed reset to be scheduled.  This can cause
         * the device to end up in a reset loop, where each attempt to
         * recover from reset triggers another reset a few seconds
         * later.
         */
        pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
        if ( ! pfn ) {
                DBGC ( gve, "GVE %p skipping reset\n", gve );
                return 0;
        }

        /* Clear admin queue page frame number */
        writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN );
        wmb();

        /* Wait for device to reset */
        for ( i = 0 ; i < GVE_RESET_MAX_WAIT_MS ; i++ ) {

                /* Delay */
                mdelay ( 1 );

                /* Check for reset completion */
                pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
                if ( ! pfn )
                        return 0;
        }

        DBGC ( gve, "GVE %p reset timed out (PFN %#08x devstat %#08x)\n",
               gve, bswap_32 ( pfn ),
               bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) );
        return -ETIMEDOUT;
}

/******************************************************************************
 *
 * Admin queue
 *
 ******************************************************************************
 */

/**
 * Get operating mode name (for debugging)
 *
 * @v mode              Operating mode
 * @ret name            Mode name
 */
static inline const char * gve_mode_name ( unsigned int mode ) {
        static char buf[ 8 /* "XXX-XXX" + NUL */ ];

        snprintf ( buf, sizeof ( buf ), "%s-%s",
                   ( ( mode & GVE_MODE_DQO ) ? "DQO" : "GQI" ),
                   ( ( mode & GVE_MODE_QPL ) ? "QPL" : "RDA" ) );
        return buf;
}

/**
 * Allocate admin queue
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_admin_alloc ( struct gve_nic *gve ) {
        struct dma_device *dma = gve->dma;
        struct gve_admin *admin = &gve->admin;
        struct gve_scratch *scratch = &gve->scratch;
        size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
        size_t scratch_len = sizeof ( *scratch->buf );
        int rc;

        /* Allocate admin queue */
        admin->cmd = dma_alloc ( dma, &admin->map, admin_len, GVE_ALIGN );
        if ( ! admin->cmd ) {
                rc = -ENOMEM;
                goto err_admin;
        }

        /* Allocate scratch buffer */
        scratch->buf = dma_alloc ( dma, &scratch->map, scratch_len, GVE_ALIGN );
        if ( ! scratch->buf ) {
                rc = -ENOMEM;
                goto err_scratch;
        }

        DBGC ( gve, "GVE %p AQ at [%08lx,%08lx) scratch [%08lx,%08lx)\n",
               gve, virt_to_phys ( admin->cmd ),
               ( virt_to_phys ( admin->cmd ) + admin_len ),
               virt_to_phys ( scratch->buf ),
               ( virt_to_phys ( scratch->buf ) + scratch_len ) );
        return 0;

        dma_free ( &scratch->map, scratch->buf, scratch_len );
 err_scratch:
        dma_free ( &admin->map, admin->cmd, admin_len );
 err_admin:
        return rc;
}

/**
 * Free admin queue
 *
 * @v gve               GVE device
 */
static void gve_admin_free ( struct gve_nic *gve ) {
        struct gve_admin *admin = &gve->admin;
        struct gve_scratch *scratch = &gve->scratch;
        size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
        size_t scratch_len = sizeof ( *scratch->buf );

        /* Free scratch buffer */
        dma_free ( &scratch->map, scratch->buf, scratch_len );

        /* Free admin queue */
        dma_free ( &admin->map, admin->cmd, admin_len );
}

/**
 * Enable admin queue
 *
 * @v gve               GVE device
 */
static void gve_admin_enable ( struct gve_nic *gve ) {
        struct gve_admin *admin = &gve->admin;
        size_t admin_len = ( GVE_ADMIN_COUNT * sizeof ( admin->cmd[0] ) );
        physaddr_t base;

        /* Reset queue */
        admin->prod = 0;

        /* Program queue addresses and capabilities */
        base = dma ( &admin->map, admin->cmd );
        writel ( bswap_32 ( base / GVE_PAGE_SIZE ),
                 gve->cfg + GVE_CFG_ADMIN_PFN );
        writel ( bswap_32 ( base & 0xffffffffUL ),
                 gve->cfg + GVE_CFG_ADMIN_BASE_LO );
        if ( sizeof ( base ) > sizeof ( uint32_t ) ) {
                writel ( bswap_32 ( ( ( uint64_t ) base ) >> 32 ),
                         gve->cfg + GVE_CFG_ADMIN_BASE_HI );
        } else {
                writel ( 0, gve->cfg + GVE_CFG_ADMIN_BASE_HI );
        }
        writel ( bswap_16 ( admin_len ), gve->cfg + GVE_CFG_ADMIN_LEN );
        writel ( bswap_32 ( GVE_CFG_DRVSTAT_RUN ), gve->cfg + GVE_CFG_DRVSTAT );
}

/**
 * Get next available admin queue command slot
 *
 * @v gve               GVE device
 * @ret cmd             Admin queue command
 */
static union gve_admin_command * gve_admin_command ( struct gve_nic *gve ) {
        struct gve_admin *admin = &gve->admin;
        union gve_admin_command *cmd;
        unsigned int index;

        /* Get next command slot */
        index = admin->prod;
        cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ];

        /* Initialise request */
        memset ( cmd, 0, sizeof ( *cmd ) );

        return cmd;
}

/**
 * Wait for admin queue command to complete
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_admin_wait ( struct gve_nic *gve ) {
        struct gve_admin *admin = &gve->admin;
        uint32_t evt;
        uint32_t pfn;
        unsigned int i;

        /* Wait for any outstanding commands to complete */
        for ( i = 0 ; i < GVE_ADMIN_MAX_WAIT_MS ; i++ ) {

                /* Check event counter */
                rmb();
                evt = bswap_32 ( readl ( gve->cfg + GVE_CFG_ADMIN_EVT ) );
                if ( evt == admin->prod )
                        return 0;

                /* Check for device reset */
                pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
                if ( ! pfn )
                        break;

                /* Delay */
                mdelay ( 1 );
        }

        DBGC ( gve, "GVE %p AQ %#02x %s (completed %#02x, status %#08x)\n",
               gve, admin->prod, ( pfn ? "timed out" : "saw reset" ), evt,
               bswap_32 ( readl ( gve->cfg + GVE_CFG_DEVSTAT ) ) );
        return ( pfn ? -ETIMEDOUT : -ECONNRESET );
}

/**
 * Issue admin queue command
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_admin ( struct gve_nic *gve ) {
        struct gve_admin *admin = &gve->admin;
        union gve_admin_command *cmd;
        unsigned int index;
        uint32_t opcode;
        uint32_t status;
        int rc;

        /* Ensure admin queue is idle */
        if ( ( rc = gve_admin_wait ( gve ) ) != 0 )
                return rc;

        /* Get next command slot */
        index = admin->prod;
        cmd = &admin->cmd[ index % GVE_ADMIN_COUNT ];
        opcode = cmd->hdr.opcode;
        DBGC2 ( gve, "GVE %p AQ %#02x command %#04x request:\n",
                gve, index, opcode );
        DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) );

        /* Increment producer counter */
        admin->prod++;

        /* Ring doorbell */
        wmb();
        writel ( bswap_32 ( admin->prod ), gve->cfg + GVE_CFG_ADMIN_DB );

        /* Wait for command to complete */
        if ( ( rc = gve_admin_wait ( gve ) ) != 0 )
                return rc;

        /* Check command status */
        status = be32_to_cpu ( cmd->hdr.status );
        if ( status != GVE_ADMIN_STATUS_OK ) {
                rc = -EIO_ADMIN ( status );
                DBGC ( gve, "GVE %p AQ %#02x command %#04x failed: %#08x\n",
                       gve, index, opcode, status );
                DBGC_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
                DBGC ( gve, "GVE %p AQ error: %s\n", gve, strerror ( rc ) );
                return rc;
        }

        DBGC2 ( gve, "GVE %p AQ %#02x command %#04x result:\n",
                gve, index, opcode );
        DBGC2_HDA ( gve, 0, cmd, sizeof ( *cmd ) );
        return 0;
}

/**
 * Issue simple admin queue command
 *
 * @v gve               GVE device
 * @v opcode            Operation code
 * @v id                ID parameter (or zero if not applicable)
 * @ret rc              Return status code
 *
 * Several admin queue commands take either an empty parameter list or
 * a single 32-bit ID parameter.
 */
static int gve_admin_simple ( struct gve_nic *gve, unsigned int opcode,
                              unsigned int id ) {
        union gve_admin_command *cmd;
        int rc;

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = opcode;
        cmd->simple.id = cpu_to_be32 ( id );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Get device descriptor
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_describe ( struct gve_nic *gve ) {
        struct net_device *netdev = gve->netdev;
        struct gve_device_descriptor *desc = &gve->scratch.buf->desc;
        union gve_admin_command *cmd;
        struct gve_option *opt;
        unsigned int count;
        unsigned int id;
        size_t offset;
        size_t max;
        size_t len;
        int rc;

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = GVE_ADMIN_DESCRIBE;
        cmd->desc.addr = cpu_to_be64 ( dma ( &gve->scratch.map, desc ) );
        cmd->desc.ver = cpu_to_be32 ( GVE_ADMIN_DESCRIBE_VER );
        cmd->desc.len = cpu_to_be32 ( sizeof ( *desc ) );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;
        DBGC2 ( gve, "GVE %p device descriptor:\n", gve );
        DBGC2_HDA ( gve, 0, desc, sizeof ( *desc ) );

        /* Extract queue parameters */
        gve->events.count = be16_to_cpu ( desc->counters );
        gve->tx.count = be16_to_cpu ( desc->tx_count );
        gve->rx.count = be16_to_cpu ( desc->rx_count );
        DBGC ( gve, "GVE %p using %d TX, %d RX, %d events\n",
               gve, gve->tx.count, gve->rx.count, gve->events.count );

        /* Extract network parameters */
        build_assert ( sizeof ( desc->mac ) == ETH_ALEN );
        memcpy ( netdev->hw_addr, &desc->mac, sizeof ( desc->mac ) );
        netdev->mtu = be16_to_cpu ( desc->mtu );
        netdev->max_pkt_len = ( netdev->mtu + ETH_HLEN );
        DBGC ( gve, "GVE %p MAC %s (\"%s\") MTU %zd\n",
               gve, eth_ntoa ( netdev->hw_addr ),
               inet_ntoa ( desc->mac.in ), netdev->mtu );

        /* Parse options */
        count = be16_to_cpu ( desc->opt_count );
        max = be16_to_cpu ( desc->len );
        gve->options = 0;
        for ( offset = offsetof ( typeof ( *desc ), opts ) ; count ;
              count--, offset += len ) {

                /* Check space for option header */
                if ( ( offset + sizeof ( *opt ) ) > max ) {
                        DBGC ( gve, "GVE %p underlength option at +%#02zx:\n",
                               gve, offset );
                        DBGC_HDA ( gve, 0, desc, sizeof ( *desc ) );
                        return -EINVAL;
                }
                opt = ( ( ( void * ) desc ) + offset );

                /* Check space for option body */
                len = ( sizeof ( *opt ) + be16_to_cpu ( opt->len ) );
                if ( ( offset + len ) > max ) {
                        DBGC ( gve, "GVE %p malformed option at +%#02zx:\n",
                               gve, offset );
                        DBGC_HDA ( gve, 0, desc, sizeof ( *desc ) );
                        return -EINVAL;
                }

                /* Record option as supported */
                id = be16_to_cpu ( opt->id );
                if ( id < ( 8 * sizeof ( gve->options ) ) )
                        gve->options |= ( 1 << id );
        }
        DBGC ( gve, "GVE %p supports options %#08x\n", gve, gve->options );

        /* Select preferred operating mode */
        if ( gve->options & ( 1 << GVE_OPT_GQI_QPL ) ) {
                /* GQI-QPL: in-order queues, queue page list addressing */
                gve->mode = GVE_MODE_QPL;
        } else if ( gve->options & ( 1 << GVE_OPT_GQI_RDA ) ) {
                /* GQI-RDA: in-order queues, raw DMA addressing */
                gve->mode = 0;
        } else if ( gve->options & ( 1 << GVE_OPT_DQO_QPL ) ) {
                /* DQO-QPL: out-of-order queues, queue page list addressing */
                gve->mode = ( GVE_MODE_DQO | GVE_MODE_QPL );
        } else if ( gve->options & ( 1 << GVE_OPT_DQO_RDA ) ) {
                /* DQO-RDA: out-of-order queues, raw DMA addressing */
                gve->mode = GVE_MODE_DQO;
        } else {
                /* No options matched: assume the original GQI-QPL mode */
                gve->mode = GVE_MODE_QPL;
        }
        DBGC ( gve, "GVE %p using %s mode\n",
               gve, gve_mode_name ( gve->mode ) );

        return 0;
}

/**
 * Configure device resources
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_configure ( struct gve_nic *gve ) {
        struct gve_events *events = &gve->events;
        struct gve_irqs *irqs = &gve->irqs;
        union gve_admin_command *cmd;
        uint32_t doorbell;
        unsigned int db_off;
        unsigned int i;
        int rc;

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = GVE_ADMIN_CONFIGURE;
        cmd->conf.events =
                cpu_to_be64 ( dma ( &events->map, events->event ) );
        cmd->conf.irqs =
                cpu_to_be64 ( dma ( &irqs->map, irqs->irq ) );
        cmd->conf.num_events = cpu_to_be32 ( events->count );
        cmd->conf.num_irqs = cpu_to_be32 ( GVE_IRQ_COUNT );
        cmd->conf.irq_stride = cpu_to_be32 ( sizeof ( irqs->irq[0] ) );
        cmd->conf.format = GVE_FORMAT ( gve->mode );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        /* Disable all interrupts */
        doorbell = ( ( gve->mode & GVE_MODE_DQO ) ?
                     0 : bswap_32 ( GVE_GQI_IRQ_DISABLE ) );
        for ( i = 0 ; i < GVE_IRQ_COUNT ; i++ ) {
                db_off = ( be32_to_cpu ( irqs->irq[i].db_idx ) *
                           sizeof ( uint32_t ) );
                DBGC ( gve, "GVE %p IRQ %d doorbell +%#04x\n", gve, i, db_off );
                irqs->db[i] = ( gve->db + db_off );
                writel ( doorbell, irqs->db[i] );
        }

        return 0;
}

/**
 * Deconfigure device resources
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_deconfigure ( struct gve_nic *gve ) {
        int rc;

        /* Issue command (with meaningless ID) */
        if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_DECONFIGURE, 0 ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Register queue page list
 *
 * @v gve               GVE device
 * @v qpl               Queue page list
 * @ret rc              Return status code
 */
static int gve_register ( struct gve_nic *gve, struct gve_qpl *qpl ) {
        struct gve_pages *pages = &gve->scratch.buf->pages;
        union gve_admin_command *cmd;
        void *addr;
        unsigned int i;
        int rc;

        /* Do nothing if using raw DMA addressing */
        if ( ! ( gve->mode & GVE_MODE_QPL ) )
                return 0;

        /* Build page address list */
        for ( i = 0 ; i < qpl->count ; i++ ) {
                addr = ( qpl->data + ( i * GVE_PAGE_SIZE ) );
                pages->addr[i] = cpu_to_be64 ( dma ( &qpl->map, addr ) );
        }

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = GVE_ADMIN_REGISTER;
        cmd->reg.id = cpu_to_be32 ( qpl->id );
        cmd->reg.count = cpu_to_be32 ( qpl->count );
        cmd->reg.addr = cpu_to_be64 ( dma ( &gve->scratch.map, pages ) );
        cmd->reg.size = cpu_to_be64 ( GVE_PAGE_SIZE );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Unregister page list
 *
 * @v gve               GVE device
 * @v qpl               Queue page list
 * @ret rc              Return status code
 */
static int gve_unregister ( struct gve_nic *gve, struct gve_qpl *qpl ) {
        int rc;

        /* Do nothing if using raw DMA addressing */
        if ( ! ( gve->mode & GVE_MODE_QPL ) )
                return 0;

        /* Issue command */
        if ( ( rc = gve_admin_simple ( gve, GVE_ADMIN_UNREGISTER,
                                       qpl->id ) ) != 0 ) {
                return rc;
        }

        return 0;
}

/**
 * Construct command to create transmit queue
 *
 * @v queue             Transmit queue
 * @v qpl               Queue page list ID
 * @v cmd               Admin queue command
 */
static void gve_create_tx_param ( struct gve_queue *queue, uint32_t qpl,
                                  union gve_admin_command *cmd ) {
        struct gve_admin_create_tx *create = &cmd->create_tx;
        const struct gve_queue_type *type = queue->type;

        /* Construct request parameters */
        create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) );
        create->desc =
                cpu_to_be64 ( dma ( &queue->desc_map, queue->desc.raw ) );
        create->qpl_id = cpu_to_be32 ( qpl );
        create->notify_id = cpu_to_be32 ( type->irq );
        create->desc_count = cpu_to_be16 ( queue->count );
        if ( queue->cmplt.raw ) {
                create->cmplt = cpu_to_be64 ( dma ( &queue->cmplt_map,
                                                    queue->cmplt.raw ) );
                create->cmplt_count = cpu_to_be16 ( queue->count );
        }
}

/**
 * Construct command to create receive queue
 *
 * @v queue             Receive queue
 * @v qpl               Queue page list ID
 * @v cmd               Admin queue command
 */
static void gve_create_rx_param ( struct gve_queue *queue, uint32_t qpl,
                                  union gve_admin_command *cmd ) {
        struct gve_admin_create_rx *create = &cmd->create_rx;
        const struct gve_queue_type *type = queue->type;

        /* Construct request parameters */
        create->notify_id = cpu_to_be32 ( type->irq );
        create->res = cpu_to_be64 ( dma ( &queue->res_map, queue->res ) );
        create->desc =
                cpu_to_be64 ( dma ( &queue->desc_map, queue->desc.raw ) );
        create->cmplt =
                cpu_to_be64 ( dma ( &queue->cmplt_map, queue->cmplt.raw ) );
        create->qpl_id = cpu_to_be32 ( qpl );
        create->desc_count = cpu_to_be16 ( queue->count );
        create->bufsz = cpu_to_be16 ( GVE_BUF_SIZE );
        create->cmplt_count = cpu_to_be16 ( queue->count );
}

/**
 * Create transmit or receive queue
 *
 * @v gve               GVE device
 * @v queue             Descriptor queue
 * @ret rc              Return status code
 */
static int gve_create_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        const struct gve_queue_type *type = queue->type;
        const struct gve_queue_stride *stride = &queue->stride;
        union gve_admin_command *cmd;
        struct gve_buffer *buf;
        unsigned int db_off;
        unsigned int evt_idx;
        unsigned int tag;
        unsigned int i;
        uint32_t qpl;
        int rc;

        /* Reset queue */
        queue->prod = 0;
        queue->cons = 0;
        queue->done = 0;
        memset ( queue->desc.raw, 0, ( queue->count * stride->desc ) );
        memset ( queue->cmplt.raw, 0, ( queue->count * stride->cmplt ) );
        for ( i = 0 ; i < queue->fill ; i++ )
                queue->tag[i] = i;

        /* Pre-populate descriptor offsets for in-order queues */
        if ( ! ( gve->mode & GVE_MODE_DQO ) ) {
                buf = ( queue->desc.raw + stride->desc - sizeof ( *buf ) );
                for ( i = 0 ; i < queue->count ; i++ ) {
                        tag = ( i & ( queue->fill - 1 ) );
                        buf->addr = cpu_to_be64 ( gve_address ( queue, tag ) );
                        buf = ( ( ( void * ) buf ) + stride->desc );
                }
        }

        /* Construct request */
        cmd = gve_admin_command ( gve );
        cmd->hdr.opcode = type->create;
        qpl = ( ( gve->mode & GVE_MODE_QPL ) ? type->qpl : GVE_RAW_QPL );
        type->param ( queue, qpl, cmd );

        /* Issue command */
        if ( ( rc = gve_admin ( gve ) ) != 0 )
                return rc;

        /* Record indices */
        db_off = ( be32_to_cpu ( queue->res->db_idx ) * sizeof ( uint32_t ) );
        evt_idx = be32_to_cpu ( queue->res->evt_idx );
        DBGC ( gve, "GVE %p %s doorbell +%#04x event counter %d\n",
               gve, type->name, db_off, evt_idx );
        queue->db = ( gve->db + db_off );
        assert ( evt_idx < gve->events.count );
        queue->event = &gve->events.event[evt_idx];
        assert ( queue->event->count == 0 );

        /* Unmask dummy interrupt */
        pci_msix_unmask ( &gve->msix, type->irq );

        /* Rearm queue interrupt if applicable */
        if ( gve->mode & GVE_MODE_DQO )
                writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[type->irq] );

        return 0;
}

/**
 * Destroy transmit or receive queue
 *
 * @v gve               GVE device
 * @v queue             Descriptor queue
 * @ret rc              Return status code
 */
static int gve_destroy_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        const struct gve_queue_type *type = queue->type;
        int rc;

        /* Mask dummy interrupt */
        pci_msix_mask ( &gve->msix, type->irq );

        /* Issue command */
        if ( ( rc = gve_admin_simple ( gve, type->destroy, 0 ) ) != 0 )
                return rc;

        return 0;
}

/******************************************************************************
 *
 * Network device interface
 *
 ******************************************************************************
 */

/**
 * Allocate shared queue resources
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_alloc_shared ( struct gve_nic *gve ) {
        struct dma_device *dma = gve->dma;
        struct gve_irqs *irqs = &gve->irqs;
        struct gve_events *events = &gve->events;
        size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
        size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) );
        int rc;

        /* Allocate interrupt channels */
        irqs->irq = dma_alloc ( dma, &irqs->map, irqs_len, GVE_ALIGN );
        if ( ! irqs->irq ) {
                rc = -ENOMEM;
                goto err_irqs;
        }
        DBGC ( gve, "GVE %p IRQs at [%08lx,%08lx)\n",
               gve, virt_to_phys ( irqs->irq ),
               ( virt_to_phys ( irqs->irq ) + irqs_len ) );

        /* Allocate event counters */
        events->event = dma_alloc ( dma, &events->map, events_len, GVE_ALIGN );
        if ( ! events->event ) {
                rc = -ENOMEM;
                goto err_events;
        }
        DBGC ( gve, "GVE %p events at [%08lx,%08lx)\n",
               gve, virt_to_phys ( events->event ),
               ( virt_to_phys ( events->event ) + events_len ) );

        return 0;

        dma_free ( &events->map, events->event, events_len );
 err_events:
        dma_free ( &irqs->map, irqs->irq, irqs_len );
 err_irqs:
        return rc;
}

/**
 * Free shared queue resources
 *
 * @v gve               GVE device
 */
static void gve_free_shared ( struct gve_nic *gve ) {
        struct gve_irqs *irqs = &gve->irqs;
        struct gve_events *events = &gve->events;
        size_t irqs_len = ( GVE_IRQ_COUNT * sizeof ( irqs->irq[0] ) );
        size_t events_len = ( gve->events.count * sizeof ( events->event[0] ) );

        /* Free event counters */
        dma_free ( &events->map, events->event, events_len );

        /* Free interrupt channels */
        dma_free ( &irqs->map, irqs->irq, irqs_len );
}

/**
 * Allocate queue page list
 *
 * @v gve               GVE device
 * @v qpl               Queue page list
 * @v id                Queue page list ID
 * @v buffers           Number of data buffers
 * @ret rc              Return status code
 */
static int gve_alloc_qpl ( struct gve_nic *gve, struct gve_qpl *qpl,
                           uint32_t id, unsigned int buffers ) {
        size_t len;

        /* Record ID */
        qpl->id = id;

        /* Calculate number of pages required */
        build_assert ( GVE_BUF_SIZE <= GVE_PAGE_SIZE );
        qpl->count = ( ( buffers + GVE_BUF_PER_PAGE - 1 ) / GVE_BUF_PER_PAGE );
        assert ( qpl->count <= GVE_QPL_MAX );

        /* Allocate pages (as a single block) */
        len = ( qpl->count * GVE_PAGE_SIZE );
        qpl->data = dma_umalloc ( gve->dma, &qpl->map, len, GVE_ALIGN );
        if ( ! qpl->data )
                return -ENOMEM;
        qpl->base = ( ( gve->mode == GVE_MODE_QPL ) ?
                      0 : dma ( &qpl->map, qpl->data ) );

        DBGC ( gve, "GVE %p QPL %#08x at [%08lx,%08lx)\n",
               gve, qpl->id, virt_to_phys ( qpl->data ),
               ( virt_to_phys ( qpl->data ) + len ) );
        return 0;
}

/**
 * Free queue page list
 *
 * @v gve               GVE device
 * @v qpl               Queue page list
 */
static void gve_free_qpl ( struct gve_nic *nic __unused,
                           struct gve_qpl *qpl ) {
        size_t len = ( qpl->count * GVE_PAGE_SIZE );

        /* Free pages */
        dma_ufree ( &qpl->map, qpl->data, len );
}

/**
 * Calculate next receive sequence number
 *
 * @v seq               Current sequence number, or zero to start sequence
 * @ret next            Next sequence number
 */
static inline __attribute__ (( always_inline )) unsigned int
gve_next ( unsigned int seq ) {

        /* The receive completion sequence number is a modulo 7
         * counter that cycles through the non-zero three-bit values 1
         * to 7 inclusive.
         *
         * Since 7 is coprime to 2^n, this ensures that the sequence
         * number changes each time that a new completion is written
         * to memory.
         *
         * Since the counter takes only non-zero values, this ensures
         * that the sequence number changes whenever a new completion
         * is first written to a zero-initialised completion ring.
         */
        seq = ( ( seq + 1 ) & GVE_GQI_RX_SEQ_MASK );
        return ( seq ? seq : 1 );
}

/**
 * Allocate descriptor queue
 *
 * @v gve               GVE device
 * @v queue             Descriptor queue
 * @ret rc              Return status code
 */
static int gve_alloc_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        const struct gve_queue_type *type = queue->type;
        struct gve_queue_stride *stride = &queue->stride;
        struct dma_device *dma = gve->dma;
        size_t desc_len;
        size_t cmplt_len;
        size_t res_len;
        int rc;

        /* Sanity checks */
        if ( ( queue->count == 0 ) ||
             ( queue->count & ( queue->count - 1 ) ) ) {
                DBGC ( gve, "GVE %p %s invalid queue size %d\n",
                       gve, type->name, queue->count );
                rc = -EINVAL;
                goto err_sanity;
        }

        /* Set queue strides and calculate total lengths */
        *stride = ( ( gve->mode & GVE_MODE_DQO ) ?
                    type->stride.dqo : type->stride.gqi );
        desc_len = ( queue->count * stride->desc );
        cmplt_len = ( queue->count * stride->cmplt );
        res_len = sizeof ( *queue->res );

        /* Calculate maximum fill level */
        assert ( ( type->fill & ( type->fill - 1 ) ) == 0 );
        queue->fill = type->fill;
        if ( queue->fill > queue->count )
                queue->fill = queue->count;
        DBGC ( gve, "GVE %p %s using QPL %#08x with %d/%d descriptors\n",
               gve, type->name, type->qpl, queue->fill, queue->count );

        /* Allocate queue page list */
        if ( ( rc = gve_alloc_qpl ( gve, &queue->qpl, type->qpl,
                                    queue->fill ) ) != 0 )
                goto err_qpl;

        /* Allocate descriptors */
        queue->desc.raw = dma_umalloc ( dma, &queue->desc_map, desc_len,
                                        GVE_ALIGN );
        if ( ! queue->desc.raw ) {
                rc = -ENOMEM;
                goto err_desc;
        }
        DBGC ( gve, "GVE %p %s descriptors at [%08lx,%08lx)\n",
               gve, type->name, virt_to_phys ( queue->desc.raw ),
               ( virt_to_phys ( queue->desc.raw ) + desc_len ) );

        /* Allocate completions */
        if ( cmplt_len ) {
                queue->cmplt.raw = dma_umalloc ( dma, &queue->cmplt_map,
                                                 cmplt_len, GVE_ALIGN );
                if ( ! queue->cmplt.raw ) {
                        rc = -ENOMEM;
                        goto err_cmplt;
                }
                DBGC ( gve, "GVE %p %s completions at [%08lx,%08lx)\n",
                       gve, type->name, virt_to_phys ( queue->cmplt.raw ),
                       ( virt_to_phys ( queue->cmplt.raw ) + cmplt_len ) );
        }

        /* Allocate queue resources */
        queue->res = dma_alloc ( dma, &queue->res_map, res_len, GVE_ALIGN );
        if ( ! queue->res ) {
                rc = -ENOMEM;
                goto err_res;
        }
        memset ( queue->res, 0, res_len );

        return 0;

        dma_free ( &queue->res_map, queue->res, res_len );
 err_res:
        if ( cmplt_len )
                dma_ufree ( &queue->cmplt_map, queue->cmplt.raw, cmplt_len );
 err_cmplt:
        dma_ufree ( &queue->desc_map, queue->desc.raw, desc_len );
 err_desc:
        gve_free_qpl ( gve, &queue->qpl );
 err_qpl:
 err_sanity:
        return rc;
}

/**
 * Free descriptor queue
 *
 * @v gve               GVE device
 * @v queue             Descriptor queue
 */
static void gve_free_queue ( struct gve_nic *gve, struct gve_queue *queue ) {
        const struct gve_queue_stride *stride = &queue->stride;
        size_t desc_len = ( queue->count * stride->desc );
        size_t cmplt_len = ( queue->count * stride->cmplt );
        size_t res_len = sizeof ( *queue->res );

        /* Free queue resources */
        dma_free ( &queue->res_map, queue->res, res_len );

        /* Free completions, if applicable */
        if ( cmplt_len )
                dma_ufree ( &queue->cmplt_map, queue->cmplt.raw, cmplt_len );

        /* Free descriptors */
        dma_ufree ( &queue->desc_map, queue->desc.raw, desc_len );

        /* Free queue page list */
        gve_free_qpl ( gve, &queue->qpl );
}

/**
 * Cancel any pending transmissions
 *
 * @v gve               GVE device
 */
static void gve_cancel_tx ( struct gve_nic *gve ) {
        struct net_device *netdev = gve->netdev;
        struct io_buffer *iobuf;
        unsigned int i;

        /* Cancel any pending transmissions */
        for ( i = 0 ; i < ( sizeof ( gve->tx_iobuf ) /
                            sizeof ( gve->tx_iobuf[0] ) ) ; i++ ) {
                iobuf = gve->tx_iobuf[i];
                gve->tx_iobuf[i] = NULL;
                if ( iobuf )
                        netdev_tx_complete_err ( netdev, iobuf, -ECANCELED );
        }
}

/**
 * Start up device
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_start ( struct gve_nic *gve ) {
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;
        int rc;

        /* Cancel any pending transmissions */
        gve_cancel_tx ( gve );

        /* Reset receive sequence */
        gve->seq = gve_next ( 0 );

        /* Configure device resources */
        if ( ( rc = gve_configure ( gve ) ) != 0 )
                goto err_configure;

        /* Register transmit queue page list */
        if ( ( rc = gve_register ( gve, &tx->qpl ) ) != 0 )
                goto err_register_tx;

        /* Register receive queue page list */
        if ( ( rc = gve_register ( gve, &rx->qpl ) ) != 0 )
                goto err_register_rx;

        /* Create transmit queue */
        if ( ( rc = gve_create_queue ( gve, tx ) ) != 0 )
                goto err_create_tx;

        /* Create receive queue */
        if ( ( rc = gve_create_queue ( gve, rx ) ) != 0 )
                goto err_create_rx;

        return 0;

        gve_destroy_queue ( gve, rx );
 err_create_rx:
        gve_destroy_queue ( gve, tx );
 err_create_tx:
        gve_unregister ( gve, &rx->qpl );
 err_register_rx:
        gve_unregister ( gve, &tx->qpl );
 err_register_tx:
        gve_deconfigure ( gve );
 err_configure:
        return rc;
}

/**
 * Stop device
 *
 * @v gve               GVE device
 */
static void gve_stop ( struct gve_nic *gve ) {
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;

        /* Destroy queues */
        gve_destroy_queue ( gve, rx );
        gve_destroy_queue ( gve, tx );

        /* Unregister page lists */
        gve_unregister ( gve, &rx->qpl );
        gve_unregister ( gve, &tx->qpl );

        /* Deconfigure device */
        gve_deconfigure ( gve );
}

/**
 * Device startup process
 *
 * @v gve               GVE device
 */
static void gve_startup ( struct gve_nic *gve ) {
        struct net_device *netdev = gve->netdev;
        int rc;

        /* Reset device */
        if ( ( rc = gve_reset ( gve ) ) != 0 )
                goto err_reset;

        /* Enable admin queue */
        gve_admin_enable ( gve );

        /* Start device */
        if ( ( rc = gve_start ( gve ) ) != 0 )
                goto err_start;

        /* Reset retry count */
        gve->retries = 0;

        /* (Ab)use link status to report startup status */
        netdev_link_up ( netdev );

        return;

        gve_stop ( gve );
 err_start:
 err_reset:
        DBGC ( gve, "GVE %p startup failed: %s\n", gve, strerror ( rc ) );
        netdev_link_err ( netdev, rc );
        if ( gve->retries++ < GVE_RESET_MAX_RETRY )
                process_add ( &gve->startup );
}

/**
 * Trigger startup process
 *
 * @v gve               GVE device
 */
static void gve_restart ( struct gve_nic *gve ) {
        struct net_device *netdev = gve->netdev;

        /* Mark link down to inhibit polling and transmit activity */
        netdev_link_down ( netdev );

        /* Schedule startup process */
        process_add ( &gve->startup );
}

/**
 * Reset recovery watchdog
 *
 * @v timer             Reset recovery watchdog timer
 * @v over              Failure indicator
 */
static void gve_watchdog ( struct retry_timer *timer, int over __unused ) {
        struct gve_nic *gve = container_of ( timer, struct gve_nic, watchdog );
        uint32_t activity;
        uint32_t pfn;
        int rc;

        /* Reschedule watchdog */
        start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT );

        /* Reset device (for test purposes) if applicable */
        if ( ( rc = inject_fault ( VM_MIGRATED_RATE ) ) != 0 ) {
                DBGC ( gve, "GVE %p synthesising host reset\n", gve );
                writel ( 0, gve->cfg + GVE_CFG_ADMIN_PFN );
        }

        /* Check for activity since last timer invocation */
        activity = ( gve->tx.cons + gve->rx.cons );
        if ( activity != gve->activity ) {
                gve->activity = activity;
                return;
        }

        /* Check for reset */
        pfn = readl ( gve->cfg + GVE_CFG_ADMIN_PFN );
        if ( pfn ) {
                DBGC2 ( gve, "GVE %p idle but not in reset\n", gve );
                return;
        }

        /* Schedule restart */
        DBGC ( gve, "GVE %p watchdog detected reset by host\n", gve );
        gve_restart ( gve );
}

/**
 * Open network device
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int gve_open ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;
        int rc;

        /* Allocate shared queue resources */
        if ( ( rc = gve_alloc_shared ( gve ) ) != 0 )
                goto err_alloc_shared;

        /* Allocate and prepopulate transmit queue */
        if ( ( rc = gve_alloc_queue ( gve, tx ) ) != 0 )
                goto err_alloc_tx;

        /* Allocate and prepopulate receive queue */
        if ( ( rc = gve_alloc_queue ( gve, rx ) ) != 0 )
                goto err_alloc_rx;

        /* Trigger startup */
        gve_restart ( gve );

        /* Start reset recovery watchdog timer */
        start_timer_fixed ( &gve->watchdog, GVE_WATCHDOG_TIMEOUT );

        return 0;

        gve_free_queue ( gve, rx );
 err_alloc_rx:
        gve_free_queue ( gve, tx );
 err_alloc_tx:
        gve_free_shared ( gve );
 err_alloc_shared:
        return rc;
}

/**
 * Close network device
 *
 * @v netdev            Network device
 */
static void gve_close ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_queue *rx = &gve->rx;

        /* Stop reset recovery timer */
        stop_timer ( &gve->watchdog );

        /* Terminate startup process */
        process_del ( &gve->startup );

        /* Stop and reset device */
        gve_stop ( gve );
        gve_reset ( gve );

        /* Cancel any pending transmissions */
        gve_cancel_tx ( gve );

        /* Free queues */
        gve_free_queue ( gve, rx );
        gve_free_queue ( gve, tx );

        /* Free shared queue resources */
        gve_free_shared ( gve );
}

/**
 * Transmit packet
 *
 * @v netdev            Network device
 * @v iobuf             I/O buffer
 * @ret rc              Return status code
 */
static int gve_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_gqi_tx_descriptor *gqi;
        struct gve_dqo_tx_descriptor *dqo;
        unsigned int count;
        unsigned int index;
        unsigned int tag;
        unsigned int chain;
        uint32_t doorbell;
        size_t frag_len;
        size_t offset;
        size_t next;
        size_t len;

        /* Do nothing if queues are not yet set up */
        if ( ! netdev_link_ok ( netdev ) )
                return -ENETDOWN;

        /* Defer packet if there is no space in the transmit ring */
        len = iob_len ( iobuf );
        count = ( ( len + GVE_BUF_SIZE - 1 ) / GVE_BUF_SIZE );
        if ( ( ( tx->prod - tx->cons ) + count ) > tx->fill ) {
                netdev_tx_defer ( netdev, iobuf );
                return 0;
        }

        /* Copy packet to queue pages and populate descriptors */
        for ( offset = 0, chain = 0 ; ; offset = next, chain = tag ) {

                /* Identify next available buffer */
                index = ( tx->prod++ & ( tx->count - 1 ) );
                tag = tx->tag[ index % GVE_TX_FILL ];

                /* Sanity check */
                assert ( gve->tx_iobuf[tag] == NULL );

                /* Copy packet fragment */
                frag_len = ( len - offset );
                if ( frag_len > GVE_BUF_SIZE )
                        frag_len = GVE_BUF_SIZE;
                memcpy ( gve_buffer ( tx, tag ),
                         ( iobuf->data + offset ), frag_len );
                next = ( offset + frag_len );

                /* Populate descriptor */
                if ( gve->mode & GVE_MODE_DQO ) {

                        /* Out-of-order descriptor */
                        dqo = &tx->desc.tx.dqo[index];
                        dqo->buf.addr =
                                cpu_to_le64 ( gve_address ( tx, tag ) );
                        if ( next == len ) {
                                dqo->type = ( GVE_DQO_TX_TYPE_PACKET |
                                              GVE_DQO_TX_TYPE_LAST );
                                dqo->tag.id = tag;
                                dqo->tag.count = count;
                        } else {
                                dqo->type = GVE_DQO_TX_TYPE_PACKET;
                                dqo->tag.id = 0;
                                dqo->tag.count = 0;
                        }
                        dqo->len = cpu_to_le16 ( frag_len );
                        gve->tx_chain[tag] = chain;

                } else {

                        /* In-order descriptor */
                        gqi = &tx->desc.tx.gqi[index];
                        if ( offset ) {
                                gqi->type = GVE_GQI_TX_TYPE_CONT;
                                gqi->count = 0;
                                gqi->total = 0;
                        } else {
                                gqi->type = GVE_GQI_TX_TYPE_START;
                                gqi->count = count;
                                gqi->total = cpu_to_be16 ( len );
                        }
                        gqi->len = cpu_to_be16 ( frag_len );

                }
                DBGC2 ( gve, "GVE %p TXD %#04x %#02x:%#02x len %#04zx/%#04zx "
                        "at %#08lx\n", gve, index, tag, count, frag_len, len,
                        gve_address ( tx, tag ) );

                /* Record I/O buffer against final descriptor */
                if ( next == len ) {
                        gve->tx_iobuf[tag] = iobuf;
                        break;
                }
        }
        assert ( ( tx->prod - tx->cons ) <= tx->fill );

        /* Ring doorbell */
        doorbell = tx->prod;
        if ( gve->mode & GVE_MODE_DQO ) {
                doorbell &= ( tx->count - 1 );
        } else {
                doorbell = bswap_32 ( doorbell );
        }
        wmb();
        writel ( doorbell, tx->db );

        return 0;
}

/**
 * Poll for completed transmissions
 *
 * @v netdev            Network device
 */
static void gve_poll_tx ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *tx = &gve->tx;
        struct gve_dqo_tx_completion *dqo;
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int gen;
        unsigned int bit;
        unsigned int tag;
        uint32_t count;

        /* Process transmit completions */
        if ( gve->mode & GVE_MODE_DQO ) {

                /* Out-of-order completions */
                while ( 1 ) {

                        /* Read next possible completion */
                        gen = ( tx->done & tx->count );
                        index = ( tx->done & ( tx->count - 1 ) );
                        dqo = &tx->cmplt.tx.dqo[index];

                        /* Check generation bit */
                        bit = ( dqo->flags & GVE_DQO_TXF_GEN );
                        if ( ( !! bit ) == ( !! gen ) )
                                break;
                        rmb();
                        tx->done++;

                        /* Ignore non-packet completions */
                        if ( ( ! ( dqo->flags & GVE_DQO_TXF_PKT ) ) ||
                             ( dqo->tag.count < 0 ) ) {
                                DBGC2 ( gve, "GVE %p TXC %#04x flags %#02x "
                                        "ignored\n", gve, index, dqo->flags );
                                continue;
                        }

                        /* Parse completion */
                        tag = dqo->tag.id;
                        count = dqo->tag.count;
                        iobuf = gve->tx_iobuf[tag];
                        gve->tx_iobuf[tag] = NULL;
                        assert ( iobuf != NULL );

                        /* Return completed descriptors to ring */
                        while ( count-- ) {
                                DBGC2 ( gve, "GVE %p TXC %#04x %#02x:%#02x "
                                        "complete\n", gve, index, tag,
                                        dqo->tag.count );
                                tx->tag[ tx->cons++ % GVE_TX_FILL ] = tag;
                                tag = gve->tx_chain[tag];
                        }

                        /* Hand off to network stack */
                        if ( iobuf )
                                netdev_tx_complete ( netdev, iobuf );
                }

        } else {

                /* Read event counter */
                count = be32_to_cpu ( tx->event->count );

                /* Process transmit completions */
                while ( count != tx->cons ) {
                        DBGC2 ( gve, "GVE %p TXC %#04x complete\n",
                                gve, tx->cons );
                        tag = ( tx->cons % GVE_TX_FILL );
                        iobuf = gve->tx_iobuf[tag];
                        gve->tx_iobuf[tag] = NULL;
                        tx->cons++;
                        if ( iobuf )
                                netdev_tx_complete ( netdev, iobuf );
                }
        }
}

/**
 * Poll for received packets
 *
 * @v netdev            Network device
 */
static void gve_poll_rx ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *rx = &gve->rx;
        struct gve_gqi_rx_completion *gqi;
        struct gve_dqo_rx_completion *dqo;
        struct io_buffer *iobuf;
        unsigned int index;
        unsigned int gen;
        unsigned int bit;
        unsigned int seq;
        unsigned int tag;
        uint32_t done;
        size_t total;
        size_t len;
        int rc;

        /* Process receive completions */
        done = rx->done;
        seq = gve->seq;
        total = 0;
        while ( 1 ) {

                /* Read next possible completion */
                rc = 0;
                gen = ( done & rx->count );
                index = ( done++ & ( rx->count - 1 ) );
                if ( gve->mode & GVE_MODE_DQO ) {

                        /* Out-of-order completion */
                        dqo = &rx->cmplt.rx.dqo[index];

                        /* Check generation bit */
                        bit = ( dqo->len & cpu_to_le16 ( GVE_DQO_RXL_GEN ) );
                        if ( ( !! bit ) == ( !! gen ) )
                                break;
                        rmb();

                        /* Parse completion */
                        len = ( le16_to_cpu ( dqo->len ) &
                                ( GVE_BUF_SIZE - 1 ) );
                        tag = dqo->tag;
                        DBGC2 ( gve, "GVE %p RXC %#04x %#02x:%#02x len %#04zx "
                                "at %#08zx\n", gve, index, tag, dqo->flags,
                                len, gve_offset ( rx, tag ) );

                        /* Accumulate a complete packet */
                        if ( dqo->status & GVE_DQO_RXS_ERROR ) {
                                rc = -EIO;
                                total = 0;
                        } else {
                                total += len;
                                if ( ! ( dqo->flags & GVE_DQO_RXF_LAST ) )
                                        continue;
                        }

                } else {

                        /* In-order completion */
                        gqi = &rx->cmplt.rx.gqi[index];

                        /* Check sequence number */
                        if ( ( gqi->seq & GVE_GQI_RX_SEQ_MASK ) != seq )
                                break;
                        rmb();
                        seq = gve_next ( seq );

                        /* Parse completion */
                        len = be16_to_cpu ( gqi->len );
                        tag = ( index % GVE_RX_FILL );
                        DBGC2 ( gve, "GVE %p RXC %#04x %#02x:%#02x len %#04zx "
                                "at %#08zx\n", gve, index, gqi->seq,
                                gqi->flags, len, gve_offset ( rx, tag ) );

                        /* Accumulate a complete packet */
                        if ( gqi->flags & GVE_GQI_RXF_ERROR ) {
                                rc = -EIO;
                                total = 0;
                        } else {
                                total += len;
                                if ( gqi->flags & GVE_GQI_RXF_MORE )
                                        continue;
                        }
                        gve->seq = seq;
                }

                /* Allocate and populate I/O buffer */
                iobuf = ( total ? alloc_iob ( total ) : NULL );
                for ( ; rx->done != done ; rx->done++ ) {

                        /* Re-read completion and return tag to ring */
                        index = ( rx->done & ( rx->count - 1 ) );
                        if ( gve->mode & GVE_MODE_DQO ) {
                                dqo = &rx->cmplt.rx.dqo[index];
                                tag = dqo->tag;
                                len = ( le16_to_cpu ( dqo->len ) &
                                        ( GVE_BUF_SIZE - 1 ) );
                                rx->tag[ rx->cons++ % GVE_RX_FILL ] = tag;
                        } else {
                                gqi = &rx->cmplt.rx.gqi[index];
                                tag = ( index % GVE_RX_FILL );
                                len = be16_to_cpu ( gqi->len );
                                assert ( rx->cons == rx->done );
                                rx->cons++;
                        }

                        /* Copy data */
                        if ( iobuf ) {
                                memcpy ( iob_put ( iobuf, len ),
                                         gve_buffer ( rx, tag ), len );
                        }
                }
                assert ( ( iobuf == NULL ) || ( iob_len ( iobuf ) == total ) );
                total = 0;

                /* Hand off packet to network stack */
                if ( iobuf ) {
                        if ( ! ( gve->mode & GVE_MODE_DQO ) )
                                iob_pull ( iobuf, GVE_GQI_RX_PAD );
                        netdev_rx ( netdev, iobuf );
                } else {
                        netdev_rx_err ( netdev, NULL, ( rc ? rc : -ENOMEM ) );
                }
        }
}

/**
 * Refill receive queue
 *
 * @v netdev            Network device
 */
static void gve_refill_rx ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;
        struct gve_queue *rx = &gve->rx;
        struct gve_dqo_rx_descriptor *dqo;
        unsigned int refill;
        unsigned int index;
        unsigned int tag;
        uint32_t doorbell;

        /* Calculate refill quantity */
        doorbell = ( rx->cons + rx->fill );
        refill = ( doorbell - rx->prod );
        if ( ! refill )
                return;

        /* Refill ring */
        if ( gve->mode & GVE_MODE_DQO ) {

                /* Out-of-order descriptors */
                while ( refill-- ) {

                        /* Identify next available buffer */
                        index = ( rx->prod++ & ( rx->count - 1 ) );
                        tag = rx->tag[ index % GVE_RX_FILL ];

                        /* Populate descriptor */
                        dqo = &rx->desc.rx.dqo[index];
                        dqo->tag = tag;
                        dqo->buf.addr =
                                cpu_to_le64 ( gve_address ( rx, tag ) );
                        DBGC2 ( gve, "GVE %p RXD %#04x:%#02x at %#08llx\n",
                                gve, index, dqo->tag,
                                ( ( unsigned long long )
                                  le64_to_cpu ( dqo->buf.addr ) ) );
                }
                wmb();
                assert ( rx->prod == doorbell );

        } else {

                /* The in-order receive descriptors are prepopulated
                 * at the time of creating the receive queue (pointing
                 * to the preallocated queue pages).  Refilling is
                 * therefore just a case of ringing the doorbell if
                 * the device is not yet aware of any available
                 * descriptors.
                 */
                rx->prod += refill;
                assert ( rx->prod == doorbell );
                DBGC2 ( gve, "GVE %p RXD %#04x ready\n", gve, rx->prod );

                /* Doorbell is big-endian */
                doorbell = bswap_32 ( doorbell );
        }

        /* Ring doorbell */
        writel ( doorbell, rx->db );
}

/**
 * Poll for completed and received packets
 *
 * @v netdev            Network device
 */
static void gve_poll ( struct net_device *netdev ) {
        struct gve_nic *gve = netdev->priv;

        /* Do nothing if queues are not yet set up */
        if ( ! netdev_link_ok ( netdev ) )
                return;

        /* Poll for transmit completions */
        gve_poll_tx ( netdev );

        /* Poll for receive completions */
        gve_poll_rx ( netdev );

        /* Refill receive queue */
        gve_refill_rx ( netdev );

        /* Rearm queue interrupts if applicable */
        if ( gve->mode & GVE_MODE_DQO ) {
                writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[GVE_TX_IRQ] );
                writel ( GVE_DQO_IRQ_REARM, gve->irqs.db[GVE_RX_IRQ] );
        }
}

/** GVE network device operations */
static struct net_device_operations gve_operations = {
        .open           = gve_open,
        .close          = gve_close,
        .transmit       = gve_transmit,
        .poll           = gve_poll,
};

/******************************************************************************
 *
 * PCI interface
 *
 ******************************************************************************
 */

/** Transmit descriptor queue type */
static const struct gve_queue_type gve_tx_type = {
        .name = "TX",
        .param = gve_create_tx_param,
        .qpl = GVE_TX_QPL,
        .irq = GVE_TX_IRQ,
        .fill = GVE_TX_FILL,
        .stride = {
                .gqi = {
                        .desc = sizeof ( struct gve_gqi_tx_descriptor ),
                },
                .dqo = {
                        .desc = sizeof ( struct gve_dqo_tx_descriptor ),
                        .cmplt = sizeof ( struct gve_dqo_tx_completion ),
                },
        },
        .create = GVE_ADMIN_CREATE_TX,
        .destroy = GVE_ADMIN_DESTROY_TX,
};

/** Receive descriptor queue type */
static const struct gve_queue_type gve_rx_type = {
        .name = "RX",
        .param = gve_create_rx_param,
        .qpl = GVE_RX_QPL,
        .irq = GVE_RX_IRQ,
        .fill = GVE_RX_FILL,
        .stride = {
                .gqi = {
                        .desc = sizeof ( struct gve_gqi_rx_descriptor ),
                        .cmplt = sizeof ( struct gve_gqi_rx_completion ),
                },
                .dqo = {
                        .desc = sizeof ( struct gve_dqo_rx_descriptor ),
                        .cmplt = sizeof ( struct gve_dqo_rx_completion ),
                },
        },
        .create = GVE_ADMIN_CREATE_RX,
        .destroy = GVE_ADMIN_DESTROY_RX,
};

/**
 * Set up admin queue and get device description
 *
 * @v gve               GVE device
 * @ret rc              Return status code
 */
static int gve_setup ( struct gve_nic *gve ) {
        unsigned int i;
        int rc;

        /* Attempt several times, since the device may decide to add
         * in a few spurious resets.
         */
        for ( i = 0 ; i < GVE_RESET_MAX_RETRY ; i++ ) {

                /* Reset device */
                if ( ( rc = gve_reset ( gve ) ) != 0 )
                        continue;

                /* Enable admin queue */
                gve_admin_enable ( gve );

                /* Fetch MAC address */
                if ( ( rc = gve_describe ( gve ) ) != 0 )
                        continue;

                /* Success */
                return 0;
        }

        DBGC ( gve, "GVE %p failed to get device description: %s\n",
               gve, strerror ( rc ) );
        return rc;
}

/** Device startup process descriptor */
static struct process_descriptor gve_startup_desc =
        PROC_DESC_ONCE ( struct gve_nic, startup, gve_startup );

/**
 * Probe PCI device
 *
 * @v pci               PCI device
 * @ret rc              Return status code
 */
static int gve_probe ( struct pci_device *pci ) {
        struct net_device *netdev;
        struct gve_nic *gve;
        unsigned long cfg_start;
        unsigned long db_start;
        unsigned long db_size;
        int rc;

        /* Allocate and initialise net device */
        netdev = alloc_etherdev ( sizeof ( *gve ) );
        if ( ! netdev ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        netdev_init ( netdev, &gve_operations );
        gve = netdev->priv;
        pci_set_drvdata ( pci, netdev );
        netdev->dev = &pci->dev;
        memset ( gve, 0, sizeof ( *gve ) );
        gve->netdev = netdev;
        gve->tx.type = &gve_tx_type;
        gve->rx.type = &gve_rx_type;
        gve->tx.tag = gve->tx_tag;
        gve->rx.tag = gve->rx_tag;
        process_init_stopped ( &gve->startup, &gve_startup_desc,
                               &netdev->refcnt );
        timer_init ( &gve->watchdog, gve_watchdog, &netdev->refcnt );

        /* Fix up PCI device */
        adjust_pci_device ( pci );

        /* Check PCI revision */
        pci_read_config_byte ( pci, PCI_REVISION, &gve->revision );
        DBGC ( gve, "GVE %p is revision %#02x\n", gve, gve->revision );

        /* Map configuration registers */
        cfg_start = pci_bar_start ( pci, GVE_CFG_BAR );
        gve->cfg = pci_ioremap ( pci, cfg_start, GVE_CFG_SIZE );
        if ( ! gve->cfg ) {
                rc = -ENODEV;
                goto err_cfg;
        }

        /* Map doorbell registers */
        db_start = pci_bar_start ( pci, GVE_DB_BAR );
        db_size = pci_bar_size ( pci, GVE_DB_BAR );
        gve->db = pci_ioremap ( pci, db_start, db_size );
        if ( ! gve->db ) {
                rc = -ENODEV;
                goto err_db;
        }

        /* Configure DMA */
        gve->dma = &pci->dma;
        dma_set_mask_64bit ( gve->dma );
        assert ( netdev->dma == NULL );

        /* Configure dummy MSI-X interrupt */
        if ( ( rc = pci_msix_enable ( pci, &gve->msix ) ) != 0 )
                goto err_msix;

        /* Allocate admin queue */
        if ( ( rc = gve_admin_alloc ( gve ) ) != 0 )
                goto err_admin;

        /* Set up the device */
        if ( ( rc = gve_setup ( gve ) ) != 0 )
                goto err_setup;

        /* Register network device */
        if ( ( rc = register_netdev ( netdev ) ) != 0 )
                goto err_register_netdev;

        return 0;

        unregister_netdev ( netdev );
 err_register_netdev:
 err_setup:
        gve_reset ( gve );
        gve_admin_free ( gve );
 err_admin:
        pci_msix_disable ( pci, &gve->msix );
 err_msix:
        iounmap ( gve->db );
 err_db:
        iounmap ( gve->cfg );
 err_cfg:
        netdev_nullify ( netdev );
        netdev_put ( netdev );
 err_alloc:
        return rc;
}

/**
 * Remove PCI device
 *
 * @v pci               PCI device
 */
static void gve_remove ( struct pci_device *pci ) {
        struct net_device *netdev = pci_get_drvdata ( pci );
        struct gve_nic *gve = netdev->priv;

        /* Unregister network device */
        unregister_netdev ( netdev );

        /* Reset device */
        gve_reset ( gve );

        /* Free admin queue */
        gve_admin_free ( gve );

        /* Disable dummy MSI-X interrupt */
        pci_msix_disable ( pci, &gve->msix );

        /* Unmap registers */
        iounmap ( gve->db );
        iounmap ( gve->cfg );

        /* Free network device */
        netdev_nullify ( netdev );
        netdev_put ( netdev );
}

/** GVE PCI device IDs */
static struct pci_device_id gve_nics[] = {
        PCI_ROM ( 0x1ae0, 0x0042, "gve", "gVNIC", 0 ),
};

/** GVE PCI driver */
struct pci_driver gve_driver __pci_driver = {
        .ids = gve_nics,
        .id_count = ( sizeof ( gve_nics ) / sizeof ( gve_nics[0] ) ),
        .probe = gve_probe,
        .remove = gve_remove,
};