gve.h

Go to the documentation of this file.

#ifndef _GVE_H
#define _GVE_H

/** @file
 *
 * Google Virtual Ethernet network driver
 *
 * The Google Virtual Ethernet NIC (GVE or gVNIC) is found only in
 * Google Cloud instances.  There is essentially zero documentation
 * available beyond the mostly uncommented source code in the Linux
 * kernel.
 */

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

#include <stdint.h>
#include <ipxe/dma.h>
#include <ipxe/pci.h>
#include <ipxe/in.h>
#include <ipxe/uaccess.h>
#include <ipxe/process.h>
#include <ipxe/retry.h>

struct gve_nic;

/**
 * A Google Cloud MAC address
 *
 * Google Cloud locally assigned MAC addresses encode the local IPv4
 * address in the trailing 32 bits, presumably as a performance
 * optimisation to allow ARP resolution to be skipped by a suitably
 * aware network stack.
 */
struct google_mac {
        /** Reserved */
        uint8_t reserved[2];
        /** Local IPv4 address */
        struct in_addr in;
} __attribute__ (( packed ));

/** Page size */
#define GVE_PAGE_SIZE 0x1000

/**
 * Address alignment
 *
 * All DMA data structure base addresses seem to need to be aligned to
 * a page boundary.  (This is not documented anywhere, but is inferred
 * from existing source code and experimentation.)
 */
#define GVE_ALIGN GVE_PAGE_SIZE

/** Configuration BAR */
#define GVE_CFG_BAR PCI_BASE_ADDRESS_0

/**
 * Configuration BAR size
 *
 * All registers within the configuration BAR are big-endian.
 */
#define GVE_CFG_SIZE 0x1000

/** Device status */
#define GVE_CFG_DEVSTAT 0x0000
#define GVE_CFG_DEVSTAT_RESET 0x00000010UL      /**< Device is reset */

/** Driver status */
#define GVE_CFG_DRVSTAT 0x0004
#define GVE_CFG_DRVSTAT_RUN 0x00000001UL        /**< Run admin queue */

/** Maximum time to wait for reset */
#define GVE_RESET_MAX_WAIT_MS 500

/** Admin queue page frame number (for older devices) */
#define GVE_CFG_ADMIN_PFN 0x0010

/** Admin queue doorbell */
#define GVE_CFG_ADMIN_DB 0x0014

/** Admin queue event counter */
#define GVE_CFG_ADMIN_EVT 0x0018

/** Driver version (8-bit register) */
#define GVE_CFG_VERSION 0x001f

/** Admin queue base address high 32 bits */
#define GVE_CFG_ADMIN_BASE_HI 0x0020

/** Admin queue base address low 32 bits */
#define GVE_CFG_ADMIN_BASE_LO 0x0024

/** Admin queue base address length (16-bit register) */
#define GVE_CFG_ADMIN_LEN 0x0028

/** Doorbell BAR */
#define GVE_DB_BAR PCI_BASE_ADDRESS_2

/**
 * Admin queue entry header
 *
 * All values within admin queue entries are big-endian.
 */
struct gve_admin_header {
        /** Reserved */
        uint8_t reserved[3];
        /** Operation code */
        uint8_t opcode;
        /** Status */
        uint32_t status;
} __attribute__ (( packed ));

/** Command succeeded */
#define GVE_ADMIN_STATUS_OK 0x00000001

/** Simple admin command */
struct gve_admin_simple {
        /** Header */
        struct gve_admin_header hdr;
        /** ID */
        uint32_t id;
} __attribute__ (( packed ));

/** Describe device command */
#define GVE_ADMIN_DESCRIBE 0x0001

/** Describe device command */
struct gve_admin_describe {
        /** Header */
        struct gve_admin_header hdr;
        /** Descriptor buffer address */
        uint64_t addr;
        /** Descriptor version */
        uint32_t ver;
        /** Descriptor maximum length */
        uint32_t len;
} __attribute__ (( packed ));

/** Device descriptor version */
#define GVE_ADMIN_DESCRIBE_VER 1

/** Device descriptor */
struct gve_device_descriptor {
        /** Reserved */
        uint8_t reserved_a[10];
        /** Number of transmit queue entries */
        uint16_t tx_count;
        /** Number of receive queue entries */
        uint16_t rx_count;
        /** Reserved */
        uint8_t reserved_b[2];
        /** Maximum transmit unit */
        uint16_t mtu;
        /** Number of event counters */
        uint16_t counters;
        /** Reserved */
        uint8_t reserved_c[4];
        /** MAC address */
        struct google_mac mac;
        /** Reserved */
        uint8_t reserved_d[10];
} __attribute__ (( packed ));

/** Configure device resources command */
#define GVE_ADMIN_CONFIGURE 0x0002

/** Configure device resources command */
struct gve_admin_configure {
        /** Header */
        struct gve_admin_header hdr;
        /** Event counter array */
        uint64_t events;
        /** IRQ doorbell address */
        uint64_t irqs;
        /** Number of event counters */
        uint32_t num_events;
        /** Number of IRQ doorbells */
        uint32_t num_irqs;
        /** IRQ doorbell stride */
        uint32_t irq_stride;
} __attribute__ (( packed ));

/** Register page list command */
#define GVE_ADMIN_REGISTER 0x0003

/** Register page list command */
struct gve_admin_register {
        /** Header */
        struct gve_admin_header hdr;
        /** Page list ID */
        uint32_t id;
        /** Number of pages */
        uint32_t count;
        /** Address list address */
        uint64_t addr;
        /** Page size */
        uint64_t size;
} __attribute__ (( packed ));

/**
 * Maximum number of pages per queue
 *
 * This is a policy decision.  Must be sufficient to allow for both
 * the transmit and receive queue fill levels.
 */
#define GVE_QPL_MAX 32

/** Page list */
struct gve_pages {
        /** Page address */
        uint64_t addr[GVE_QPL_MAX];
} __attribute__ (( packed ));

/** Unregister page list command */
#define GVE_ADMIN_UNREGISTER 0x0004

/** Create transmit queue command */
#define GVE_ADMIN_CREATE_TX 0x0005

/** Create transmit queue command */
struct gve_admin_create_tx {
        /** Header */
        struct gve_admin_header hdr;
        /** Queue ID */
        uint32_t id;
        /** Reserved */
        uint8_t reserved_a[4];
        /** Queue resources address */
        uint64_t res;
        /** Descriptor ring address */
        uint64_t desc;
        /** Queue page list ID */
        uint32_t qpl_id;
        /** Notification channel ID */
        uint32_t notify_id;
} __attribute__ (( packed ));

/** Create receive queue command */
#define GVE_ADMIN_CREATE_RX 0x0006

/** Create receive queue command */
struct gve_admin_create_rx {
        /** Header */
        struct gve_admin_header hdr;
        /** Queue ID */
        uint32_t id;
        /** Index */
        uint32_t index;
        /** Reserved */
        uint8_t reserved_a[4];
        /** Notification channel ID */
        uint32_t notify_id;
        /** Queue resources address */
        uint64_t res;
        /** Completion ring address */
        uint64_t cmplt;
        /** Descriptor ring address */
        uint64_t desc;
        /** Queue page list ID */
        uint32_t qpl_id;
        /** Reserved */
        uint8_t reserved_b[2];
        /** Packet buffer size */
        uint16_t bufsz;
} __attribute__ (( packed ));

/** Destroy transmit queue command */
#define GVE_ADMIN_DESTROY_TX 0x0007

/** Destroy receive queue command */
#define GVE_ADMIN_DESTROY_RX 0x0008

/** Deconfigure device resources command */
#define GVE_ADMIN_DECONFIGURE 0x0009

/** An admin queue command */
union gve_admin_command {
        /** Header */
        struct gve_admin_header hdr;
        /** Simple command */
        struct gve_admin_simple simple;
        /** Describe device */
        struct gve_admin_describe desc;
        /** Configure device resources */
        struct gve_admin_configure conf;
        /** Register page list */
        struct gve_admin_register reg;
        /** Create transmit queue */
        struct gve_admin_create_tx create_tx;
        /** Create receive queue */
        struct gve_admin_create_rx create_rx;
        /** Padding */
        uint8_t pad[64];
};

/**
 * Number of admin queue commands
 *
 * This is theoretically a policy decision.  However, older revisions
 * of the hardware seem to have only the "admin queue page frame
 * number" register and no "admin queue length" register, with the
 * implication that the admin queue must be exactly one page in
 * length.
 *
 * Choose to use a one page (4kB) admin queue for both older and newer
 * versions of the hardware, to minimise variability.
 */
#define GVE_ADMIN_COUNT ( GVE_PAGE_SIZE / sizeof ( union gve_admin_command ) )

/** Admin queue */
struct gve_admin {
        /** Commands */
        union gve_admin_command *cmd;
        /** Producer counter */
        uint32_t prod;
        /** DMA mapping */
        struct dma_mapping map;
};

/** Scratch buffer for admin queue commands */
struct gve_scratch {
        /** Buffer contents */
        union {
                /** Device descriptor */
                struct gve_device_descriptor desc;
                /** Page address list */
                struct gve_pages pages;
        } *buf;
        /** DMA mapping */
        struct dma_mapping map;
};

/**
 * An event counter
 *
 * Written by the device to indicate completions.  The device chooses
 * which counter to use for each transmit queue, and stores the index
 * of the chosen counter in the queue resources.
 */
struct gve_event {
        /** Number of events that have occurred */
        volatile uint32_t count;
} __attribute__ (( packed ));

/** Event counter array */
struct gve_events {
        /** Event counters */
        struct gve_event *event;
        /** DMA mapping */
        struct dma_mapping map;
        /** Actual number of event counters */
        unsigned int count;
};

/** An interrupt channel */
struct gve_irq {
        /** Interrupt doorbell index (within doorbell BAR) */
        uint32_t db_idx;
        /** Reserved */
        uint8_t reserved[60];
} __attribute__ (( packed ));

/**
 * Number of interrupt channels
 *
 * We tell the device how many interrupt channels we have provided via
 * the "configure device resources" admin queue command.  The device
 * will accept being given zero interrupt channels, but will
 * subsequently fail to create more than a single queue (either
 * transmit or receive).
 *
 * There is, of course, no documentation indicating how may interrupt
 * channels actually need to be provided.  In the absence of evidence
 * to the contrary, assume that two channels (one for transmit, one
 * for receive) will be sufficient.
 */
#define GVE_IRQ_COUNT 2

/** Interrupt channel array */
struct gve_irqs {
        /** Interrupt channels */
        struct gve_irq *irq;
        /** DMA mapping */
        struct dma_mapping map;
        /** Interrupt doorbells */
        volatile uint32_t *db[GVE_IRQ_COUNT];
};

/** Disable interrupts */
#define GVE_IRQ_DISABLE 0x40000000UL

/**
 * Queue resources
 *
 * Written by the device to indicate the indices of the chosen event
 * counter and descriptor doorbell register.
 *
 * This appears to be a largely pointless data structure: the relevant
 * information is static for the lifetime of the queue and could
 * trivially have been returned in the response for the "create
 * transmit/receive queue" command, instead of requiring yet another
 * page-aligned coherent DMA buffer allocation.
 */
struct gve_resources {
        /** Descriptor doorbell index (within doorbell BAR) */
        uint32_t db_idx;
        /** Event counter index (within event counter array) */
        uint32_t evt_idx;
        /** Reserved */
        uint8_t reserved[56];
} __attribute__ (( packed ));

/**
 * Queue data buffer size
 *
 * In theory, we may specify the size of receive buffers.  However,
 * the original version of the device seems not to have a parameter
 * for this, and assumes the use of half-page (2kB) buffers.  Choose
 * to use this as the buffer size, on the assumption that older
 * devices will not support any other buffer size.
 */
#define GVE_BUF_SIZE ( GVE_PAGE_SIZE / 2 )

/** Number of data buffers per page */
#define GVE_BUF_PER_PAGE ( GVE_PAGE_SIZE / GVE_BUF_SIZE )

/**
 * Queue page list
 *
 * The device uses preregistered pages for fast-path DMA operations
 * (i.e. transmit and receive buffers).  A list of device addresses
 * for each page must be registered before the transmit or receive
 * queue is created, and cannot subsequently be modified.
 *
 * The Linux driver allocates pages as DMA_TO_DEVICE or
 * DMA_FROM_DEVICE as appropriate, and uses dma_sync_single_for_cpu()
 * etc to ensure that data is copied to/from bounce buffers as needed.
 *
 * Unfortunately there is no such sync operation available within our
 * DMA API, since we are constrained by the limitations imposed by
 * EFI_PCI_IO_PROTOCOL.  There is no way to synchronise a buffer
 * without also [un]mapping it, and no way to force the reuse of the
 * same device address for a subsequent remapping.  We are therefore
 * constrained to use only DMA-coherent buffers, since this is the
 * only way we can repeatedly reuse the same device address.
 *
 * Newer versions of the gVNIC device support "raw DMA addressing
 * (RDA)", which is essentially a prebuilt queue page list covering
 * the whole of the guest address space.  Unfortunately we cannot rely
 * on this, since older versions will not support it.
 *
 * Experimentation suggests that the device will accept a request to
 * create a queue page list covering the whole of the guest address
 * space via two giant "pages" of 2^63 bytes each.  However,
 * experimentation also suggests that the device will accept any old
 * garbage value as the "page size".  In the total absence of any
 * documentation, it is probably unsafe to conclude that the device is
 * bothering to look at or respect the "page size" parameter: it is
 * most likely just presuming the use of 4kB pages.
 */
struct gve_qpl {
        /** Page addresses */
        userptr_t data;
        /** Page mapping */
        struct dma_mapping map;
        /** Number of pages */
        unsigned int count;
        /** Queue page list ID */
        unsigned int id;
};

/**
 * Maximum number of transmit buffers
 *
 * This is a policy decision.
 */
#define GVE_TX_FILL 8

/** Transmit queue page list ID */
#define GVE_TX_QPL 0x18ae5458

/** Tranmsit queue interrupt channel */
#define GVE_TX_IRQ 0

/** A transmit or receive buffer descriptor */
struct gve_buffer {
        /** Address (within queue page list address space) */
        uint64_t addr;
} __attribute__ (( packed ));

/** A transmit packet descriptor */
struct gve_tx_packet {
        /** Type */
        uint8_t type;
        /** Reserved */
        uint8_t reserved_a[2];
        /** Number of descriptors in this packet */
        uint8_t count;
        /** Total length of this packet */
        uint16_t total;
        /** Length of this descriptor */
        uint16_t len;
} __attribute__ (( packed ));

/** A transmit descriptor */
struct gve_tx_descriptor {
        /** Packet descriptor */
        struct gve_tx_packet pkt;
        /** Buffer descriptor */
        struct gve_buffer buf;
} __attribute__ (( packed ));

/** Start of packet transmit descriptor type */
#define GVE_TX_TYPE_START 0x00

/** Continuation of packet transmit descriptor type */
#define GVE_TX_TYPE_CONT 0x20

/**
 * Maximum number of receive buffers
 *
 * This is a policy decision.  Experiments suggest that using fewer
 * than 64 receive buffers leads to excessive packet drop rates on
 * some instance types.
 */
#define GVE_RX_FILL 64

/** Receive queue page list ID */
#define GVE_RX_QPL 0x18ae5258

/** Receive queue interrupt channel */
#define GVE_RX_IRQ 1

/** A receive descriptor */
struct gve_rx_descriptor {
        /** Buffer descriptor */
        struct gve_buffer buf;
} __attribute__ (( packed ));

/** A receive packet descriptor */
struct gve_rx_packet {
        /** Length */
        uint16_t len;
        /** Flags */
        uint8_t flags;
        /** Sequence number */
        uint8_t seq;
} __attribute__ (( packed ));

/** Receive error */
#define GVE_RXF_ERROR 0x08

/** Receive packet continues into next descriptor */
#define GVE_RXF_MORE 0x20

/** Receive sequence number mask */
#define GVE_RX_SEQ_MASK 0x07

/** A receive completion descriptor */
struct gve_rx_completion {
        /** Reserved */
        uint8_t reserved[60];
        /** Packet descriptor */
        struct gve_rx_packet pkt;
} __attribute__ (( packed ));

/** Padding at the start of all received packets */
#define GVE_RX_PAD 2

/** A descriptor queue */
struct gve_queue {
        /** Descriptor ring */
        userptr_t desc;
        /** Completion ring */
        userptr_t cmplt;
        /** Queue resources */
        struct gve_resources *res;

        /** Queue type */
        const struct gve_queue_type *type;
        /** Number of descriptors (must be a power of two) */
        unsigned int count;
        /** Maximum fill level (must be a power of two) */
        unsigned int fill;

        /** Descriptor mapping */
        struct dma_mapping desc_map;
        /** Completion mapping */
        struct dma_mapping cmplt_map;
        /** Queue resources mapping */
        struct dma_mapping res_map;

        /** Doorbell register */
        volatile uint32_t *db;
        /** Event counter */
        struct gve_event *event;

        /** Producer counter */
        uint32_t prod;
        /** Consumer counter */
        uint32_t cons;

        /** Queue page list */
        struct gve_qpl qpl;
};

/** A descriptor queue type */
struct gve_queue_type {
        /** Name */
        const char *name;
        /**
         * Populate command parameters to create queue
         *
         * @v queue             Descriptor queue
         * @v cmd               Admin queue command
         */
        void ( * param ) ( struct gve_queue *queue,
                           union gve_admin_command *cmd );
        /** Queue page list ID */
        uint32_t qpl;
        /** Interrupt channel */
        uint8_t irq;
        /** Maximum fill level */
        uint8_t fill;
        /** Descriptor size */
        uint8_t desc_len;
        /** Completion size */
        uint8_t cmplt_len;
        /** Command to create queue */
        uint8_t create;
        /** Command to destroy queue */
        uint8_t destroy;
};

/** A Google Virtual Ethernet NIC */
struct gve_nic {
        /** Configuration registers */
        void *cfg;
        /** Doorbell registers */
        void *db;
        /** PCI revision */
        uint8_t revision;
        /** Network device */
        struct net_device *netdev;
        /** DMA device */
        struct dma_device *dma;

        /** Admin queue */
        struct gve_admin admin;
        /** Interrupt channels */
        struct gve_irqs irqs;
        /** Event counters */
        struct gve_events events;
        /** Scratch buffer */
        struct gve_scratch scratch;

        /** Transmit queue */
        struct gve_queue tx;
        /** Receive queue */
        struct gve_queue rx;
        /** Transmit I/O buffers */
        struct io_buffer *tx_iobuf[GVE_TX_FILL];
        /** Receive sequence number */
        unsigned int seq;

        /** Startup process */
        struct process startup;
        /** Startup process retry counter */
        unsigned int retries;
        /** Reset recovery watchdog timer */
        struct retry_timer watchdog;
        /** Reset recovery recorded activity counter */
        uint32_t activity;
};

/** Maximum time to wait for admin queue commands */
#define GVE_ADMIN_MAX_WAIT_MS 500

/** Maximum number of times to reattempt device reset */
#define GVE_RESET_MAX_RETRY 5

/** Time between reset recovery checks */
#define GVE_WATCHDOG_TIMEOUT ( 1 * TICKS_PER_SEC )

#endif /* _GVE_H */