intelxlvf.c

Go to the documentation of this file.

/*
 * Copyright (C) 2019 Michael Brown <mbrown@fensystems.co.uk>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 *
 * You can also choose to distribute this program under the terms of
 * the Unmodified Binary Distribution Licence (as given in the file
 * COPYING.UBDL), provided that you have satisfied its requirements.
 */

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );

#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/pci.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include "intelxlvf.h"

/** @file
 *
 * Intel 40 Gigabit Ethernet virtual function network card driver
 *
 */

/******************************************************************************
 *
 * Device reset
 *
 ******************************************************************************
 */

/**
 * Wait for admin event queue to be torn down
 *
 * @v intelxl           Intel device
 * @ret rc              Return status code
 */
static int intelxlvf_reset_wait_teardown ( struct intelxl_nic *intelxl ) {
        uint32_t admin_evt_len;
        unsigned int i;

        /* Wait for admin event queue to be torn down */
        for ( i = 0 ; i < INTELXLVF_RESET_MAX_WAIT_MS ; i++ ) {

                /* Check admin event queue length register */
                admin_evt_len = readl ( intelxl->regs + INTELXLVF_ADMIN +
                                        INTELXLVF_ADMIN_EVT_LEN );
                if ( ! ( admin_evt_len & INTELXL_ADMIN_LEN_ENABLE ) )
                        return 0;

                /* Delay */
                mdelay ( 1 );
        }

        DBGC ( intelxl, "INTELXL %p timed out waiting for teardown (%#08x)\n",
               intelxl, admin_evt_len );
        return -ETIMEDOUT;
}

/**
 * Wait for virtual function to be marked as active
 *
 * @v intelxl           Intel device
 * @ret rc              Return status code
 */
static int intelxlvf_reset_wait_active ( struct intelxl_nic *intelxl ) {
        uint32_t vfgen_rstat;
        unsigned int vfr_state;
        unsigned int i;

        /* Wait for virtual function to be marked as active */
        for ( i = 0 ; i < INTELXLVF_RESET_MAX_WAIT_MS ; i++ ) {

                /* Check status as written by physical function driver */
                vfgen_rstat = readl ( intelxl->regs + INTELXLVF_VFGEN_RSTAT );
                vfr_state = INTELXLVF_VFGEN_RSTAT_VFR_STATE ( vfgen_rstat );
                if ( vfr_state == INTELXLVF_VFGEN_RSTAT_VFR_STATE_ACTIVE )
                        return 0;

                /* Delay */
                mdelay ( 1 );
        }

        DBGC ( intelxl, "INTELXL %p timed out waiting for activation "
               "(%#08x)\n", intelxl, vfgen_rstat );
        return -ETIMEDOUT;
}

/**
 * Wait for reset to complete
 *
 * @v intelxl           Intel device
 * @ret rc              Return status code
 */
static int intelxlvf_reset_wait ( struct intelxl_nic *intelxl ) {
        int rc;

        /* Wait for minimum reset time */
        mdelay ( INTELXLVF_RESET_DELAY_MS );

        /* Wait for reset to take effect */
        if ( ( rc = intelxlvf_reset_wait_teardown ( intelxl ) ) != 0 )
                goto err_teardown;

        /* Wait for virtual function to become active */
        if ( ( rc = intelxlvf_reset_wait_active ( intelxl ) ) != 0 )
                goto err_active;

 err_active:
 err_teardown:
        intelxl_reopen_admin ( intelxl );
        return rc;
}

/**
 * Reset hardware via admin queue
 *
 * @v intelxl           Intel device
 * @ret rc              Return status code
 */
static int intelxlvf_reset_admin ( struct intelxl_nic *intelxl ) {
        struct intelxlvf_admin_descriptor *cmd;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->opcode = cpu_to_le16 ( INTELXLVF_ADMIN_SEND_TO_PF );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_RESET );

        /* Issue command */
        if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
                return rc;

        /* Wait for reset to complete */
        if ( ( rc = intelxlvf_reset_wait ( intelxl ) ) != 0 )
                return rc;

        return 0;
}

/******************************************************************************
 *
 * Admin queue
 *
 ******************************************************************************
 */

/** Admin command queue register offsets */
static const struct intelxl_admin_offsets intelxlvf_admin_command_offsets = {
        .bal = INTELXLVF_ADMIN_CMD_BAL,
        .bah = INTELXLVF_ADMIN_CMD_BAH,
        .len = INTELXLVF_ADMIN_CMD_LEN,
        .head = INTELXLVF_ADMIN_CMD_HEAD,
        .tail = INTELXLVF_ADMIN_CMD_TAIL,
};

/** Admin event queue register offsets */
static const struct intelxl_admin_offsets intelxlvf_admin_event_offsets = {
        .bal = INTELXLVF_ADMIN_EVT_BAL,
        .bah = INTELXLVF_ADMIN_EVT_BAH,
        .len = INTELXLVF_ADMIN_EVT_LEN,
        .head = INTELXLVF_ADMIN_EVT_HEAD,
        .tail = INTELXLVF_ADMIN_EVT_TAIL,
};

/**
 * Issue admin queue virtual function command
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_command ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxl_admin *admin = &intelxl->command;
        struct intelxl_admin_descriptor *xlcmd;
        struct intelxlvf_admin_descriptor *cmd;
        unsigned int i;
        int rc;

        /* Populate descriptor */
        xlcmd = &admin->desc[ admin->index % INTELXL_ADMIN_NUM_DESC ];
        cmd = container_of ( xlcmd, struct intelxlvf_admin_descriptor, xl );
        cmd->opcode = cpu_to_le16 ( INTELXLVF_ADMIN_SEND_TO_PF );

        /* Record opcode */
        intelxl->vopcode = le32_to_cpu ( cmd->vopcode );

        /* Issue command */
        if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
                goto err_command;

        /* Wait for response */
        for ( i = 0 ; i < INTELXLVF_ADMIN_MAX_WAIT_MS ; i++ ) {

                /* Poll admin event queue */
                intelxl_poll_admin ( netdev );

                /* If response has not arrived, delay 1ms and retry */
                if ( intelxl->vopcode ) {
                        mdelay ( 1 );
                        continue;
                }

                /* Check for errors */
                if ( cmd->vret != 0 )
                        return -EIO;

                return 0;
        }

        rc = -ETIMEDOUT;
        DBGC ( intelxl, "INTELXL %p timed out waiting for admin VF command "
               "%#x\n", intelxl, intelxl->vopcode );
 err_command:
        intelxl->vopcode = 0;
        return rc;
}

/**
 * Handle link status event
 *
 * @v netdev            Network device
 * @v link              Link status
 */
static void intelxlvf_admin_link ( struct net_device *netdev,
                                   struct intelxlvf_admin_status_link *link ) {
        struct intelxl_nic *intelxl = netdev->priv;

        DBGC ( intelxl, "INTELXL %p link %#02x speed %#02x\n", intelxl,
               link->status, link->speed );

        /* Update network device */
        if ( link->status ) {
                netdev_link_up ( netdev );
        } else {
                netdev_link_down ( netdev );
        }
}

/**
 * Handle status change event
 *
 * @v netdev            Network device
 * @v stat              Status change event
 */
static void
intelxlvf_admin_status ( struct net_device *netdev,
                         struct intelxlvf_admin_status_buffer *stat ) {
        struct intelxl_nic *intelxl = netdev->priv;

        /* Handle event */
        switch ( stat->event ) {
        case cpu_to_le32 ( INTELXLVF_ADMIN_STATUS_LINK ):
                intelxlvf_admin_link ( netdev, &stat->data.link );
                break;
        default:
                DBGC ( intelxl, "INTELXL %p unrecognised status change "
                       "event %#x:\n", intelxl, le32_to_cpu ( stat->event ) );
                DBGC_HDA ( intelxl, 0, stat, sizeof ( *stat ) );
                break;
        }
}

/**
 * Handle admin event
 *
 * @v netdev            Network device
 * @v xlevt             Admin queue event descriptor
 * @v xlbuf             Admin queue event data buffer
 */
static void intelxlvf_admin_event ( struct net_device *netdev,
                                    struct intelxl_admin_descriptor *xlevt,
                                    union intelxl_admin_buffer *xlbuf ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxl_admin *admin = &intelxl->command;
        struct intelxlvf_admin_descriptor *evt =
                container_of ( xlevt, struct intelxlvf_admin_descriptor, xl );
        union intelxlvf_admin_buffer *buf =
                container_of ( xlbuf, union intelxlvf_admin_buffer, xl );
        unsigned int vopcode;
        unsigned int index;

        /* Ignore unrecognised events */
        if ( evt->opcode != cpu_to_le16 ( INTELXLVF_ADMIN_SEND_TO_VF ) ) {
                DBGC ( intelxl, "INTELXL %p unrecognised event opcode "
                       "%#04x\n", intelxl, le16_to_cpu ( evt->opcode ) );
                return;
        }

        /* Record command response if applicable */
        vopcode = le32_to_cpu ( evt->vopcode );
        if ( vopcode == intelxl->vopcode ) {
                index = ( ( admin->index - 1 ) % INTELXL_ADMIN_NUM_DESC );
                memcpy ( &admin->desc[index], evt, sizeof ( *evt ) );
                memcpy ( &admin->buf[index], buf, sizeof ( *buf ) );
                intelxl->vopcode = 0;
                if ( evt->vret != 0 ) {
                        DBGC ( intelxl, "INTELXL %p admin VF command %#x "
                               "error %d\n", intelxl, vopcode,
                               le32_to_cpu ( evt->vret ) );
                        DBGC_HDA ( intelxl, virt_to_phys ( evt ), evt,
                                   sizeof ( *evt ) );
                        DBGC_HDA ( intelxl, virt_to_phys ( buf ), buf,
                                   le16_to_cpu ( evt->len ) );
                }
                return;
        }

        /* Handle unsolicited events */
        switch ( vopcode ) {
        case INTELXLVF_ADMIN_STATUS:
                intelxlvf_admin_status ( netdev, &buf->stat );
                break;
        default:
                DBGC ( intelxl, "INTELXL %p unrecognised VF event %#x:\n",
                       intelxl, vopcode );
                DBGC_HDA ( intelxl, virt_to_phys ( evt ), evt,
                           sizeof ( *evt ) );
                DBGC_HDA ( intelxl, virt_to_phys ( buf ), buf,
                           le16_to_cpu ( evt->len ) );
                break;
        }
}

/**
 * Negotiate API version
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_version ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        unsigned int api;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_VERSION );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->ver ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->ver.major = cpu_to_le32 ( INTELXLVF_ADMIN_API_MAJOR );
        buf->ver.minor = cpu_to_le32 ( INTELXLVF_ADMIN_API_MINOR );

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;
        api = le32_to_cpu ( buf->ver.major );
        DBGC ( intelxl, "INTELXL %p API v%d.%d\n",
               intelxl, api, le32_to_cpu ( buf->ver.minor ) );

        /* Check for API compatibility */
        if ( api > INTELXLVF_ADMIN_API_MAJOR ) {
                DBGC ( intelxl, "INTELXL %p unsupported API v%d\n",
                       intelxl, api );
                return -ENOTSUP;
        }

        return 0;
}

/**
 * Get resources
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_get_resources ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_GET_RESOURCES );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->caps ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->caps.caps = cpu_to_le32 ( INTELXLVF_ADMIN_CAP_L2 |
                                       INTELXLVF_ADMIN_CAP_RQPS );

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;

        /* Parse response */
        intelxl->caps = le32_to_cpu ( buf->res.caps );
        intelxl->vsi = le16_to_cpu ( buf->res.vsi );
        memcpy ( netdev->hw_addr, buf->res.mac, ETH_ALEN );
        DBGC ( intelxl, "INTELXL %p capabilities %#08x VSI %#04x\n",
               intelxl, intelxl->caps, intelxl->vsi );

        return 0;
}

/**
 * Get statistics (for debugging)
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_stats ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        struct intelxlvf_admin_stats *tx;
        struct intelxlvf_admin_stats *rx;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_GET_STATS );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->queues ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->queues.vsi = cpu_to_le16 ( intelxl->vsi );
        tx = &buf->stats.tx;
        rx = &buf->stats.rx;

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;
        DBGC ( intelxl, "INTELXL %p TX bytes %#llx discards %#llx errors "
               "%#llx\n", intelxl,
               ( ( unsigned long long ) le64_to_cpu ( tx->bytes ) ),
               ( ( unsigned long long ) le64_to_cpu ( tx->discards ) ),
               ( ( unsigned long long ) le64_to_cpu ( tx->errors ) ) );
        DBGC ( intelxl, "INTELXL %p TX unicasts %#llx multicasts %#llx "
               "broadcasts %#llx\n", intelxl,
               ( ( unsigned long long ) le64_to_cpu ( tx->unicasts ) ),
               ( ( unsigned long long ) le64_to_cpu ( tx->multicasts ) ),
               ( ( unsigned long long ) le64_to_cpu ( tx->broadcasts ) ) );
        DBGC ( intelxl, "INTELXL %p RX bytes %#llx discards %#llx errors "
               "%#llx\n", intelxl,
               ( ( unsigned long long ) le64_to_cpu ( rx->bytes ) ),
               ( ( unsigned long long ) le64_to_cpu ( rx->discards ) ),
               ( ( unsigned long long ) le64_to_cpu ( rx->errors ) ) );
        DBGC ( intelxl, "INTELXL %p RX unicasts %#llx multicasts %#llx "
               "broadcasts %#llx\n", intelxl,
               ( ( unsigned long long ) le64_to_cpu ( rx->unicasts ) ),
               ( ( unsigned long long ) le64_to_cpu ( rx->multicasts ) ),
               ( ( unsigned long long ) le64_to_cpu ( rx->broadcasts ) ) );

        return 0;
}

/**
 * Configure number of queue pairs
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_request_qps ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->opcode = cpu_to_le16 ( INTELXLVF_ADMIN_SEND_TO_PF );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_REQUEST_QPS );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->rqps ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->rqps.count = cpu_to_le16 ( 1 );

        /* Issue command (which will trigger a reset) */
        if ( ( rc = intelxl_admin_command ( intelxl ) ) != 0 )
                return rc;

        /* Wait for reset to complete */
        if ( ( rc = intelxlvf_reset_wait ( intelxl ) ) != 0 )
                return rc;

        /* Reestablish capabilities to reactivate VF after reset */
        if ( ( rc = intelxlvf_admin_get_resources ( netdev ) ) != 0 )
                return rc;

        return 0;
}

/******************************************************************************
 *
 * Network device interface
 *
 ******************************************************************************
 */

/**
 * Configure queues
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_configure ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_CONFIGURE );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->cfg ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->cfg.vsi = cpu_to_le16 ( intelxl->vsi );
        buf->cfg.count = cpu_to_le16 ( 1 );
        buf->cfg.tx.vsi = cpu_to_le16 ( intelxl->vsi );
        buf->cfg.tx.count = cpu_to_le16 ( INTELXL_TX_NUM_DESC );
        buf->cfg.tx.base = cpu_to_le64 ( dma ( &intelxl->tx.map,
                                               intelxl->tx.desc.raw ) );
        buf->cfg.rx.vsi = cpu_to_le16 ( intelxl->vsi );
        buf->cfg.rx.count = cpu_to_le32 ( INTELXL_RX_NUM_DESC );
        buf->cfg.rx.len = cpu_to_le32 ( intelxl->mfs );
        buf->cfg.rx.mfs = cpu_to_le32 ( intelxl->mfs );
        buf->cfg.rx.base = cpu_to_le64 ( dma ( &intelxl->rx.map,
                                               intelxl->rx.desc.raw ) );

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Configure IRQ mapping
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_irq_map ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_IRQ_MAP );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->irq ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->irq.count = cpu_to_le16 ( 1 );
        buf->irq.vsi = cpu_to_le16 ( intelxl->vsi );
        buf->irq.vec = cpu_to_le16 ( INTELXLVF_MSIX_VECTOR );
        buf->irq.rxmap = cpu_to_le16 ( 0x0001 );
        buf->irq.txmap = cpu_to_le16 ( 0x0001 );

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Enable/disable queues
 *
 * @v netdev            Network device
 * @v enable            Enable queues
 * @ret rc              Return status code
 */
static int intelxlvf_admin_queues ( struct net_device *netdev, int enable ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = ( enable ? cpu_to_le32 ( INTELXLVF_ADMIN_ENABLE ) :
                         cpu_to_le32 ( INTELXLVF_ADMIN_DISABLE ) );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->queues ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->queues.vsi = cpu_to_le16 ( intelxl->vsi );
        buf->queues.rx = cpu_to_le32 ( 1 );
        buf->queues.tx = cpu_to_le32 ( 1 );

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Configure promiscuous mode
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_admin_promisc ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        struct intelxlvf_admin_descriptor *cmd;
        union intelxlvf_admin_buffer *buf;
        int rc;

        /* Populate descriptor */
        cmd = intelxlvf_admin_command_descriptor ( intelxl );
        cmd->vopcode = cpu_to_le32 ( INTELXLVF_ADMIN_PROMISC );
        cmd->flags = cpu_to_le16 ( INTELXL_ADMIN_FL_RD | INTELXL_ADMIN_FL_BUF );
        cmd->len = cpu_to_le16 ( sizeof ( buf->promisc ) );
        buf = intelxlvf_admin_command_buffer ( intelxl );
        buf->promisc.vsi = cpu_to_le16 ( intelxl->vsi );
        buf->promisc.flags = cpu_to_le16 ( INTELXL_ADMIN_PROMISC_FL_UNICAST |
                                           INTELXL_ADMIN_PROMISC_FL_MULTICAST );

        /* Issue command */
        if ( ( rc = intelxlvf_admin_command ( netdev ) ) != 0 )
                return rc;

        return 0;
}

/**
 * Open network device
 *
 * @v netdev            Network device
 * @ret rc              Return status code
 */
static int intelxlvf_open ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        int rc;

        /* Calculate maximum frame size */
        intelxl->mfs = ( ( ETH_HLEN + netdev->mtu + 4 /* CRC */ +
                           INTELXL_ALIGN - 1 ) & ~( INTELXL_ALIGN - 1 ) );

        /* Allocate transmit descriptor ring */
        if ( ( rc = intelxl_alloc_ring ( intelxl, &intelxl->tx ) ) != 0 )
                goto err_alloc_tx;

        /* Allocate receive descriptor ring */
        if ( ( rc = intelxl_alloc_ring ( intelxl, &intelxl->rx ) ) != 0 )
                goto err_alloc_rx;

        /* Configure queues */
        if ( ( rc = intelxlvf_admin_configure ( netdev ) ) != 0 )
                goto err_configure;

        /* Configure IRQ map */
        if ( ( rc = intelxlvf_admin_irq_map ( netdev ) ) != 0 )
                goto err_irq_map;

        /* Enable queues */
        if ( ( rc = intelxlvf_admin_queues ( netdev, 1 ) ) != 0 )
                goto err_enable;

        /* Configure promiscuous mode */
        if ( ( rc = intelxlvf_admin_promisc ( netdev ) ) != 0 )
                goto err_promisc;

        /* Reset statistics counters (if debugging) */
        if ( DBG_LOG )
                intelxlvf_admin_stats ( netdev );

        return 0;

 err_promisc:
        intelxlvf_admin_queues ( netdev, 0 );
 err_enable:
 err_irq_map:
 err_configure:
        intelxl_free_ring ( intelxl, &intelxl->rx );
 err_alloc_rx:
        intelxl_free_ring ( intelxl, &intelxl->tx );
 err_alloc_tx:
        return rc;
}

/**
 * Close network device
 *
 * @v netdev            Network device
 */
static void intelxlvf_close ( struct net_device *netdev ) {
        struct intelxl_nic *intelxl = netdev->priv;
        int rc;

        /* Show statistics (if debugging) */
        if ( DBG_LOG )
                intelxlvf_admin_stats ( netdev );

        /* Disable queues */
        if ( ( rc = intelxlvf_admin_queues ( netdev, 0 ) ) != 0 ) {
                /* Leak memory; there's nothing else we can do */
                return;
        }

        /* Free receive descriptor ring */
        intelxl_free_ring ( intelxl, &intelxl->rx );

        /* Free transmit descriptor ring */
        intelxl_free_ring ( intelxl, &intelxl->tx );

        /* Discard any unused receive buffers */
        intelxl_empty_rx ( intelxl );
}

/** Network device operations */
static struct net_device_operations intelxlvf_operations = {
        .open           = intelxlvf_open,
        .close          = intelxlvf_close,
        .transmit       = intelxl_transmit,
        .poll           = intelxl_poll,
};

/******************************************************************************
 *
 * PCI interface
 *
 ******************************************************************************
 */

/**
 * Probe PCI device
 *
 * @v pci               PCI device
 * @ret rc              Return status code
 */
static int intelxlvf_probe ( struct pci_device *pci ) {
        struct net_device *netdev;
        struct intelxl_nic *intelxl;
        int rc;

        /* Allocate and initialise net device */
        netdev = alloc_etherdev ( sizeof ( *intelxl ) );
        if ( ! netdev ) {
                rc = -ENOMEM;
                goto err_alloc;
        }
        netdev_init ( netdev, &intelxlvf_operations );
        intelxl = netdev->priv;
        pci_set_drvdata ( pci, netdev );
        netdev->dev = &pci->dev;
        memset ( intelxl, 0, sizeof ( *intelxl ) );
        intelxl->intr = INTELXLVF_VFINT_DYN_CTLN ( INTELXLVF_MSIX_VECTOR );
        intelxl->handle = intelxlvf_admin_event;
        intelxl_init_admin ( &intelxl->command, INTELXLVF_ADMIN,
                             &intelxlvf_admin_command_offsets );
        intelxl_init_admin ( &intelxl->event, INTELXLVF_ADMIN,
                             &intelxlvf_admin_event_offsets );
        intelxlvf_init_ring ( &intelxl->tx, INTELXL_TX_NUM_DESC,
                              sizeof ( intelxl->tx.desc.tx[0] ),
                              INTELXLVF_QTX_TAIL );
        intelxlvf_init_ring ( &intelxl->rx, INTELXL_RX_NUM_DESC,
                              sizeof ( intelxl->rx.desc.rx[0] ),
                              INTELXLVF_QRX_TAIL );

        /* Fix up PCI device */
        adjust_pci_device ( pci );

        /* Map registers */
        intelxl->regs = pci_ioremap ( pci, pci->membase, INTELXLVF_BAR_SIZE );
        if ( ! intelxl->regs ) {
                rc = -ENODEV;
                goto err_ioremap;
        }

        /* Configure DMA */
        intelxl->dma = &pci->dma;
        dma_set_mask_64bit ( intelxl->dma );
        netdev->dma = intelxl->dma;

        /* Locate PCI Express capability */
        intelxl->exp = pci_find_capability ( pci, PCI_CAP_ID_EXP );
        if ( ! intelxl->exp ) {
                DBGC ( intelxl, "INTELXL %p missing PCIe capability\n",
                       intelxl );
                rc = -ENXIO;
                goto err_exp;
        }

        /* Reset the function via PCIe FLR */
        pci_reset ( pci, intelxl->exp );

        /* Enable MSI-X dummy interrupt */
        if ( ( rc = intelxl_msix_enable ( intelxl, pci,
                                          INTELXLVF_MSIX_VECTOR ) ) != 0 )
                goto err_msix;

        /* Open admin queues */
        if ( ( rc = intelxl_open_admin ( intelxl ) ) != 0 )
                goto err_open_admin;

        /* Reset the function via admin queue */
        if ( ( rc = intelxlvf_reset_admin ( intelxl ) ) != 0 )
                goto err_reset_admin;

        /* Negotiate API version */
        if ( ( rc = intelxlvf_admin_version ( netdev ) ) != 0 )
                goto err_version;

        /* Get MAC address */
        if ( ( rc = intelxlvf_admin_get_resources ( netdev ) ) != 0 )
                goto err_get_resources;

        /* Configure number of queue pairs, if applicable */
        if ( ( intelxl->caps & INTELXLVF_ADMIN_CAP_RQPS ) &&
             ( ( rc = intelxlvf_admin_request_qps ( netdev ) ) != 0 ) )
                goto err_rqps;

        /* Register network device */
        if ( ( rc = register_netdev ( netdev ) ) != 0 )
                goto err_register_netdev;

        return 0;

        unregister_netdev ( netdev );
 err_register_netdev:
 err_rqps:
 err_get_resources:
 err_version:
 err_reset_admin:
        intelxl_close_admin ( intelxl );
 err_open_admin:
        intelxl_msix_disable ( intelxl, pci, INTELXLVF_MSIX_VECTOR );
 err_msix:
        pci_reset ( pci, intelxl->exp );
 err_exp:
        iounmap ( intelxl->regs );
 err_ioremap:
        netdev_nullify ( netdev );
        netdev_put ( netdev );
 err_alloc:
        return rc;
}

/**
 * Remove PCI device
 *
 * @v pci               PCI device
 */
static void intelxlvf_remove ( struct pci_device *pci ) {
        struct net_device *netdev = pci_get_drvdata ( pci );
        struct intelxl_nic *intelxl = netdev->priv;

        /* Unregister network device */
        unregister_netdev ( netdev );

        /* Reset the function via admin queue */
        intelxlvf_reset_admin ( intelxl );

        /* Close admin queues */
        intelxl_close_admin ( intelxl );

        /* Disable MSI-X dummy interrupt */
        intelxl_msix_disable ( intelxl, pci, INTELXLVF_MSIX_VECTOR );

        /* Reset the function via PCIe FLR */
        pci_reset ( pci, intelxl->exp );

        /* Free network device */
        iounmap ( intelxl->regs );
        netdev_nullify ( netdev );
        netdev_put ( netdev );
}

/** PCI device IDs */
static struct pci_device_id intelxlvf_nics[] = {
        PCI_ROM ( 0x8086, 0x154c, "xl710-vf", "XL710 VF", 0 ),
        PCI_ROM ( 0x8086, 0x1571, "xl710-vf-hv", "XL710 VF (Hyper-V)", 0 ),
        PCI_ROM ( 0x8086, 0x1889, "iavf", "Intel adaptive VF", 0 ),
        PCI_ROM ( 0x8086, 0x37cd, "x722-vf", "X722 VF", 0 ),
        PCI_ROM ( 0x8086, 0x37d9, "x722-vf-hv", "X722 VF (Hyper-V)", 0 ),
};

/** PCI driver */
struct pci_driver intelxlvf_driver __pci_driver = {
        .ids = intelxlvf_nics,
        .id_count = ( sizeof ( intelxlvf_nics ) /
                      sizeof ( intelxlvf_nics[0] ) ),
        .probe = intelxlvf_probe,
        .remove = intelxlvf_remove,
};