iPXE
golan.h
Go to the documentation of this file.
00001 #ifndef _GOLAN_H_
00002 #define _GOLAN_H_
00003 
00004 /*
00005  * Copyright (C) 2013-2015 Mellanox Technologies Ltd.
00006  *
00007  * This program is free software; you can redistribute it and/or
00008  * modify it under the terms of the GNU General Public License as
00009  * published by the Free Software Foundation; either version 2 of the
00010  * License, or any later version.
00011  *
00012  * This program is distributed in the hope that it will be useful, but
00013  * WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00015  * General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU General Public License
00018  * along with this program; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
00020  * 02110-1301, USA.
00021  */
00022 
00023 FILE_LICENCE ( GPL2_OR_LATER );
00024 
00025 #include <ipxe/pci.h>
00026 #include <ipxe/pcibackup.h>
00027 #include <byteswap.h>
00028 #include <errno.h>
00029 #include <ipxe/io.h>
00030 #include <stdio.h>
00031 #include <unistd.h>
00032 #include "CIB_PRM.h"
00033 #include "mlx_utils/include/public/mlx_utils.h"
00034 
00035 #define GOLAN_PCI_CONFIG_BAR_SIZE       0x100000//HERMON_PCI_CONFIG_BAR_SIZE //TODO: What is the BAR size?
00036 
00037 #define GOLAN_PAS_SIZE  sizeof(uint64_t)
00038 
00039 #define GOLAN_INVALID_LKEY 0x00000100UL
00040 
00041 #define GOLAN_MAX_PORTS 2
00042 #define GOLAN_PORT_BASE 1
00043 
00044 #define MELLANOX_VID    0x15b3
00045 #define GOLAN_HCA_BAR   PCI_BASE_ADDRESS_0      //BAR 0
00046 
00047 #define GOLAN_HCR_MAX_WAIT_MS   10000
00048 
00049 #define min(a,b) ((a)<(b)?(a):(b))
00050 
00051 #define GOLAN_PAGE_SHIFT        12
00052 #define GOLAN_PAGE_SIZE         (1 << GOLAN_PAGE_SHIFT)
00053 #define GOLAN_PAGE_MASK         (GOLAN_PAGE_SIZE - 1)
00054 
00055 #define MAX_MBOX        ( GOLAN_PAGE_SIZE / MAILBOX_STRIDE )
00056 #define DEF_CMD_IDX     1
00057 #define MEM_CMD_IDX     0
00058 #define NO_MBOX         0xffff
00059 #define MEM_MBOX        MEM_CMD_IDX
00060 #define GEN_MBOX        DEF_CMD_IDX
00061 
00062 #define CMD_IF_REV      4
00063 
00064 #define MAX_PASE_MBOX   ((GOLAN_CMD_PAS_CNT) - 2)
00065 
00066 #define CMD_STATUS( golan , idx )               ((struct golan_outbox_hdr *)(get_cmd( (golan) , (idx) )->out))->status
00067 #define CMD_SYND( golan , idx )         ((struct golan_outbox_hdr *)(get_cmd( (golan) , (idx) )->out))->syndrome
00068 #define QRY_PAGES_OUT( golan, idx )             ((struct golan_query_pages_outbox *)(get_cmd( (golan) , (idx) )->out))
00069 
00070 #define VIRT_2_BE64_BUS( addr )         cpu_to_be64(((unsigned long long )virt_to_bus(addr)))
00071 #define BE64_BUS_2_VIRT( addr )         bus_to_virt(be64_to_cpu(addr))
00072 #define USR_2_BE64_BUS( addr )          cpu_to_be64(((unsigned long long )user_to_phys(addr, 0)))
00073 #define BE64_BUS_2_USR( addr )          be64_to_cpu(phys_to_user(addr))
00074 
00075 #define GET_INBOX(golan, idx)           (&(((struct mbox *)(golan->mboxes.inbox))[idx]))
00076 #define GET_OUTBOX(golan, idx)          (&(((struct mbox *)(golan->mboxes.outbox))[idx]))
00077 
00078 #define GOLAN_MBOX_IN( cmd_ptr, in_ptr ) ( {                              \
00079         union {                                                           \
00080                 __be32 raw[4];                                            \
00081                 typeof ( *(in_ptr) ) cooked;                              \
00082         } *u = container_of ( &(cmd_ptr)->in[0], typeof ( *u ), raw[0] ); \
00083         &u->cooked; } )
00084 
00085 #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
00086 
00087 /* Fw status fields */
00088 typedef enum {
00089         NO_ERRORS          = 0x0,
00090         SIGNATURE_ERROR    = 0x1,
00091         TOKEN_ERROR        = 0x2,
00092         BAD_BLOCK_NUMBER   = 0x3,
00093         BAD_OUTPUT_POINTER = 0x4,   // pointer not align to mailbox size
00094         BAD_INPUT_POINTER  = 0x5,   // pointer not align to mailbox size
00095         INTERNAL_ERROR     = 0x6,
00096         INPUT_LEN_ERROR    = 0x7,   // input  length less than 0x8.
00097         OUTPUT_LEN_ERROR   = 0x8,   // output length less than 0x8.
00098         RESERVE_NOT_ZERO   = 0x9,
00099         BAD_CMD_TYPE       = 0x10,
00100 } return_hdr_t;
00101 
00102 struct golan_cmdq_md {
00103         void    *addr;
00104         u16     log_stride;
00105         u16     size;
00106 };
00107 
00108 struct golan_uar {
00109     uint32_t            index;
00110     void                *virt;
00111     unsigned long       phys;
00112 };
00113 
00114 
00115 struct golan_firmware_area {
00116         /* length of area in pages */
00117         uint32_t npages;
00118         /** Firmware area in external memory
00119          *
00120          * This is allocated when first needed, and freed only on
00121          * final teardown, in order to avoid memory map changes at
00122          * runtime.
00123          */
00124         userptr_t area;
00125 };
00126 /* Queue Pair */
00127 #define GOLAN_SEND_WQE_BB_SIZE                  64
00128 #define GOLAN_SEND_UD_WQE_SIZE                  sizeof(struct golan_send_wqe_ud)
00129 #define GOLAN_RECV_WQE_SIZE                             sizeof(struct golan_recv_wqe_ud)
00130 #define GOLAN_WQEBBS_PER_SEND_UD_WQE    DIV_ROUND_UP(GOLAN_SEND_UD_WQE_SIZE, GOLAN_SEND_WQE_BB_SIZE)
00131 #define GOLAN_SEND_OPCODE                       0x0a
00132 #define GOLAN_WQE_CTRL_WQE_IDX_BIT      8
00133 
00134 enum golan_ib_qp_state {
00135         GOLAN_IB_QPS_RESET,
00136         GOLAN_IB_QPS_INIT,
00137         GOLAN_IB_QPS_RTR,
00138         GOLAN_IB_QPS_RTS,
00139         GOLAN_IB_QPS_SQD,
00140         GOLAN_IB_QPS_SQE,
00141         GOLAN_IB_QPS_ERR
00142 };
00143 
00144 struct golan_send_wqe_ud {
00145         struct golan_wqe_ctrl_seg ctrl;
00146         struct golan_av datagram;
00147         struct golan_wqe_data_seg data;
00148 };
00149 
00150 union golan_send_wqe {
00151         struct golan_send_wqe_ud ud;
00152         uint8_t pad[GOLAN_WQEBBS_PER_SEND_UD_WQE * GOLAN_SEND_WQE_BB_SIZE];
00153 };
00154 
00155 struct golan_recv_wqe_ud {
00156         struct golan_wqe_data_seg data[2];
00157 };
00158 
00159 struct golan_recv_wq {
00160         struct golan_recv_wqe_ud *wqes;
00161         /* WQ size in bytes */
00162         int     size;
00163         /* In SQ, it will be increased in wqe_size (number of WQEBBs per WQE) */
00164         u16 next_idx;
00165         /** GRH buffers (if applicable) */
00166         struct ib_global_route_header *grh;
00167         /** Size of GRH buffers */
00168         size_t grh_size;
00169 };
00170 
00171 struct golan_send_wq {
00172         union golan_send_wqe *wqes;
00173         /* WQ size in bytes */
00174         int size;
00175         /* In SQ, it will be increased in wqe_size (number of WQEBBs per WQE) */
00176         u16 next_idx;
00177 };
00178 
00179 struct golan_queue_pair {
00180         void *wqes;
00181         int size;
00182         struct golan_recv_wq rq;
00183         struct golan_send_wq sq;
00184         struct golan_qp_db *doorbell_record;
00185         u32 doorbell_qpn;
00186         enum golan_ib_qp_state state;
00187 };
00188 
00189 /* Completion Queue */
00190 #define GOLAN_CQE_OPCODE_NOT_VALID      0x0f
00191 #define GOLAN_CQE_OPCODE_BIT            4
00192 #define GOLAN_CQ_DB_RECORD_SIZE         sizeof(uint64_t)
00193 #define GOLAN_CQE_OWNER_MASK            1
00194 
00195 #define MANAGE_PAGES_PSA_OFFSET         0
00196 #define PXE_CMDIF_REF                   5
00197 
00198 enum {
00199         GOLAN_CQE_SW_OWNERSHIP = 0x0,
00200         GOLAN_CQE_HW_OWNERSHIP = 0x1
00201 };
00202 
00203 enum {
00204         GOLAN_CQE_SIZE_64       = 0,
00205         GOLAN_CQE_SIZE_128      = 1
00206 };
00207 
00208 struct golan_completion_queue {
00209         struct golan_cqe64      *cqes;
00210         int                                     size;
00211         __be64          *doorbell_record;
00212 };
00213 
00214 
00215 /* Event Queue */
00216 #define GOLAN_EQE_SIZE                          sizeof(struct golan_eqe)
00217 #define GOLAN_NUM_EQES                          8
00218 #define GOLAN_EQ_DOORBELL_OFFSET                0x40
00219 #define DB_BUFFER0_EVEN_OFFSET  0x800
00220 #define DB_BUFFER0_ODD_OFFSET   0x900
00221 
00222 #define GOLAN_EQ_MAP_ALL_EVENTS                                 \
00223         ((1 << GOLAN_EVENT_TYPE_PATH_MIG                )|      \
00224         (1 << GOLAN_EVENT_TYPE_COMM_EST                 )|      \
00225         (1 << GOLAN_EVENT_TYPE_SQ_DRAINED               )|      \
00226         (1 << GOLAN_EVENT_TYPE_SRQ_LAST_WQE             )|      \
00227         (1 << GOLAN_EVENT_TYPE_SRQ_RQ_LIMIT             )|      \
00228         (1 << GOLAN_EVENT_TYPE_CQ_ERROR                 )|      \
00229         (1 << GOLAN_EVENT_TYPE_WQ_CATAS_ERROR           )|      \
00230         (1 << GOLAN_EVENT_TYPE_PATH_MIG_FAILED          )|      \
00231         (1 << GOLAN_EVENT_TYPE_WQ_INVAL_REQ_ERROR       )|      \
00232         (1 << GOLAN_EVENT_TYPE_WQ_ACCESS_ERROR          )|      \
00233         (1 << GOLAN_EVENT_TYPE_SRQ_CATAS_ERROR          )|      \
00234         (1 << GOLAN_EVENT_TYPE_INTERNAL_ERROR           )|      \
00235         (1 << GOLAN_EVENT_TYPE_PORT_CHANGE              )|      \
00236         (1 << GOLAN_EVENT_TYPE_GPIO_EVENT               )|      \
00237         (1 << GOLAN_EVENT_TYPE_CLIENT_RE_REGISTER       )|      \
00238         (1 << GOLAN_EVENT_TYPE_REMOTE_CONFIG            )|      \
00239         (1 << GOLAN_EVENT_TYPE_DB_BF_CONGESTION         )|      \
00240         (1 << GOLAN_EVENT_TYPE_STALL_EVENT              )|      \
00241         (1 << GOLAN_EVENT_TYPE_PACKET_DROPPED           )|      \
00242         (1 << GOLAN_EVENT_TYPE_CMD                      )|      \
00243         (1 << GOLAN_EVENT_TYPE_PAGE_REQUEST             ))
00244 
00245 enum golan_event {
00246         GOLAN_EVENT_TYPE_COMP                   = 0x0,
00247 
00248         GOLAN_EVENT_TYPE_PATH_MIG               = 0x01,
00249         GOLAN_EVENT_TYPE_COMM_EST               = 0x02,
00250         GOLAN_EVENT_TYPE_SQ_DRAINED             = 0x03,
00251         GOLAN_EVENT_TYPE_SRQ_LAST_WQE           = 0x13,
00252         GOLAN_EVENT_TYPE_SRQ_RQ_LIMIT           = 0x14,
00253 
00254         GOLAN_EVENT_TYPE_CQ_ERROR               = 0x04,
00255         GOLAN_EVENT_TYPE_WQ_CATAS_ERROR         = 0x05,
00256         GOLAN_EVENT_TYPE_PATH_MIG_FAILED        = 0x07,
00257         GOLAN_EVENT_TYPE_WQ_INVAL_REQ_ERROR     = 0x10,
00258         GOLAN_EVENT_TYPE_WQ_ACCESS_ERROR        = 0x11,
00259         GOLAN_EVENT_TYPE_SRQ_CATAS_ERROR        = 0x12,
00260 
00261         GOLAN_EVENT_TYPE_INTERNAL_ERROR         = 0x08,
00262         GOLAN_EVENT_TYPE_PORT_CHANGE            = 0x09,
00263         GOLAN_EVENT_TYPE_GPIO_EVENT             = 0x15,
00264 //      GOLAN_EVENT_TYPE_CLIENT_RE_REGISTER     = 0x16,
00265         GOLAN_EVENT_TYPE_REMOTE_CONFIG          = 0x19,
00266 
00267         GOLAN_EVENT_TYPE_DB_BF_CONGESTION       = 0x1a,
00268         GOLAN_EVENT_TYPE_STALL_EVENT            = 0x1b,
00269 
00270         GOLAN_EVENT_TYPE_PACKET_DROPPED         = 0x1f,
00271 
00272         GOLAN_EVENT_TYPE_CMD                    = 0x0a,
00273         GOLAN_EVENT_TYPE_PAGE_REQUEST           = 0x0b,
00274         GOLAN_EVENT_TYPE_PAGE_FAULT             = 0x0C,
00275 };
00276 
00277 enum golan_port_sub_event {
00278     GOLAN_PORT_CHANGE_SUBTYPE_DOWN              = 1,
00279     GOLAN_PORT_CHANGE_SUBTYPE_ACTIVE            = 4,
00280     GOLAN_PORT_CHANGE_SUBTYPE_INITIALIZED       = 5,
00281     GOLAN_PORT_CHANGE_SUBTYPE_LID               = 6,
00282     GOLAN_PORT_CHANGE_SUBTYPE_PKEY              = 7,
00283     GOLAN_PORT_CHANGE_SUBTYPE_GUID              = 8,
00284     GOLAN_PORT_CHANGE_SUBTYPE_CLIENT_REREG      = 9
00285 };
00286 
00287 
00288 enum {
00289         GOLAN_EQE_SW_OWNERSHIP = 0x0,
00290         GOLAN_EQE_HW_OWNERSHIP = 0x1
00291 };
00292 
00293 enum {
00294         GOLAN_EQ_UNARMED        = 0,
00295         GOLAN_EQ_ARMED          = 1,
00296 };
00297 
00298 struct golan_event_queue {
00299         uint8_t                 eqn;
00300         uint64_t                mask;
00301         struct golan_eqe        *eqes;
00302         int                     size;
00303         __be32                  *doorbell;
00304         uint32_t                cons_index;
00305 };
00306 
00307 struct golan_port {
00308         /** Infiniband device */
00309         struct ib_device        *ibdev;
00310         /** Network device */
00311         struct net_device       *netdev;
00312         /** VEP number */
00313         u8 vep_number;
00314 };
00315 
00316 struct golan_mboxes {
00317         void    *inbox;
00318         void    *outbox;
00319 };
00320 
00321 #define GOLAN_OPEN      0x1
00322 
00323 struct golan {
00324         struct pci_device               *pci;
00325         struct golan_hca_init_seg       *iseg;
00326         struct golan_cmdq_md            cmd;
00327         struct golan_hca_cap            caps; /* stored as big indian*/
00328         struct golan_mboxes             mboxes;
00329         struct list_head                pages;
00330         uint32_t                        cmd_bm;
00331         uint32_t                        total_dma_pages;
00332         struct golan_uar                uar;
00333         struct golan_event_queue        eq;
00334         uint32_t                        pdn;
00335         u32                             mkey;
00336         u32                             flags;
00337         mlx_utils               *utils;
00338 
00339         struct golan_port               ports[GOLAN_MAX_PORTS];
00340 #define GOLAN_FW_AREAS_NUM 2
00341         struct golan_firmware_area fw_areas[GOLAN_FW_AREAS_NUM];
00342 };
00343 
00344 #endif /* _GOLAN_H_*/