iPXE
Functions | Variables
arbel.c File Reference

Mellanox Arbel Infiniband HCA. More...

#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/io.h>
#include <ipxe/pci.h>
#include <ipxe/pcibackup.h>
#include <ipxe/malloc.h>
#include <ipxe/umalloc.h>
#include <ipxe/iobuf.h>
#include <ipxe/netdevice.h>
#include <ipxe/infiniband.h>
#include <ipxe/ib_smc.h>
#include "arbel.h"

Go to the source code of this file.

Functions

 FILE_LICENCE (GPL2_OR_LATER_OR_UBDL)
 
static int arbel_bitmask_alloc (arbel_bitmask_t *bits, unsigned int bits_len)
 Allocate offset within usage bitmask. More...
 
static void arbel_bitmask_free (arbel_bitmask_t *bits, int bit)
 Free offset within usage bitmask. More...
 
static int arbel_cmd_wait (struct arbel *arbel, struct arbelprm_hca_command_register *hcr)
 Wait for Arbel command completion. More...
 
static int arbel_cmd (struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
 Issue HCA command. More...
 
static int arbel_cmd_query_dev_lim (struct arbel *arbel, struct arbelprm_query_dev_lim *dev_lim)
 
static int arbel_cmd_query_fw (struct arbel *arbel, struct arbelprm_query_fw *fw)
 
static int arbel_cmd_init_hca (struct arbel *arbel, const struct arbelprm_init_hca *init_hca)
 
static int arbel_cmd_close_hca (struct arbel *arbel)
 
static int arbel_cmd_init_ib (struct arbel *arbel, unsigned int port, const struct arbelprm_init_ib *init_ib)
 
static int arbel_cmd_close_ib (struct arbel *arbel, unsigned int port)
 
static int arbel_cmd_sw2hw_mpt (struct arbel *arbel, unsigned int index, const struct arbelprm_mpt *mpt)
 
static int arbel_cmd_map_eq (struct arbel *arbel, unsigned long index_map, const struct arbelprm_event_mask *mask)
 
static int arbel_cmd_sw2hw_eq (struct arbel *arbel, unsigned int index, const struct arbelprm_eqc *eqctx)
 
static int arbel_cmd_hw2sw_eq (struct arbel *arbel, unsigned int index, struct arbelprm_eqc *eqctx)
 
static int arbel_cmd_sw2hw_cq (struct arbel *arbel, unsigned long cqn, const struct arbelprm_completion_queue_context *cqctx)
 
static int arbel_cmd_hw2sw_cq (struct arbel *arbel, unsigned long cqn, struct arbelprm_completion_queue_context *cqctx)
 
static int arbel_cmd_query_cq (struct arbel *arbel, unsigned long cqn, struct arbelprm_completion_queue_context *cqctx)
 
static int arbel_cmd_rst2init_qpee (struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
 
static int arbel_cmd_init2rtr_qpee (struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
 
static int arbel_cmd_rtr2rts_qpee (struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
 
static int arbel_cmd_rts2rts_qpee (struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
 
static int arbel_cmd_2rst_qpee (struct arbel *arbel, unsigned long qpn)
 
static int arbel_cmd_query_qpee (struct arbel *arbel, unsigned long qpn, struct arbelprm_qp_ee_state_transitions *ctx)
 
static int arbel_cmd_conf_special_qp (struct arbel *arbel, unsigned int qp_type, unsigned long base_qpn)
 
static int arbel_cmd_mad_ifc (struct arbel *arbel, unsigned int port, union arbelprm_mad *mad)
 
static int arbel_cmd_read_mgm (struct arbel *arbel, unsigned int index, struct arbelprm_mgm_entry *mgm)
 
static int arbel_cmd_write_mgm (struct arbel *arbel, unsigned int index, const struct arbelprm_mgm_entry *mgm)
 
static int arbel_cmd_mgid_hash (struct arbel *arbel, const union ib_gid *gid, struct arbelprm_mgm_hash *hash)
 
static int arbel_cmd_run_fw (struct arbel *arbel)
 
static int arbel_cmd_disable_lam (struct arbel *arbel)
 
static int arbel_cmd_enable_lam (struct arbel *arbel, struct arbelprm_access_lam *lam)
 
static int arbel_cmd_unmap_icm (struct arbel *arbel, unsigned int page_count, const struct arbelprm_scalar_parameter *offset)
 
static int arbel_cmd_map_icm (struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *map)
 
static int arbel_cmd_unmap_icm_aux (struct arbel *arbel)
 
static int arbel_cmd_map_icm_aux (struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *map)
 
static int arbel_cmd_set_icm_size (struct arbel *arbel, const struct arbelprm_scalar_parameter *icm_size, struct arbelprm_scalar_parameter *icm_aux_size)
 
static int arbel_cmd_unmap_fa (struct arbel *arbel)
 
static int arbel_cmd_map_fa (struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *map)
 
static int arbel_mad (struct ib_device *ibdev, union ib_mad *mad)
 Issue management datagram. More...
 
static int arbel_dump_cqctx (struct arbel *arbel, struct ib_completion_queue *cq)
 Dump completion queue context (for debugging only) More...
 
static int arbel_create_cq (struct ib_device *ibdev, struct ib_completion_queue *cq)
 Create completion queue. More...
 
static void arbel_destroy_cq (struct ib_device *ibdev, struct ib_completion_queue *cq)
 Destroy completion queue. More...
 
static int arbel_alloc_qpn (struct ib_device *ibdev, struct ib_queue_pair *qp)
 Assign queue pair number. More...
 
static void arbel_free_qpn (struct ib_device *ibdev, struct ib_queue_pair *qp)
 Free queue pair number. More...
 
static unsigned int arbel_rate (struct ib_address_vector *av)
 Calculate transmission rate. More...
 
static int arbel_dump_qpctx (struct arbel *arbel, struct ib_queue_pair *qp)
 Dump queue pair context (for debugging only) More...
 
static int arbel_create_send_wq (struct arbel_send_work_queue *arbel_send_wq, unsigned int num_wqes)
 Create send work queue. More...
 
static int arbel_create_recv_wq (struct arbel_recv_work_queue *arbel_recv_wq, unsigned int num_wqes, enum ib_queue_pair_type type)
 Create receive work queue. More...
 
static int arbel_create_qp (struct ib_device *ibdev, struct ib_queue_pair *qp)
 Create queue pair. More...
 
static int arbel_modify_qp (struct ib_device *ibdev, struct ib_queue_pair *qp)
 Modify queue pair. More...
 
static void arbel_destroy_qp (struct ib_device *ibdev, struct ib_queue_pair *qp)
 Destroy queue pair. More...
 
static void arbel_ring_doorbell (struct arbel *arbel, union arbelprm_doorbell_register *db_reg, unsigned int offset)
 Ring doorbell register in UAR. More...
 
static size_t arbel_fill_ud_send_wqe (struct ib_device *ibdev, struct ib_queue_pair *qp __unused, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
 Construct UD send work queue entry. More...
 
static size_t arbel_fill_mlx_send_wqe (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
 Construct MLX send work queue entry. More...
 
static size_t arbel_fill_rc_send_wqe (struct ib_device *ibdev, struct ib_queue_pair *qp __unused, struct ib_address_vector *dest __unused, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
 Construct RC send work queue entry. More...
 
static int arbel_post_send (struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf)
 Post send work queue entry. More...
 
static int arbel_post_recv (struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf)
 Post receive work queue entry. More...
 
static int arbel_complete (struct ib_device *ibdev, struct ib_completion_queue *cq, union arbelprm_completion_entry *cqe)
 Handle completion. More...
 
static void arbel_poll_cq (struct ib_device *ibdev, struct ib_completion_queue *cq)
 Poll completion queue. More...
 
static int arbel_create_eq (struct arbel *arbel)
 Create event queue. More...
 
static void arbel_destroy_eq (struct arbel *arbel)
 Destroy event queue. More...
 
static void arbel_event_port_state_change (struct arbel *arbel, union arbelprm_event_entry *eqe)
 Handle port state event. More...
 
static void arbel_poll_eq (struct ib_device *ibdev)
 Poll event queue. More...
 
static int arbel_map_vpm (struct arbel *arbel, int(*map)(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *), uint64_t va, physaddr_t pa, size_t len)
 Map virtual to physical address for firmware usage. More...
 
static int arbel_start_firmware (struct arbel *arbel)
 Start firmware running. More...
 
static void arbel_stop_firmware (struct arbel *arbel)
 Stop firmware running. More...
 
static int arbel_get_limits (struct arbel *arbel)
 Get device limits. More...
 
static size_t icm_align (size_t icm_offset, size_t len)
 Align ICM table. More...
 
static int arbel_alloc_icm (struct arbel *arbel, struct arbelprm_init_hca *init_hca)
 Allocate ICM. More...
 
static void arbel_free_icm (struct arbel *arbel)
 Free ICM. More...
 
static void arbel_reset (struct arbel *arbel)
 Reset device. More...
 
static int arbel_setup_mpt (struct arbel *arbel)
 Set up memory protection table. More...
 
static int arbel_configure_special_qps (struct arbel *arbel)
 Configure special queue pairs. More...
 
static int arbel_start (struct arbel *arbel, int running)
 Start Arbel device. More...
 
static void arbel_stop (struct arbel *arbel)
 Stop Arbel device. More...
 
static int arbel_open (struct arbel *arbel)
 Open Arbel device. More...
 
static void arbel_close (struct arbel *arbel)
 Close Arbel device. More...
 
static int arbel_ib_open (struct ib_device *ibdev)
 Initialise Infiniband link. More...
 
static void arbel_ib_close (struct ib_device *ibdev)
 Close Infiniband link. More...
 
static int arbel_inform_sma (struct ib_device *ibdev, union ib_mad *mad)
 Inform embedded subnet management agent of a received MAD. More...
 
static int arbel_mcast_attach (struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
 Attach to multicast group. More...
 
static void arbel_mcast_detach (struct ib_device *ibdev, struct ib_queue_pair *qp __unused, union ib_gid *gid)
 Detach from multicast group. More...
 
static struct arbelarbel_alloc (void)
 Allocate Arbel device. More...
 
static void arbel_free (struct arbel *arbel)
 Free Arbel device. More...
 
static int arbel_probe (struct pci_device *pci)
 Probe PCI device. More...
 
static void arbel_remove (struct pci_device *pci)
 Remove PCI device. More...
 

Variables

static uint8_t arbel_qp_st []
 Queue pair transport service type map. More...
 
static const union ib_gid arbel_no_gid
 GID used for GID-less send work queue entries. More...
 
static size_t(* arbel_fill_send_wqe [])(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
 Work queue entry constructors. More...
 
static struct ib_device_operations arbel_ib_operations
 Arbel Infiniband operations. More...
 
static struct pci_device_id arbel_nics []
 
struct pci_driver arbel_driver __pci_driver
 

Detailed Description

Mellanox Arbel Infiniband HCA.

Definition in file arbel.c.

Function Documentation

◆ FILE_LICENCE()

FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL  )

◆ arbel_bitmask_alloc()

static int arbel_bitmask_alloc ( arbel_bitmask_t bits,
unsigned int  bits_len 
)
static

Allocate offset within usage bitmask.

Parameters
bitsUsage bitmask
bits_lenLength of usage bitmask
Return values
bitFirst free bit within bitmask, or negative error

Definition at line 69 of file arbel.c.

70  {
71  unsigned int bit = 0;
72  arbel_bitmask_t mask = 1;
73 
74  while ( bit < bits_len ) {
75  if ( ( mask & *bits ) == 0 ) {
76  *bits |= mask;
77  return bit;
78  }
79  bit++;
80  mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) );
81  if ( mask == 1 )
82  bits++;
83  }
84  return -ENFILE;
85 }
static unsigned int unsigned int bit
Definition: bigint.h:208
uint32_t arbel_bitmask_t
An Arbel resource bitmask.
Definition: arbel.h:461
#define ENFILE
Too many open files in system.
Definition: errno.h:493
static volatile void * bits
Definition: bitops.h:27

References bit, bits, and ENFILE.

Referenced by arbel_alloc_qpn(), and arbel_create_cq().

◆ arbel_bitmask_free()

static void arbel_bitmask_free ( arbel_bitmask_t bits,
int  bit 
)
static

Free offset within usage bitmask.

Parameters
bitsUsage bitmask
bitBit within bitmask

Definition at line 93 of file arbel.c.

93  {
94  arbel_bitmask_t mask;
95 
96  mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) );
97  bits += ( bit / ( 8 * sizeof ( mask ) ) );
98  *bits &= ~mask;
99 }
static unsigned int unsigned int bit
Definition: bigint.h:208
uint32_t arbel_bitmask_t
An Arbel resource bitmask.
Definition: arbel.h:461
static volatile void * bits
Definition: bitops.h:27

References bit, and bits.

Referenced by arbel_create_cq(), arbel_destroy_cq(), and arbel_free_qpn().

◆ arbel_cmd_wait()

static int arbel_cmd_wait ( struct arbel arbel,
struct arbelprm_hca_command_register *  hcr 
)
static

Wait for Arbel command completion.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 114 of file arbel.c.

115  {
116  unsigned int wait;
117 
118  for ( wait = ARBEL_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
119  hcr->u.dwords[6] =
120  readl ( arbel->config + ARBEL_HCR_REG ( 6 ) );
121  if ( MLX_GET ( hcr, go ) == 0 )
122  return 0;
123  mdelay ( 1 );
124  }
125  return -EBUSY;
126 }
#define EBUSY
Device or resource busy.
Definition: errno.h:338
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
void * config
PCI configuration registers.
Definition: arbel.h:473
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
#define ARBEL_HCR_MAX_WAIT_MS
Definition: arbel.h:548
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
#define ARBEL_HCR_REG(x)
Definition: arbel.h:547
An Arbel device.
Definition: arbel.h:469

References ARBEL_HCR_MAX_WAIT_MS, ARBEL_HCR_REG, arbel::config, EBUSY, mdelay(), MLX_GET, and readl().

Referenced by arbel_cmd().

◆ arbel_cmd()

static int arbel_cmd ( struct arbel arbel,
unsigned long  command,
unsigned int  op_mod,
const void *  in,
unsigned int  in_mod,
void *  out 
)
static

Issue HCA command.

Parameters
arbelArbel device
commandCommand opcode, flags and input/output lengths
op_modOpcode modifier (0 if no modifier applicable)
inInput parameters
in_modInput modifier (0 if no modifier applicable)
outOutput parameters
Return values
rcReturn status code

Definition at line 139 of file arbel.c.

141  {
142  struct arbelprm_hca_command_register hcr;
143  unsigned int opcode = ARBEL_HCR_OPCODE ( command );
144  size_t in_len = ARBEL_HCR_IN_LEN ( command );
145  size_t out_len = ARBEL_HCR_OUT_LEN ( command );
146  void *in_buffer;
147  void *out_buffer;
148  unsigned int status;
149  unsigned int i;
150  int rc;
151 
152  assert ( in_len <= ARBEL_MBOX_SIZE );
153  assert ( out_len <= ARBEL_MBOX_SIZE );
154 
155  DBGC2 ( arbel, "Arbel %p command %02x in %zx%s out %zx%s\n",
156  arbel, opcode, in_len,
157  ( ( command & ARBEL_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len,
158  ( ( command & ARBEL_HCR_OUT_MBOX ) ? "(mbox)" : "" ) );
159 
160  /* Check that HCR is free */
161  if ( ( rc = arbel_cmd_wait ( arbel, &hcr ) ) != 0 ) {
162  DBGC ( arbel, "Arbel %p command interface locked\n", arbel );
163  return rc;
164  }
165 
166  /* Prepare HCR */
167  memset ( &hcr, 0, sizeof ( hcr ) );
168  in_buffer = &hcr.u.dwords[0];
169  if ( in_len && ( command & ARBEL_HCR_IN_MBOX ) ) {
170  in_buffer = arbel->mailbox_in;
171  MLX_FILL_H ( &hcr, 0, in_param_h, virt_to_bus ( in_buffer ) );
172  MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) );
173  }
174  memcpy ( in_buffer, in, in_len );
175  MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
176  out_buffer = &hcr.u.dwords[3];
177  if ( out_len && ( command & ARBEL_HCR_OUT_MBOX ) ) {
178  out_buffer = arbel->mailbox_out;
179  MLX_FILL_H ( &hcr, 3, out_param_h,
180  virt_to_bus ( out_buffer ) );
181  MLX_FILL_1 ( &hcr, 4, out_param_l,
182  virt_to_bus ( out_buffer ) );
183  }
184  MLX_FILL_3 ( &hcr, 6,
185  opcode, opcode,
186  opcode_modifier, op_mod,
187  go, 1 );
188  DBGC ( arbel, "Arbel %p issuing command %04x\n", arbel, opcode );
190  &hcr, sizeof ( hcr ) );
191  if ( in_len && ( command & ARBEL_HCR_IN_MBOX ) ) {
192  DBGC2 ( arbel, "Input mailbox:\n" );
193  DBGC2_HDA ( arbel, virt_to_phys ( in_buffer ), in_buffer,
194  ( ( in_len < 512 ) ? in_len : 512 ) );
195  }
196 
197  /* Issue command */
198  for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
199  i++ ) {
200  writel ( hcr.u.dwords[i],
201  arbel->config + ARBEL_HCR_REG ( i ) );
202  barrier();
203  }
204 
205  /* Wait for command completion */
206  if ( ( rc = arbel_cmd_wait ( arbel, &hcr ) ) != 0 ) {
207  DBGC ( arbel, "Arbel %p timed out waiting for command:\n",
208  arbel );
209  DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
210  return rc;
211  }
212 
213  /* Check command status */
214  status = MLX_GET ( &hcr, status );
215  if ( status != 0 ) {
216  DBGC ( arbel, "Arbel %p command failed with status %02x:\n",
217  arbel, status );
218  DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
219  return -EIO;
220  }
221 
222  /* Read output parameters, if any */
223  hcr.u.dwords[3] = readl ( arbel->config + ARBEL_HCR_REG ( 3 ) );
224  hcr.u.dwords[4] = readl ( arbel->config + ARBEL_HCR_REG ( 4 ) );
225  memcpy ( out, out_buffer, out_len );
226  if ( out_len ) {
227  DBGC2 ( arbel, "Output%s:\n",
228  ( command & ARBEL_HCR_OUT_MBOX ) ? " mailbox" : "" );
229  DBGC2_HDA ( arbel, virt_to_phys ( out_buffer ), out_buffer,
230  ( ( out_len < 512 ) ? out_len : 512 ) );
231  }
232 
233  return 0;
234 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define ARBEL_HCR_OUT_LEN(_command)
Definition: arbel.h:567
__be32 in[4]
Definition: CIB_PRM.h:35
uint8_t opcode
Opcode.
Definition: ena.h:16
#define ARBEL_MBOX_SIZE
Definition: arbel.h:550
A command-line command.
Definition: command.h:9
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
void * mailbox_out
Command output mailbox.
Definition: arbel.h:482
#define DBGC(...)
Definition: compiler.h:505
void * config
PCI configuration registers.
Definition: arbel.h:473
static int arbel_cmd_wait(struct arbel *arbel, struct arbelprm_hca_command_register *hcr)
Wait for Arbel command completion.
Definition: arbel.c:114
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
__be32 out[4]
Definition: CIB_PRM.h:36
uint8_t status
Status.
Definition: ena.h:16
#define MLX_FILL_3(_ptr, _index,...)
Definition: mlx_bitops.h:175
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define ARBEL_HCR_BASE
Definition: arbel.h:546
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
#define DBGC2_HDA(...)
Definition: compiler.h:523
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
#define DBGC_HD(...)
Definition: compiler.h:507
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
#define ARBEL_HCR_IN_LEN(_command)
Definition: arbel.h:566
#define ARBEL_HCR_IN_MBOX
Definition: arbel.h:563
#define ARBEL_HCR_OPCODE(_command)
Definition: arbel.h:565
#define DBGC2(...)
Definition: compiler.h:522
#define ARBEL_HCR_REG(x)
Definition: arbel.h:547
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
#define EIO
Input/output error.
Definition: errno.h:433
#define ARBEL_HCR_OUT_MBOX
Definition: arbel.h:564
An Arbel device.
Definition: arbel.h:469
void * mailbox_in
Command input mailbox.
Definition: arbel.h:480
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_wait(), ARBEL_HCR_BASE, ARBEL_HCR_IN_LEN, ARBEL_HCR_IN_MBOX, ARBEL_HCR_OPCODE, ARBEL_HCR_OUT_LEN, ARBEL_HCR_OUT_MBOX, ARBEL_HCR_REG, ARBEL_MBOX_SIZE, assert(), barrier, arbel::config, DBGC, DBGC2, DBGC2_HDA, DBGC_HD, EIO, in, arbel::mailbox_in, arbel::mailbox_out, memcpy(), memset(), MLX_FILL_1, MLX_FILL_3, MLX_FILL_H, MLX_GET, opcode, out, rc, readl(), status, virt_to_bus(), virt_to_phys(), and writel().

Referenced by arbel_cmd_2rst_qpee(), arbel_cmd_close_hca(), arbel_cmd_close_ib(), arbel_cmd_conf_special_qp(), arbel_cmd_disable_lam(), arbel_cmd_enable_lam(), arbel_cmd_hw2sw_cq(), arbel_cmd_hw2sw_eq(), arbel_cmd_init2rtr_qpee(), arbel_cmd_init_hca(), arbel_cmd_init_ib(), arbel_cmd_mad_ifc(), arbel_cmd_map_eq(), arbel_cmd_map_fa(), arbel_cmd_map_icm(), arbel_cmd_map_icm_aux(), arbel_cmd_mgid_hash(), arbel_cmd_query_cq(), arbel_cmd_query_dev_lim(), arbel_cmd_query_fw(), arbel_cmd_query_qpee(), arbel_cmd_read_mgm(), arbel_cmd_rst2init_qpee(), arbel_cmd_rtr2rts_qpee(), arbel_cmd_rts2rts_qpee(), arbel_cmd_run_fw(), arbel_cmd_set_icm_size(), arbel_cmd_sw2hw_cq(), arbel_cmd_sw2hw_eq(), arbel_cmd_sw2hw_mpt(), arbel_cmd_unmap_fa(), arbel_cmd_unmap_icm(), arbel_cmd_unmap_icm_aux(), and arbel_cmd_write_mgm().

◆ arbel_cmd_query_dev_lim()

static int arbel_cmd_query_dev_lim ( struct arbel arbel,
struct arbelprm_query_dev_lim *  dev_lim 
)
inlinestatic

Definition at line 237 of file arbel.c.

238  {
239  return arbel_cmd ( arbel,
241  1, sizeof ( *dev_lim ) ),
242  0, NULL, 0, dev_lim );
243 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
#define ARBEL_HCR_QUERY_DEV_LIM
Definition: arbel.h:53
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_OUT_CMD, ARBEL_HCR_QUERY_DEV_LIM, and NULL.

Referenced by arbel_get_limits().

◆ arbel_cmd_query_fw()

static int arbel_cmd_query_fw ( struct arbel arbel,
struct arbelprm_query_fw *  fw 
)
inlinestatic

Definition at line 246 of file arbel.c.

246  {
247  return arbel_cmd ( arbel,
249  1, sizeof ( *fw ) ),
250  0, NULL, 0, fw );
251 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
#define ARBEL_HCR_QUERY_FW
Definition: arbel.h:54

References arbel_cmd(), ARBEL_HCR_OUT_CMD, ARBEL_HCR_QUERY_FW, and NULL.

Referenced by arbel_start_firmware().

◆ arbel_cmd_init_hca()

static int arbel_cmd_init_hca ( struct arbel arbel,
const struct arbelprm_init_hca *  init_hca 
)
inlinestatic

Definition at line 254 of file arbel.c.

255  {
256  return arbel_cmd ( arbel,
258  1, sizeof ( *init_hca ) ),
259  0, init_hca, 0, NULL );
260 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
#define ARBEL_HCR_INIT_HCA
Definition: arbel.h:55
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_INIT_HCA, and NULL.

Referenced by arbel_start().

◆ arbel_cmd_close_hca()

static int arbel_cmd_close_hca ( struct arbel arbel)
inlinestatic

Definition at line 263 of file arbel.c.

263  {
264  return arbel_cmd ( arbel,
266  0, NULL, 0, NULL );
267 }
#define ARBEL_HCR_CLOSE_HCA
Definition: arbel.h:56
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_CLOSE_HCA, ARBEL_HCR_VOID_CMD, and NULL.

Referenced by arbel_start(), and arbel_stop().

◆ arbel_cmd_init_ib()

static int arbel_cmd_init_ib ( struct arbel arbel,
unsigned int  port,
const struct arbelprm_init_ib *  init_ib 
)
inlinestatic

Definition at line 270 of file arbel.c.

271  {
272  return arbel_cmd ( arbel,
274  1, sizeof ( *init_ib ) ),
275  0, init_ib, port, NULL );
276 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
u8 port
Port number.
Definition: CIB_PRM.h:31
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
#define ARBEL_HCR_INIT_IB
Definition: arbel.h:57
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_INIT_IB, NULL, and port.

Referenced by arbel_ib_open().

◆ arbel_cmd_close_ib()

static int arbel_cmd_close_ib ( struct arbel arbel,
unsigned int  port 
)
inlinestatic

Definition at line 279 of file arbel.c.

279  {
280  return arbel_cmd ( arbel,
282  0, NULL, port, NULL );
283 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
u8 port
Port number.
Definition: CIB_PRM.h:31
#define ARBEL_HCR_CLOSE_IB
Definition: arbel.h:58
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_CLOSE_IB, ARBEL_HCR_VOID_CMD, NULL, and port.

Referenced by arbel_ib_close().

◆ arbel_cmd_sw2hw_mpt()

static int arbel_cmd_sw2hw_mpt ( struct arbel arbel,
unsigned int  index,
const struct arbelprm_mpt *  mpt 
)
inlinestatic

Definition at line 286 of file arbel.c.

287  {
288  return arbel_cmd ( arbel,
290  1, sizeof ( *mpt ) ),
291  0, mpt, index, NULL );
292 }
#define ARBEL_HCR_SW2HW_MPT
Definition: arbel.h:59
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_SW2HW_MPT, index, and NULL.

Referenced by arbel_setup_mpt().

◆ arbel_cmd_map_eq()

static int arbel_cmd_map_eq ( struct arbel arbel,
unsigned long  index_map,
const struct arbelprm_event_mask *  mask 
)
inlinestatic

Definition at line 295 of file arbel.c.

296  {
297  return arbel_cmd ( arbel,
299  0, sizeof ( *mask ) ),
300  0, mask, index_map, NULL );
301 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
#define ARBEL_HCR_MAP_EQ
Definition: arbel.h:60
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_MAP_EQ, and NULL.

Referenced by arbel_create_eq(), and arbel_destroy_eq().

◆ arbel_cmd_sw2hw_eq()

static int arbel_cmd_sw2hw_eq ( struct arbel arbel,
unsigned int  index,
const struct arbelprm_eqc *  eqctx 
)
inlinestatic

Definition at line 304 of file arbel.c.

305  {
306  return arbel_cmd ( arbel,
308  1, sizeof ( *eqctx ) ),
309  0, eqctx, index, NULL );
310 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
#define ARBEL_HCR_SW2HW_EQ
Definition: arbel.h:61
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_SW2HW_EQ, index, and NULL.

Referenced by arbel_create_eq().

◆ arbel_cmd_hw2sw_eq()

static int arbel_cmd_hw2sw_eq ( struct arbel arbel,
unsigned int  index,
struct arbelprm_eqc *  eqctx 
)
inlinestatic

Definition at line 313 of file arbel.c.

314  {
315  return arbel_cmd ( arbel,
317  1, sizeof ( *eqctx ) ),
318  1, NULL, index, eqctx );
319 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
#define ARBEL_HCR_HW2SW_EQ
Definition: arbel.h:62
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_HW2SW_EQ, ARBEL_HCR_OUT_CMD, index, and NULL.

Referenced by arbel_create_eq(), and arbel_destroy_eq().

◆ arbel_cmd_sw2hw_cq()

static int arbel_cmd_sw2hw_cq ( struct arbel arbel,
unsigned long  cqn,
const struct arbelprm_completion_queue_context *  cqctx 
)
inlinestatic

Definition at line 322 of file arbel.c.

323  {
324  return arbel_cmd ( arbel,
326  1, sizeof ( *cqctx ) ),
327  0, cqctx, cqn, NULL );
328 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
#define ARBEL_HCR_SW2HW_CQ
Definition: arbel.h:63
__be32 cqn
Definition: CIB_PRM.h:29
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_SW2HW_CQ, cqn, and NULL.

Referenced by arbel_create_cq().

◆ arbel_cmd_hw2sw_cq()

static int arbel_cmd_hw2sw_cq ( struct arbel arbel,
unsigned long  cqn,
struct arbelprm_completion_queue_context *  cqctx 
)
inlinestatic

Definition at line 331 of file arbel.c.

332  {
333  return arbel_cmd ( arbel,
335  1, sizeof ( *cqctx ) ),
336  0, NULL, cqn, cqctx );
337 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
#define ARBEL_HCR_HW2SW_CQ
Definition: arbel.h:64
__be32 cqn
Definition: CIB_PRM.h:29
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_HW2SW_CQ, ARBEL_HCR_OUT_CMD, cqn, and NULL.

Referenced by arbel_destroy_cq().

◆ arbel_cmd_query_cq()

static int arbel_cmd_query_cq ( struct arbel arbel,
unsigned long  cqn,
struct arbelprm_completion_queue_context *  cqctx 
)
inlinestatic

Definition at line 340 of file arbel.c.

341  {
342  return arbel_cmd ( arbel,
344  1, sizeof ( *cqctx ) ),
345  0, NULL, cqn, cqctx );
346 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
#define ARBEL_HCR_QUERY_CQ
Definition: arbel.h:65
__be32 cqn
Definition: CIB_PRM.h:29
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_OUT_CMD, ARBEL_HCR_QUERY_CQ, cqn, and NULL.

Referenced by arbel_dump_cqctx().

◆ arbel_cmd_rst2init_qpee()

static int arbel_cmd_rst2init_qpee ( struct arbel arbel,
unsigned long  qpn,
const struct arbelprm_qp_ee_state_transitions *  ctx 
)
inlinestatic

Definition at line 349 of file arbel.c.

350  {
351  return arbel_cmd ( arbel,
353  1, sizeof ( *ctx ) ),
354  0, ctx, qpn, NULL );
355 }
__be32 qpn
Definition: CIB_PRM.h:29
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
#define ARBEL_HCR_RST2INIT_QPEE
Definition: arbel.h:66
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_RST2INIT_QPEE, ctx, NULL, and qpn.

Referenced by arbel_create_qp().

◆ arbel_cmd_init2rtr_qpee()

static int arbel_cmd_init2rtr_qpee ( struct arbel arbel,
unsigned long  qpn,
const struct arbelprm_qp_ee_state_transitions *  ctx 
)
inlinestatic

Definition at line 358 of file arbel.c.

359  {
360  return arbel_cmd ( arbel,
362  1, sizeof ( *ctx ) ),
363  0, ctx, qpn, NULL );
364 }
#define ARBEL_HCR_INIT2RTR_QPEE
Definition: arbel.h:67
__be32 qpn
Definition: CIB_PRM.h:29
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_INIT2RTR_QPEE, ctx, NULL, and qpn.

Referenced by arbel_modify_qp().

◆ arbel_cmd_rtr2rts_qpee()

static int arbel_cmd_rtr2rts_qpee ( struct arbel arbel,
unsigned long  qpn,
const struct arbelprm_qp_ee_state_transitions *  ctx 
)
inlinestatic

Definition at line 367 of file arbel.c.

368  {
369  return arbel_cmd ( arbel,
371  1, sizeof ( *ctx ) ),
372  0, ctx, qpn, NULL );
373 }
__be32 qpn
Definition: CIB_PRM.h:29
#define ARBEL_HCR_RTR2RTS_QPEE
Definition: arbel.h:68
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_RTR2RTS_QPEE, ctx, NULL, and qpn.

Referenced by arbel_modify_qp().

◆ arbel_cmd_rts2rts_qpee()

static int arbel_cmd_rts2rts_qpee ( struct arbel arbel,
unsigned long  qpn,
const struct arbelprm_qp_ee_state_transitions *  ctx 
)
inlinestatic

Definition at line 376 of file arbel.c.

377  {
378  return arbel_cmd ( arbel,
380  1, sizeof ( *ctx ) ),
381  0, ctx, qpn, NULL );
382 }
#define ARBEL_HCR_RTS2RTS_QPEE
Definition: arbel.h:69
__be32 qpn
Definition: CIB_PRM.h:29
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_RTS2RTS_QPEE, ctx, NULL, and qpn.

Referenced by arbel_modify_qp().

◆ arbel_cmd_2rst_qpee()

static int arbel_cmd_2rst_qpee ( struct arbel arbel,
unsigned long  qpn 
)
inlinestatic

Definition at line 385 of file arbel.c.

385  {
386  return arbel_cmd ( arbel,
388  0x03, NULL, qpn, NULL );
389 }
__be32 qpn
Definition: CIB_PRM.h:29
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_2RST_QPEE
Definition: arbel.h:70
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_2RST_QPEE, ARBEL_HCR_VOID_CMD, NULL, and qpn.

Referenced by arbel_create_qp(), and arbel_destroy_qp().

◆ arbel_cmd_query_qpee()

static int arbel_cmd_query_qpee ( struct arbel arbel,
unsigned long  qpn,
struct arbelprm_qp_ee_state_transitions *  ctx 
)
inlinestatic

Definition at line 392 of file arbel.c.

393  {
394  return arbel_cmd ( arbel,
396  1, sizeof ( *ctx ) ),
397  0, NULL, qpn, ctx );
398 }
#define ARBEL_HCR_QUERY_QPEE
Definition: arbel.h:71
__be32 qpn
Definition: CIB_PRM.h:29
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
struct golan_eq_context ctx
Definition: CIB_PRM.h:28
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_OUT_CMD, ARBEL_HCR_QUERY_QPEE, ctx, NULL, and qpn.

Referenced by arbel_dump_qpctx().

◆ arbel_cmd_conf_special_qp()

static int arbel_cmd_conf_special_qp ( struct arbel arbel,
unsigned int  qp_type,
unsigned long  base_qpn 
)
inlinestatic

Definition at line 401 of file arbel.c.

402  {
403  return arbel_cmd ( arbel,
405  qp_type, NULL, base_qpn, NULL );
406 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define ARBEL_HCR_CONF_SPECIAL_QP
Definition: arbel.h:72
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_CONF_SPECIAL_QP, ARBEL_HCR_VOID_CMD, and NULL.

Referenced by arbel_configure_special_qps().

◆ arbel_cmd_mad_ifc()

static int arbel_cmd_mad_ifc ( struct arbel arbel,
unsigned int  port,
union arbelprm_mad mad 
)
inlinestatic

Definition at line 409 of file arbel.c.

410  {
411  return arbel_cmd ( arbel,
413  1, sizeof ( *mad ),
414  1, sizeof ( *mad ) ),
415  0x03, mad, port, mad );
416 }
#define ARBEL_HCR_INOUT_CMD(_opcode, _in_mbox, _in_len, _out_mbox, _out_len)
Build HCR command from component parts.
Definition: arbel.h:570
#define ARBEL_HCR_MAD_IFC
Definition: arbel.h:73
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
u8 port
Port number.
Definition: CIB_PRM.h:31
An Arbel device.
Definition: arbel.h:469
union ib_mad mad
Definition: arbel.h:12

References arbel_cmd(), ARBEL_HCR_INOUT_CMD, ARBEL_HCR_MAD_IFC, mad, and port.

Referenced by arbel_mad().

◆ arbel_cmd_read_mgm()

static int arbel_cmd_read_mgm ( struct arbel arbel,
unsigned int  index,
struct arbelprm_mgm_entry *  mgm 
)
inlinestatic

Definition at line 419 of file arbel.c.

420  {
421  return arbel_cmd ( arbel,
423  1, sizeof ( *mgm ) ),
424  0, NULL, index, mgm );
425 }
#define ARBEL_HCR_READ_MGM
Definition: arbel.h:74
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_OUT_CMD, ARBEL_HCR_READ_MGM, index, and NULL.

Referenced by arbel_mcast_attach().

◆ arbel_cmd_write_mgm()

static int arbel_cmd_write_mgm ( struct arbel arbel,
unsigned int  index,
const struct arbelprm_mgm_entry *  mgm 
)
inlinestatic

Definition at line 428 of file arbel.c.

429  {
430  return arbel_cmd ( arbel,
432  1, sizeof ( *mgm ) ),
433  0, mgm, index, NULL );
434 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
#define ARBEL_HCR_WRITE_MGM
Definition: arbel.h:75
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_WRITE_MGM, index, and NULL.

Referenced by arbel_mcast_attach(), and arbel_mcast_detach().

◆ arbel_cmd_mgid_hash()

static int arbel_cmd_mgid_hash ( struct arbel arbel,
const union ib_gid gid,
struct arbelprm_mgm_hash *  hash 
)
inlinestatic

Definition at line 437 of file arbel.c.

438  {
439  return arbel_cmd ( arbel,
441  1, sizeof ( *gid ),
442  0, sizeof ( *hash ) ),
443  0, gid, 0, hash );
444 }
#define ARBEL_HCR_INOUT_CMD(_opcode, _in_mbox, _in_len, _out_mbox, _out_len)
Build HCR command from component parts.
Definition: arbel.h:570
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
pseudo_bit_t hash[0x00010]
Hash algorithm.
Definition: arbel.h:13
#define ARBEL_HCR_MGID_HASH
Definition: arbel.h:76
u8 gid[16]
Definition: CIB_PRM.h:31
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_INOUT_CMD, ARBEL_HCR_MGID_HASH, gid, and hash.

Referenced by arbel_mcast_attach(), and arbel_mcast_detach().

◆ arbel_cmd_run_fw()

static int arbel_cmd_run_fw ( struct arbel arbel)
inlinestatic

Definition at line 447 of file arbel.c.

447  {
448  return arbel_cmd ( arbel,
450  0, NULL, 0, NULL );
451 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_RUN_FW
Definition: arbel.h:77
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_RUN_FW, ARBEL_HCR_VOID_CMD, and NULL.

Referenced by arbel_start_firmware().

◆ arbel_cmd_disable_lam()

static int arbel_cmd_disable_lam ( struct arbel arbel)
inlinestatic

Definition at line 454 of file arbel.c.

454  {
455  return arbel_cmd ( arbel,
457  0, NULL, 0, NULL );
458 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define ARBEL_HCR_DISABLE_LAM
Definition: arbel.h:78
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_DISABLE_LAM, ARBEL_HCR_VOID_CMD, and NULL.

◆ arbel_cmd_enable_lam()

static int arbel_cmd_enable_lam ( struct arbel arbel,
struct arbelprm_access_lam *  lam 
)
inlinestatic

Definition at line 461 of file arbel.c.

461  {
462  return arbel_cmd ( arbel,
464  1, sizeof ( *lam ) ),
465  1, NULL, 0, lam );
466 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_OUT_CMD(_opcode, _out_mbox, _out_len)
Definition: arbel.h:581
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
#define ARBEL_HCR_ENABLE_LAM
Definition: arbel.h:79

References arbel_cmd(), ARBEL_HCR_ENABLE_LAM, ARBEL_HCR_OUT_CMD, and NULL.

Referenced by arbel_start_firmware().

◆ arbel_cmd_unmap_icm()

static int arbel_cmd_unmap_icm ( struct arbel arbel,
unsigned int  page_count,
const struct arbelprm_scalar_parameter *  offset 
)
inlinestatic

Definition at line 469 of file arbel.c.

470  {
471  return arbel_cmd ( arbel,
473  sizeof ( *offset ) ),
474  0, offset, page_count, NULL );
475 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
#define ARBEL_HCR_UNMAP_ICM
Definition: arbel.h:80
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_UNMAP_ICM, NULL, and offset.

Referenced by arbel_alloc_icm(), and arbel_free_icm().

◆ arbel_cmd_map_icm()

static int arbel_cmd_map_icm ( struct arbel arbel,
const struct arbelprm_virtual_physical_mapping *  map 
)
inlinestatic

Definition at line 478 of file arbel.c.

479  {
480  return arbel_cmd ( arbel,
482  1, sizeof ( *map ) ),
483  0, map, 1, NULL );
484 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
static __always_inline int struct dma_mapping * map
Definition: dma.h:181
#define ARBEL_HCR_MAP_ICM
Definition: arbel.h:81
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_MAP_ICM, map, and NULL.

Referenced by arbel_alloc_icm().

◆ arbel_cmd_unmap_icm_aux()

static int arbel_cmd_unmap_icm_aux ( struct arbel arbel)
inlinestatic

Definition at line 487 of file arbel.c.

487  {
488  return arbel_cmd ( arbel,
490  0, NULL, 0, NULL );
491 }
#define ARBEL_HCR_UNMAP_ICM_AUX
Definition: arbel.h:82
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_UNMAP_ICM_AUX, ARBEL_HCR_VOID_CMD, and NULL.

Referenced by arbel_alloc_icm(), and arbel_free_icm().

◆ arbel_cmd_map_icm_aux()

static int arbel_cmd_map_icm_aux ( struct arbel arbel,
const struct arbelprm_virtual_physical_mapping *  map 
)
inlinestatic

Definition at line 494 of file arbel.c.

495  {
496  return arbel_cmd ( arbel,
498  1, sizeof ( *map ) ),
499  0, map, 1, NULL );
500 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
static __always_inline int struct dma_mapping * map
Definition: dma.h:181
#define ARBEL_HCR_MAP_ICM_AUX
Definition: arbel.h:83
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_MAP_ICM_AUX, map, and NULL.

Referenced by arbel_alloc_icm().

◆ arbel_cmd_set_icm_size()

static int arbel_cmd_set_icm_size ( struct arbel arbel,
const struct arbelprm_scalar_parameter *  icm_size,
struct arbelprm_scalar_parameter *  icm_aux_size 
)
inlinestatic

Definition at line 503 of file arbel.c.

505  {
506  return arbel_cmd ( arbel,
508  0, sizeof ( *icm_size ),
509  0, sizeof ( *icm_aux_size ) ),
510  0, icm_size, 0, icm_aux_size );
511 }
#define ARBEL_HCR_INOUT_CMD(_opcode, _in_mbox, _in_len, _out_mbox, _out_len)
Build HCR command from component parts.
Definition: arbel.h:570
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_SET_ICM_SIZE
Definition: arbel.h:84
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_INOUT_CMD, and ARBEL_HCR_SET_ICM_SIZE.

Referenced by arbel_alloc_icm().

◆ arbel_cmd_unmap_fa()

static int arbel_cmd_unmap_fa ( struct arbel arbel)
inlinestatic

Definition at line 514 of file arbel.c.

514  {
515  return arbel_cmd ( arbel,
517  0, NULL, 0, NULL );
518 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_UNMAP_FA
Definition: arbel.h:85
#define ARBEL_HCR_VOID_CMD(_opcode)
Definition: arbel.h:584
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_UNMAP_FA, ARBEL_HCR_VOID_CMD, and NULL.

Referenced by arbel_start_firmware(), and arbel_stop_firmware().

◆ arbel_cmd_map_fa()

static int arbel_cmd_map_fa ( struct arbel arbel,
const struct arbelprm_virtual_physical_mapping *  map 
)
inlinestatic

Definition at line 521 of file arbel.c.

522  {
523  return arbel_cmd ( arbel,
525  1, sizeof ( *map ) ),
526  0, map, 1, NULL );
527 }
static int arbel_cmd(struct arbel *arbel, unsigned long command, unsigned int op_mod, const void *in, unsigned int in_mod, void *out)
Issue HCA command.
Definition: arbel.c:139
#define ARBEL_HCR_IN_CMD(_opcode, _in_mbox, _in_len)
Definition: arbel.h:578
static __always_inline int struct dma_mapping * map
Definition: dma.h:181
#define ARBEL_HCR_MAP_FA
Definition: arbel.h:86
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_cmd(), ARBEL_HCR_IN_CMD, ARBEL_HCR_MAP_FA, map, and NULL.

Referenced by arbel_start_firmware().

◆ arbel_mad()

static int arbel_mad ( struct ib_device ibdev,
union ib_mad mad 
)
static

Issue management datagram.

Parameters
ibdevInfiniband device
madManagement datagram
Return values
rcReturn status code

Definition at line 543 of file arbel.c.

543  {
544  struct arbel *arbel = ib_get_drvdata ( ibdev );
545  union arbelprm_mad mad_ifc;
546  int rc;
547 
548  /* Sanity check */
549  static_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ) );
550 
551  /* Copy in request packet */
552  memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
553 
554  /* Issue MAD */
555  if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
556  &mad_ifc ) ) != 0 ) {
557  DBGC ( arbel, "Arbel %p port %d could not issue MAD IFC: %s\n",
558  arbel, ibdev->port, strerror ( rc ) );
559  return rc;
560  }
561 
562  /* Copy out reply packet */
563  memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
564 
565  if ( mad->hdr.status != 0 ) {
566  DBGC ( arbel, "Arbel %p port %d MAD IFC status %04x\n",
567  arbel, ibdev->port, ntohs ( mad->hdr.status ) );
568  return -EIO;
569  }
570  return 0;
571 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
#define ntohs(value)
Definition: byteswap.h:136
#define static_assert(x)
Assert a condition at build time.
Definition: assert.h:65
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static int arbel_cmd_mad_ifc(struct arbel *arbel, unsigned int port, union arbelprm_mad *mad)
Definition: arbel.c:409
unsigned int port
Port number.
Definition: infiniband.h:418
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
struct ib_mad_hdr hdr
Definition: ib_mad.h:611
#define EIO
Input/output error.
Definition: errno.h:433
An Arbel device.
Definition: arbel.h:469
union ib_mad mad
Definition: arbel.h:12
uint16_t status
Definition: ib_mad.h:543

References arbel_cmd_mad_ifc(), DBGC, EIO, ib_mad::hdr, ib_get_drvdata(), arbel::ibdev, mad, arbelprm_mad::mad, memcpy(), ntohs, ib_device::port, rc, static_assert, ib_mad_hdr::status, and strerror().

Referenced by arbel_event_port_state_change(), arbel_ib_open(), arbel_inform_sma(), arbel_poll_eq(), and arbel_probe().

◆ arbel_dump_cqctx()

static int arbel_dump_cqctx ( struct arbel arbel,
struct ib_completion_queue cq 
)
static

Dump completion queue context (for debugging only)

Parameters
arbelArbel device
cqCompletion queue
Return values
rcReturn status code

Definition at line 588 of file arbel.c.

588  {
589  struct arbelprm_completion_queue_context cqctx;
590  int rc;
591 
592  memset ( &cqctx, 0, sizeof ( cqctx ) );
593  if ( ( rc = arbel_cmd_query_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
594  DBGC ( arbel, "Arbel %p CQN %#lx QUERY_CQ failed: %s\n",
595  arbel, cq->cqn, strerror ( rc ) );
596  return rc;
597  }
598  DBGC ( arbel, "Arbel %p CQN %#lx context:\n", arbel, cq->cqn );
599  DBGC_HDA ( arbel, 0, &cqctx, sizeof ( cqctx ) );
600 
601  return 0;
602 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_cmd_query_cq(struct arbel *arbel, unsigned long cqn, struct arbelprm_completion_queue_context *cqctx)
Definition: arbel.c:340
#define DBGC(...)
Definition: compiler.h:505
#define DBGC_HDA(...)
Definition: compiler.h:506
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
unsigned long cqn
Completion queue number.
Definition: infiniband.h:230
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_query_cq(), ib_completion_queue::cqn, DBGC, DBGC_HDA, memset(), rc, and strerror().

◆ arbel_create_cq()

static int arbel_create_cq ( struct ib_device ibdev,
struct ib_completion_queue cq 
)
static

Create completion queue.

Parameters
ibdevInfiniband device
cqCompletion queue
Return values
rcReturn status code

Definition at line 611 of file arbel.c.

612  {
613  struct arbel *arbel = ib_get_drvdata ( ibdev );
614  struct arbel_completion_queue *arbel_cq;
615  struct arbelprm_completion_queue_context cqctx;
616  struct arbelprm_cq_ci_db_record *ci_db_rec;
617  struct arbelprm_cq_arm_db_record *arm_db_rec;
618  int cqn_offset;
619  unsigned int i;
620  int rc;
621 
622  /* Find a free completion queue number */
623  cqn_offset = arbel_bitmask_alloc ( arbel->cq_inuse, ARBEL_MAX_CQS );
624  if ( cqn_offset < 0 ) {
625  DBGC ( arbel, "Arbel %p out of completion queues\n", arbel );
626  rc = cqn_offset;
627  goto err_cqn_offset;
628  }
629  cq->cqn = ( arbel->limits.reserved_cqs + cqn_offset );
630 
631  /* Allocate control structures */
632  arbel_cq = zalloc ( sizeof ( *arbel_cq ) );
633  if ( ! arbel_cq ) {
634  rc = -ENOMEM;
635  goto err_arbel_cq;
636  }
637  arbel_cq->ci_doorbell_idx = arbel_cq_ci_doorbell_idx ( arbel, cq );
639 
640  /* Allocate completion queue itself */
641  arbel_cq->cqe_size = ( cq->num_cqes * sizeof ( arbel_cq->cqe[0] ) );
642  arbel_cq->cqe = malloc_phys ( arbel_cq->cqe_size,
643  sizeof ( arbel_cq->cqe[0] ) );
644  if ( ! arbel_cq->cqe ) {
645  rc = -ENOMEM;
646  goto err_cqe;
647  }
648  memset ( arbel_cq->cqe, 0, arbel_cq->cqe_size );
649  for ( i = 0 ; i < cq->num_cqes ; i++ ) {
650  MLX_FILL_1 ( &arbel_cq->cqe[i].normal, 7, owner, 1 );
651  }
652  barrier();
653 
654  /* Initialise doorbell records */
655  ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
656  MLX_FILL_1 ( ci_db_rec, 0, counter, 0 );
657  MLX_FILL_2 ( ci_db_rec, 1,
658  res, ARBEL_UAR_RES_CQ_CI,
659  cq_number, cq->cqn );
660  arm_db_rec = &arbel->db_rec[arbel_cq->arm_doorbell_idx].cq_arm;
661  MLX_FILL_1 ( arm_db_rec, 0, counter, 0 );
662  MLX_FILL_2 ( arm_db_rec, 1,
664  cq_number, cq->cqn );
665 
666  /* Hand queue over to hardware */
667  memset ( &cqctx, 0, sizeof ( cqctx ) );
668  MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
669  MLX_FILL_H ( &cqctx, 1, start_address_h,
670  virt_to_bus ( arbel_cq->cqe ) );
671  MLX_FILL_1 ( &cqctx, 2, start_address_l,
672  virt_to_bus ( arbel_cq->cqe ) );
673  MLX_FILL_2 ( &cqctx, 3,
674  usr_page, arbel->limits.reserved_uars,
675  log_cq_size, fls ( cq->num_cqes - 1 ) );
676  MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eq.eqn );
677  MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
678  MLX_FILL_1 ( &cqctx, 7, l_key, arbel->lkey );
679  MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn );
680  MLX_FILL_1 ( &cqctx, 13,
681  cq_ci_db_record, arbel_cq->ci_doorbell_idx );
682  MLX_FILL_1 ( &cqctx, 14,
683  cq_state_db_record, arbel_cq->arm_doorbell_idx );
684  if ( ( rc = arbel_cmd_sw2hw_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
685  DBGC ( arbel, "Arbel %p CQN %#lx SW2HW_CQ failed: %s\n",
686  arbel, cq->cqn, strerror ( rc ) );
687  goto err_sw2hw_cq;
688  }
689 
690  DBGC ( arbel, "Arbel %p CQN %#lx ring [%08lx,%08lx), doorbell %08lx\n",
691  arbel, cq->cqn, virt_to_phys ( arbel_cq->cqe ),
692  ( virt_to_phys ( arbel_cq->cqe ) + arbel_cq->cqe_size ),
693  virt_to_phys ( ci_db_rec ) );
694  ib_cq_set_drvdata ( cq, arbel_cq );
695  return 0;
696 
697  err_sw2hw_cq:
698  MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
699  MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
700  free_phys ( arbel_cq->cqe, arbel_cq->cqe_size );
701  err_cqe:
702  free ( arbel_cq );
703  err_arbel_cq:
704  arbel_bitmask_free ( arbel->cq_inuse, cqn_offset );
705  err_cqn_offset:
706  return rc;
707 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
arbel_bitmask_t cq_inuse[ARBEL_BITMASK_SIZE(ARBEL_MAX_CQS)]
Completion queue in-use bitmask.
Definition: arbel.h:520
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
static unsigned int arbel_cq_arm_doorbell_idx(struct arbel *arbel, struct ib_completion_queue *cq)
Get arm completion queue doorbell index.
Definition: arbel.h:612
#define ARBEL_UAR_RES_CQ_CI
Definition: arbel.h:41
u8 owner
Definition: CIB_PRM.h:36
static unsigned int arbel_cq_ci_doorbell_idx(struct arbel *arbel, struct ib_completion_queue *cq)
Get completion queue consumer counter doorbell index.
Definition: arbel.h:653
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define ARBEL_UAR_RES_NONE
Definition: arbel.h:40
#define DBGC(...)
Definition: compiler.h:505
#define ARBEL_GLOBAL_PD
Global protection domain.
Definition: arbel.h:536
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
#define ARBEL_UAR_RES_CQ_ARM
Definition: arbel.h:42
unsigned long eqn
Event queue number.
Definition: arbel.h:446
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
unsigned int ci_doorbell_idx
Consumer counter doorbell record number.
Definition: arbel.h:424
struct arbel_event_queue eq
Event queue.
Definition: arbel.h:512
struct sockaddr_tcpip st
Definition: syslog.c:56
#define ENOMEM
Not enough space.
Definition: errno.h:534
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
union arbelprm_completion_entry * cqe
Completion queue entries.
Definition: arbel.h:428
struct arbelprm_cq_ci_db_record cq_ci
Definition: arbel.h:267
size_t cqe_size
Size of completion queue.
Definition: arbel.h:430
unsigned int arm_doorbell_idx
Arm queue doorbell record number.
Definition: arbel.h:426
An Arbel completion queue.
Definition: arbel.h:422
static __always_inline void ib_cq_set_drvdata(struct ib_completion_queue *cq, void *priv)
Set Infiniband completion queue driver-private data.
Definition: infiniband.h:675
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
#define ARBEL_MAX_CQS
Maximum number of allocatable completion queues.
Definition: arbel.h:419
struct arbelprm_completion_queue_entry normal
Definition: arbel.h:256
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:624
unsigned int reserved_cqs
Number of reserved CQs.
Definition: arbel.h:310
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
__be16 c_eqn
Definition: CIB_PRM.h:38
static void arbel_bitmask_free(arbel_bitmask_t *bits, int bit)
Free offset within usage bitmask.
Definition: arbel.c:93
static int arbel_bitmask_alloc(arbel_bitmask_t *bits, unsigned int bits_len)
Allocate offset within usage bitmask.
Definition: arbel.c:69
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
unsigned long cqn
Completion queue number.
Definition: infiniband.h:230
unsigned int num_cqes
Number of completion queue entries.
Definition: infiniband.h:232
static int arbel_cmd_sw2hw_cq(struct arbel *arbel, unsigned long cqn, const struct arbelprm_completion_queue_context *cqctx)
Definition: arbel.c:322
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
unsigned int reserved_uars
Number of reserved UARs.
Definition: arbel.h:328
__be32 cqn
Definition: CIB_PRM.h:29
#define fls(x)
Find last (i.e.
Definition: strings.h:166
struct arbelprm_cq_arm_db_record cq_arm
Definition: arbel.h:266
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_bitmask_alloc(), arbel_bitmask_free(), arbel_cmd_sw2hw_cq(), arbel_cq_arm_doorbell_idx(), arbel_cq_ci_doorbell_idx(), ARBEL_GLOBAL_PD, ARBEL_MAX_CQS, ARBEL_UAR_RES_CQ_ARM, ARBEL_UAR_RES_CQ_CI, ARBEL_UAR_RES_NONE, arbel_completion_queue::arm_doorbell_idx, barrier, c_eqn, arbel_completion_queue::ci_doorbell_idx, arbelprm_doorbell_record::cq_arm, arbelprm_doorbell_record::cq_ci, arbel::cq_inuse, arbel_completion_queue::cqe, arbel_completion_queue::cqe_size, cqn, ib_completion_queue::cqn, arbel::db_rec, DBGC, ENOMEM, arbel::eq, arbel_event_queue::eqn, fls, free, free_phys(), ib_cq_set_drvdata(), ib_get_drvdata(), arbel::ibdev, arbel::limits, arbel::lkey, malloc_phys(), memset(), MLX_FILL_1, MLX_FILL_2, MLX_FILL_H, arbelprm_completion_entry::normal, ib_completion_queue::num_cqes, owner, rc, arbel_dev_limits::reserved_cqs, arbel_dev_limits::reserved_uars, st, strerror(), virt_to_bus(), virt_to_phys(), and zalloc().

◆ arbel_destroy_cq()

static void arbel_destroy_cq ( struct ib_device ibdev,
struct ib_completion_queue cq 
)
static

Destroy completion queue.

Parameters
ibdevInfiniband device
cqCompletion queue

Definition at line 715 of file arbel.c.

716  {
717  struct arbel *arbel = ib_get_drvdata ( ibdev );
718  struct arbel_completion_queue *arbel_cq = ib_cq_get_drvdata ( cq );
719  struct arbelprm_completion_queue_context cqctx;
720  struct arbelprm_cq_ci_db_record *ci_db_rec;
721  struct arbelprm_cq_arm_db_record *arm_db_rec;
722  int cqn_offset;
723  int rc;
724 
725  /* Take ownership back from hardware */
726  if ( ( rc = arbel_cmd_hw2sw_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
727  DBGC ( arbel, "Arbel %p CQN %#lx FATAL HW2SW_CQ failed: "
728  "%s\n", arbel, cq->cqn, strerror ( rc ) );
729  /* Leak memory and return; at least we avoid corruption */
730  return;
731  }
732 
733  /* Clear doorbell records */
734  ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
735  arm_db_rec = &arbel->db_rec[arbel_cq->arm_doorbell_idx].cq_arm;
736  MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
737  MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
738 
739  /* Free memory */
740  free_phys ( arbel_cq->cqe, arbel_cq->cqe_size );
741  free ( arbel_cq );
742 
743  /* Mark queue number as free */
744  cqn_offset = ( cq->cqn - arbel->limits.reserved_cqs );
745  arbel_bitmask_free ( arbel->cq_inuse, cqn_offset );
746 
747  ib_cq_set_drvdata ( cq, NULL );
748 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
arbel_bitmask_t cq_inuse[ARBEL_BITMASK_SIZE(ARBEL_MAX_CQS)]
Completion queue in-use bitmask.
Definition: arbel.h:520
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
static int arbel_cmd_hw2sw_cq(struct arbel *arbel, unsigned long cqn, struct arbelprm_completion_queue_context *cqctx)
Definition: arbel.c:331
#define ARBEL_UAR_RES_NONE
Definition: arbel.h:40
#define DBGC(...)
Definition: compiler.h:505
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
unsigned int ci_doorbell_idx
Consumer counter doorbell record number.
Definition: arbel.h:424
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
union arbelprm_completion_entry * cqe
Completion queue entries.
Definition: arbel.h:428
struct arbelprm_cq_ci_db_record cq_ci
Definition: arbel.h:267
size_t cqe_size
Size of completion queue.
Definition: arbel.h:430
unsigned int arm_doorbell_idx
Arm queue doorbell record number.
Definition: arbel.h:426
An Arbel completion queue.
Definition: arbel.h:422
static __always_inline void ib_cq_set_drvdata(struct ib_completion_queue *cq, void *priv)
Set Infiniband completion queue driver-private data.
Definition: infiniband.h:675
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
unsigned int reserved_cqs
Number of reserved CQs.
Definition: arbel.h:310
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
static void arbel_bitmask_free(arbel_bitmask_t *bits, int bit)
Free offset within usage bitmask.
Definition: arbel.c:93
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
unsigned long cqn
Completion queue number.
Definition: infiniband.h:230
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
struct arbelprm_cq_arm_db_record cq_arm
Definition: arbel.h:266
static __always_inline void * ib_cq_get_drvdata(struct ib_completion_queue *cq)
Get Infiniband completion queue driver-private data.
Definition: infiniband.h:686
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469

References arbel_bitmask_free(), arbel_cmd_hw2sw_cq(), ARBEL_UAR_RES_NONE, arbel_completion_queue::arm_doorbell_idx, arbel_completion_queue::ci_doorbell_idx, arbelprm_doorbell_record::cq_arm, arbelprm_doorbell_record::cq_ci, arbel::cq_inuse, arbel_completion_queue::cqe, arbel_completion_queue::cqe_size, ib_completion_queue::cqn, arbel::db_rec, DBGC, free, free_phys(), ib_cq_get_drvdata(), ib_cq_set_drvdata(), ib_get_drvdata(), arbel::ibdev, arbel::limits, MLX_FILL_1, NULL, rc, arbel_dev_limits::reserved_cqs, and strerror().

◆ arbel_alloc_qpn()

static int arbel_alloc_qpn ( struct ib_device ibdev,
struct ib_queue_pair qp 
)
static

Assign queue pair number.

Parameters
ibdevInfiniband device
qpQueue pair
Return values
rcReturn status code

Definition at line 764 of file arbel.c.

765  {
766  struct arbel *arbel = ib_get_drvdata ( ibdev );
767  unsigned int port_offset;
768  int qpn_offset;
769 
770  /* Calculate queue pair number */
771  port_offset = ( ibdev->port - ARBEL_PORT_BASE );
772 
773  switch ( qp->type ) {
774  case IB_QPT_SMI:
775  qp->qpn = ( arbel->special_qpn_base + port_offset );
776  return 0;
777  case IB_QPT_GSI:
778  qp->qpn = ( arbel->special_qpn_base + 2 + port_offset );
779  return 0;
780  case IB_QPT_UD:
781  case IB_QPT_RC:
782  /* Find a free queue pair number */
783  qpn_offset = arbel_bitmask_alloc ( arbel->qp_inuse,
784  ARBEL_MAX_QPS );
785  if ( qpn_offset < 0 ) {
786  DBGC ( arbel, "Arbel %p out of queue pairs\n",
787  arbel );
788  return qpn_offset;
789  }
790  qp->qpn = ( ( random() & ARBEL_QPN_RANDOM_MASK ) |
791  ( arbel->qpn_base + qpn_offset ) );
792  return 0;
793  default:
794  DBGC ( arbel, "Arbel %p unsupported QP type %d\n",
795  arbel, qp->type );
796  return -ENOTSUP;
797  }
798 }
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
#define ENOTSUP
Operation not supported.
Definition: errno.h:589
#define ARBEL_QPN_RANDOM_MASK
Queue pair number randomisation mask.
Definition: arbel.h:395
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
unsigned int port
Port number.
Definition: infiniband.h:418
long int random(void)
Generate a pseudo-random number between 0 and 2147483647L or 2147483562?
Definition: random.c:31
#define ARBEL_PORT_BASE
Definition: arbel.h:25
static int arbel_bitmask_alloc(arbel_bitmask_t *bits, unsigned int bits_len)
Allocate offset within usage bitmask.
Definition: arbel.c:69
unsigned long qpn_base
QPN base.
Definition: arbel.h:529
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
unsigned long special_qpn_base
Special QPN base.
Definition: arbel.h:527
An Arbel device.
Definition: arbel.h:469
arbel_bitmask_t qp_inuse[ARBEL_BITMASK_SIZE(ARBEL_MAX_QPS)]
Queue pair in-use bitmask.
Definition: arbel.h:522
#define ARBEL_MAX_QPS
Maximum number of allocatable queue pairs.
Definition: arbel.h:392

References arbel_bitmask_alloc(), ARBEL_MAX_QPS, ARBEL_PORT_BASE, ARBEL_QPN_RANDOM_MASK, DBGC, ENOTSUP, ib_get_drvdata(), IB_QPT_GSI, IB_QPT_RC, IB_QPT_SMI, IB_QPT_UD, arbel::ibdev, ib_device::port, qp, arbel::qp_inuse, arbel::qpn_base, random(), and arbel::special_qpn_base.

Referenced by arbel_create_qp().

◆ arbel_free_qpn()

static void arbel_free_qpn ( struct ib_device ibdev,
struct ib_queue_pair qp 
)
static

Free queue pair number.

Parameters
ibdevInfiniband device
qpQueue pair

Definition at line 806 of file arbel.c.

807  {
808  struct arbel *arbel = ib_get_drvdata ( ibdev );
809  int qpn_offset;
810 
811  qpn_offset = ( ( qp->qpn & ~ARBEL_QPN_RANDOM_MASK ) - arbel->qpn_base );
812  if ( qpn_offset >= 0 )
813  arbel_bitmask_free ( arbel->qp_inuse, qpn_offset );
814 }
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define ARBEL_QPN_RANDOM_MASK
Queue pair number randomisation mask.
Definition: arbel.h:395
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static void arbel_bitmask_free(arbel_bitmask_t *bits, int bit)
Free offset within usage bitmask.
Definition: arbel.c:93
unsigned long qpn_base
QPN base.
Definition: arbel.h:529
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
An Arbel device.
Definition: arbel.h:469
arbel_bitmask_t qp_inuse[ARBEL_BITMASK_SIZE(ARBEL_MAX_QPS)]
Queue pair in-use bitmask.
Definition: arbel.h:522

References arbel_bitmask_free(), ARBEL_QPN_RANDOM_MASK, ib_get_drvdata(), arbel::ibdev, qp, arbel::qp_inuse, and arbel::qpn_base.

Referenced by arbel_create_qp(), and arbel_destroy_qp().

◆ arbel_rate()

static unsigned int arbel_rate ( struct ib_address_vector av)
static

Calculate transmission rate.

Parameters
avAddress vector
Return values
arbel_rateArbel rate

Definition at line 822 of file arbel.c.

822  {
823  return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) )
824  ? ( av->rate + 5 ) : 0 );
825 }
enum ib_rate rate
Rate.
Definition: infiniband.h:86

References IB_RATE_120, IB_RATE_2_5, and ib_address_vector::rate.

Referenced by arbel_fill_mlx_send_wqe(), arbel_fill_ud_send_wqe(), and arbel_modify_qp().

◆ arbel_dump_qpctx()

static int arbel_dump_qpctx ( struct arbel arbel,
struct ib_queue_pair qp 
)
static

Dump queue pair context (for debugging only)

Parameters
arbelArbel device
qpQueue pair
Return values
rcReturn status code

Definition at line 843 of file arbel.c.

843  {
844  struct arbelprm_qp_ee_state_transitions qpctx;
845  int rc;
846 
847  memset ( &qpctx, 0, sizeof ( qpctx ) );
848  if ( ( rc = arbel_cmd_query_qpee ( arbel, qp->qpn, &qpctx ) ) != 0 ) {
849  DBGC ( arbel, "Arbel %p QPN %#lx QUERY_QPEE failed: %s\n",
850  arbel, qp->qpn, strerror ( rc ) );
851  return rc;
852  }
853  DBGC ( arbel, "Arbel %p QPN %#lx context:\n", arbel, qp->qpn );
854  DBGC_HDA ( arbel, 0, &qpctx.u.dwords[2], ( sizeof ( qpctx ) - 8 ) );
855 
856  return 0;
857 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define DBGC(...)
Definition: compiler.h:505
#define DBGC_HDA(...)
Definition: compiler.h:506
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
static int arbel_cmd_query_qpee(struct arbel *arbel, unsigned long qpn, struct arbelprm_qp_ee_state_transitions *ctx)
Definition: arbel.c:392
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_query_qpee(), DBGC, DBGC_HDA, memset(), qp, rc, and strerror().

◆ arbel_create_send_wq()

static int arbel_create_send_wq ( struct arbel_send_work_queue arbel_send_wq,
unsigned int  num_wqes 
)
static

Create send work queue.

Parameters
arbel_send_wqSend work queue
num_wqesNumber of work queue entries
Return values
rcReturn status code

Definition at line 866 of file arbel.c.

867  {
868  union arbel_send_wqe *wqe;
869  union arbel_send_wqe *next_wqe;
870  unsigned int wqe_idx_mask;
871  unsigned int i;
872 
873  /* Allocate work queue */
874  arbel_send_wq->wqe_size = ( num_wqes *
875  sizeof ( arbel_send_wq->wqe[0] ) );
876  arbel_send_wq->wqe = malloc_phys ( arbel_send_wq->wqe_size,
877  sizeof ( arbel_send_wq->wqe[0] ) );
878  if ( ! arbel_send_wq->wqe )
879  return -ENOMEM;
880  memset ( arbel_send_wq->wqe, 0, arbel_send_wq->wqe_size );
881 
882  /* Link work queue entries */
883  wqe_idx_mask = ( num_wqes - 1 );
884  for ( i = 0 ; i < num_wqes ; i++ ) {
885  wqe = &arbel_send_wq->wqe[i];
886  next_wqe = &arbel_send_wq->wqe[ ( i + 1 ) & wqe_idx_mask ];
887  MLX_FILL_1 ( &wqe->next, 0, nda_31_6,
888  ( virt_to_bus ( next_wqe ) >> 6 ) );
889  MLX_FILL_1 ( &wqe->next, 1, always1, 1 );
890  }
891 
892  return 0;
893 }
An Arbel send work queue entry.
Definition: arbel.h:337
size_t wqe_size
Size of work queue.
Definition: arbel.h:352
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
union arbel_send_wqe * wqe
Work queue entries.
Definition: arbel.h:350
#define ENOMEM
Not enough space.
Definition: errno.h:534
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
struct arbelprm_wqe_segment_next next
Definition: arbel.h:338
void * memset(void *dest, int character, size_t len) __nonnull

References ENOMEM, malloc_phys(), memset(), MLX_FILL_1, arbel_send_wqe::next, virt_to_bus(), arbel_send_work_queue::wqe, and arbel_send_work_queue::wqe_size.

Referenced by arbel_create_qp().

◆ arbel_create_recv_wq()

static int arbel_create_recv_wq ( struct arbel_recv_work_queue arbel_recv_wq,
unsigned int  num_wqes,
enum ib_queue_pair_type  type 
)
static

Create receive work queue.

Parameters
arbel_recv_wqReceive work queue
num_wqesNumber of work queue entries
typeQueue pair type
Return values
rcReturn status code

Definition at line 903 of file arbel.c.

905  {
906  struct arbelprm_recv_wqe *wqe;
907  struct arbelprm_recv_wqe *next_wqe;
908  unsigned int wqe_idx_mask;
909  size_t nds;
910  unsigned int i;
911  unsigned int j;
912  int rc;
913 
914  /* Allocate work queue */
915  arbel_recv_wq->wqe_size = ( num_wqes *
916  sizeof ( arbel_recv_wq->wqe[0] ) );
917  arbel_recv_wq->wqe = malloc_phys ( arbel_recv_wq->wqe_size,
918  sizeof ( arbel_recv_wq->wqe[0] ) );
919  if ( ! arbel_recv_wq->wqe ) {
920  rc = -ENOMEM;
921  goto err_alloc_wqe;
922  }
923  memset ( arbel_recv_wq->wqe, 0, arbel_recv_wq->wqe_size );
924 
925  /* Allocate GRH entries, if needed */
926  if ( ( type == IB_QPT_SMI ) || ( type == IB_QPT_GSI ) ||
927  ( type == IB_QPT_UD ) ) {
928  arbel_recv_wq->grh_size = ( num_wqes *
929  sizeof ( arbel_recv_wq->grh[0] ) );
930  arbel_recv_wq->grh = malloc_phys ( arbel_recv_wq->grh_size,
931  sizeof ( void * ) );
932  if ( ! arbel_recv_wq->grh ) {
933  rc = -ENOMEM;
934  goto err_alloc_grh;
935  }
936  }
937 
938  /* Link work queue entries */
939  wqe_idx_mask = ( num_wqes - 1 );
940  nds = ( ( offsetof ( typeof ( *wqe ), data ) +
941  sizeof ( wqe->data[0] ) ) >> 4 );
942  for ( i = 0 ; i < num_wqes ; i++ ) {
943  wqe = &arbel_recv_wq->wqe[i].recv;
944  next_wqe = &arbel_recv_wq->wqe[( i + 1 ) & wqe_idx_mask].recv;
945  MLX_FILL_1 ( &wqe->next, 0, nda_31_6,
946  ( virt_to_bus ( next_wqe ) >> 6 ) );
947  MLX_FILL_1 ( &wqe->next, 1, nds, nds );
948  for ( j = 0 ; ( ( ( void * ) &wqe->data[j] ) <
949  ( ( void * ) ( wqe + 1 ) ) ) ; j++ ) {
950  MLX_FILL_1 ( &wqe->data[j], 1,
951  l_key, ARBEL_INVALID_LKEY );
952  }
953  }
954 
955  return 0;
956 
957  free_phys ( arbel_recv_wq->grh, arbel_recv_wq->grh_size );
958  err_alloc_grh:
959  free_phys ( arbel_recv_wq->wqe, arbel_recv_wq->wqe_size );
960  err_alloc_wqe:
961  return rc;
962 }
struct ib_global_route_header * grh
GRH buffers (if applicable)
Definition: arbel.h:373
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
union arbel_recv_wqe * wqe
Work queue entries.
Definition: arbel.h:369
struct arbelprm_wqe_segment_data_ptr data[ARBEL_MAX_SCATTER]
Definition: arbel.h:252
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
#define offsetof(type, field)
Get offset of a field within a structure.
Definition: stddef.h:24
struct arbelprm_recv_wqe recv
Definition: arbel.h:11
size_t grh_size
Size of GRB buffers.
Definition: arbel.h:375
#define ENOMEM
Not enough space.
Definition: errno.h:534
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define ARBEL_INVALID_LKEY
Definition: arbel.h:98
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
size_t wqe_size
Size of work queue.
Definition: arbel.h:371
uint32_t type
Operating system type.
Definition: ena.h:12
struct arbelprm_recv_wqe_segment_next next
Definition: arbel.h:250
struct arbelprm_recv_wqe recv
Definition: arbel.h:360
uint8_t data[48]
Additional event data.
Definition: ena.h:22
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
typeof(acpi_finder=acpi_find)
ACPI table finder.
Definition: acpi.c:45
void * memset(void *dest, int character, size_t len) __nonnull

References ARBEL_INVALID_LKEY, data, arbelprm_recv_wqe::data, ENOMEM, free_phys(), arbel_recv_work_queue::grh, arbel_recv_work_queue::grh_size, IB_QPT_GSI, IB_QPT_SMI, IB_QPT_UD, malloc_phys(), memset(), MLX_FILL_1, arbelprm_recv_wqe::next, offsetof, rc, arbel_recv_wqe::recv, recv, type, typeof(), virt_to_bus(), arbel_recv_work_queue::wqe, and arbel_recv_work_queue::wqe_size.

Referenced by arbel_create_qp().

◆ arbel_create_qp()

static int arbel_create_qp ( struct ib_device ibdev,
struct ib_queue_pair qp 
)
static

Create queue pair.

Parameters
ibdevInfiniband device
qpQueue pair
Return values
rcReturn status code

Definition at line 971 of file arbel.c.

972  {
973  struct arbel *arbel = ib_get_drvdata ( ibdev );
974  struct arbel_queue_pair *arbel_qp;
975  struct arbelprm_qp_ee_state_transitions qpctx;
976  struct arbelprm_qp_db_record *send_db_rec;
977  struct arbelprm_qp_db_record *recv_db_rec;
978  physaddr_t send_wqe_base_adr;
979  physaddr_t recv_wqe_base_adr;
980  physaddr_t wqe_base_adr;
981  int rc;
982 
983  /* Warn about dysfunctional code
984  *
985  * Arbel seems to crash the system as soon as the first send
986  * WQE completes on an RC queue pair. (NOPs complete
987  * successfully, so this is a problem specific to the work
988  * queue rather than the completion queue.) The cause of this
989  * problem has remained unknown for over a year. Patches to
990  * fix this are welcome.
991  */
992  if ( qp->type == IB_QPT_RC )
993  DBG ( "*** WARNING: Arbel RC support is non-functional ***\n" );
994 
995  /* Calculate queue pair number */
996  if ( ( rc = arbel_alloc_qpn ( ibdev, qp ) ) != 0 )
997  goto err_alloc_qpn;
998 
999  /* Allocate control structures */
1000  arbel_qp = zalloc ( sizeof ( *arbel_qp ) );
1001  if ( ! arbel_qp ) {
1002  rc = -ENOMEM;
1003  goto err_arbel_qp;
1004  }
1007 
1008  /* Create send and receive work queues */
1009  if ( ( rc = arbel_create_send_wq ( &arbel_qp->send,
1010  qp->send.num_wqes ) ) != 0 )
1011  goto err_create_send_wq;
1012  if ( ( rc = arbel_create_recv_wq ( &arbel_qp->recv, qp->recv.num_wqes,
1013  qp->type ) ) != 0 )
1014  goto err_create_recv_wq;
1015 
1016  /* Send and receive work queue entries must be within the same 4GB */
1017  send_wqe_base_adr = virt_to_bus ( arbel_qp->send.wqe );
1018  recv_wqe_base_adr = virt_to_bus ( arbel_qp->recv.wqe );
1019  if ( ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) &&
1020  ( ( ( ( uint64_t ) send_wqe_base_adr ) >> 32 ) !=
1021  ( ( ( uint64_t ) recv_wqe_base_adr ) >> 32 ) ) ) {
1022  DBGC ( arbel, "Arbel %p QPN %#lx cannot support send %08lx "
1023  "recv %08lx\n", arbel, qp->qpn,
1024  send_wqe_base_adr, recv_wqe_base_adr );
1025  rc = -ENOTSUP;
1026  goto err_unsupported_address_split;
1027  }
1028  wqe_base_adr = send_wqe_base_adr;
1029 
1030  /* Initialise doorbell records */
1031  send_db_rec = &arbel->db_rec[arbel_qp->send.doorbell_idx].qp;
1032  MLX_FILL_1 ( send_db_rec, 0, counter, 0 );
1033  MLX_FILL_2 ( send_db_rec, 1,
1034  res, ARBEL_UAR_RES_SQ,
1035  qp_number, qp->qpn );
1036  recv_db_rec = &arbel->db_rec[arbel_qp->recv.doorbell_idx].qp;
1037  MLX_FILL_1 ( recv_db_rec, 0, counter, 0 );
1038  MLX_FILL_2 ( recv_db_rec, 1,
1039  res, ARBEL_UAR_RES_RQ,
1040  qp_number, qp->qpn );
1041 
1042  /* Transition queue to INIT state */
1043  memset ( &qpctx, 0, sizeof ( qpctx ) );
1044  MLX_FILL_3 ( &qpctx, 2,
1045  qpc_eec_data.de, 1,
1046  qpc_eec_data.pm_state, ARBEL_PM_STATE_MIGRATED,
1047  qpc_eec_data.st, arbel_qp_st[qp->type] );
1048  MLX_FILL_4 ( &qpctx, 4,
1049  qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
1050  qpc_eec_data.log_rq_stride,
1051  ( fls ( sizeof ( arbel_qp->recv.wqe[0] ) - 1 ) - 4 ),
1052  qpc_eec_data.log_sq_size, fls ( qp->send.num_wqes - 1 ),
1053  qpc_eec_data.log_sq_stride,
1054  ( fls ( sizeof ( arbel_qp->send.wqe[0] ) - 1 ) - 4 ) );
1055  MLX_FILL_1 ( &qpctx, 5,
1056  qpc_eec_data.usr_page, arbel->limits.reserved_uars );
1057  MLX_FILL_1 ( &qpctx, 10, qpc_eec_data.primary_address_path.port_number,
1058  ibdev->port );
1059  MLX_FILL_1 ( &qpctx, 27, qpc_eec_data.pd, ARBEL_GLOBAL_PD );
1060  MLX_FILL_H ( &qpctx, 28, qpc_eec_data.wqe_base_adr_h, wqe_base_adr );
1061  MLX_FILL_1 ( &qpctx, 29, qpc_eec_data.wqe_lkey, arbel->lkey );
1062  MLX_FILL_1 ( &qpctx, 30, qpc_eec_data.ssc, 1 );
1063  MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
1064  MLX_FILL_1 ( &qpctx, 34, qpc_eec_data.snd_wqe_base_adr_l,
1065  ( send_wqe_base_adr >> 6 ) );
1066  MLX_FILL_1 ( &qpctx, 35, qpc_eec_data.snd_db_record_index,
1067  arbel_qp->send.doorbell_idx );
1068  MLX_FILL_4 ( &qpctx, 38,
1069  qpc_eec_data.rre, 1,
1070  qpc_eec_data.rwe, 1,
1071  qpc_eec_data.rae, 1,
1072  qpc_eec_data.rsc, 1 );
1073  MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
1074  MLX_FILL_1 ( &qpctx, 42, qpc_eec_data.rcv_wqe_base_adr_l,
1075  ( recv_wqe_base_adr >> 6 ) );
1076  MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.rcv_db_record_index,
1077  arbel_qp->recv.doorbell_idx );
1078  if ( ( rc = arbel_cmd_rst2init_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
1079  DBGC ( arbel, "Arbel %p QPN %#lx RST2INIT_QPEE failed: %s\n",
1080  arbel, qp->qpn, strerror ( rc ) );
1081  goto err_rst2init_qpee;
1082  }
1083  arbel_qp->state = ARBEL_QP_ST_INIT;
1084 
1085  DBGC ( arbel, "Arbel %p QPN %#lx send ring [%08lx,%08lx), doorbell "
1086  "%08lx\n", arbel, qp->qpn, virt_to_phys ( arbel_qp->send.wqe ),
1087  ( virt_to_phys ( arbel_qp->send.wqe ) +
1088  arbel_qp->send.wqe_size ),
1089  virt_to_phys ( send_db_rec ) );
1090  DBGC ( arbel, "Arbel %p QPN %#lx receive ring [%08lx,%08lx), doorbell "
1091  "%08lx\n", arbel, qp->qpn, virt_to_phys ( arbel_qp->recv.wqe ),
1092  ( virt_to_phys ( arbel_qp->recv.wqe ) +
1093  arbel_qp->recv.wqe_size ),
1094  virt_to_phys ( recv_db_rec ) );
1095  DBGC ( arbel, "Arbel %p QPN %#lx send CQN %#lx receive CQN %#lx\n",
1096  arbel, qp->qpn, qp->send.cq->cqn, qp->recv.cq->cqn );
1097  ib_qp_set_drvdata ( qp, arbel_qp );
1098  return 0;
1099 
1100  arbel_cmd_2rst_qpee ( arbel, qp->qpn );
1101  err_rst2init_qpee:
1102  MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
1103  MLX_FILL_1 ( recv_db_rec, 1, res, ARBEL_UAR_RES_NONE );
1104  err_unsupported_address_split:
1105  free_phys ( arbel_qp->recv.grh, arbel_qp->recv.grh_size );
1106  free_phys ( arbel_qp->recv.wqe, arbel_qp->recv.wqe_size );
1107  err_create_recv_wq:
1108  free_phys ( arbel_qp->send.wqe, arbel_qp->send.wqe_size );
1109  err_create_send_wq:
1110  free ( arbel_qp );
1111  err_arbel_qp:
1112  arbel_free_qpn ( ibdev, qp );
1113  err_alloc_qpn:
1114  return rc;
1115 }
struct ib_global_route_header * grh
GRH buffers (if applicable)
Definition: arbel.h:373
static int arbel_cmd_rst2init_qpee(struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
Definition: arbel.c:349
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
static unsigned int arbel_send_doorbell_idx(struct arbel *arbel, struct ib_queue_pair *qp)
Get send work request doorbell index.
Definition: arbel.h:625
#define MLX_FILL_4(_ptr, _index,...)
Definition: mlx_bitops.h:179
union arbel_recv_wqe * wqe
Work queue entries.
Definition: arbel.h:369
#define ARBEL_UAR_RES_SQ
Definition: arbel.h:43
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define ARBEL_UAR_RES_NONE
Definition: arbel.h:40
static int arbel_create_send_wq(struct arbel_send_work_queue *arbel_send_wq, unsigned int num_wqes)
Create send work queue.
Definition: arbel.c:866
static int arbel_alloc_qpn(struct ib_device *ibdev, struct ib_queue_pair *qp)
Assign queue pair number.
Definition: arbel.c:764
unsigned int doorbell_idx
Doorbell record number.
Definition: arbel.h:367
#define DBGC(...)
Definition: compiler.h:505
size_t wqe_size
Size of work queue.
Definition: arbel.h:352
#define ARBEL_GLOBAL_PD
Global protection domain.
Definition: arbel.h:536
unsigned long long uint64_t
Definition: stdint.h:13
unsigned int doorbell_idx
Doorbell record number.
Definition: arbel.h:348
static int arbel_create_recv_wq(struct arbel_recv_work_queue *arbel_recv_wq, unsigned int num_wqes, enum ib_queue_pair_type type)
Create receive work queue.
Definition: arbel.c:903
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
union arbel_send_wqe * wqe
Work queue entries.
Definition: arbel.h:350
struct arbelprm_qp_db_record qp
Definition: arbel.h:268
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
size_t grh_size
Size of GRB buffers.
Definition: arbel.h:375
#define ENOTSUP
Operation not supported.
Definition: errno.h:589
#define MLX_FILL_3(_ptr, _index,...)
Definition: mlx_bitops.h:175
#define ENOMEM
Not enough space.
Definition: errno.h:534
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
static void arbel_free_qpn(struct ib_device *ibdev, struct ib_queue_pair *qp)
Free queue pair number.
Definition: arbel.c:806
static uint8_t arbel_qp_st[]
Queue pair transport service type map.
Definition: arbel.c:828
#define ARBEL_PM_STATE_MIGRATED
Definition: arbel.h:118
unsigned int port
Port number.
Definition: infiniband.h:418
An Arbel queue pair.
Definition: arbel.h:406
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:624
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
size_t wqe_size
Size of work queue.
Definition: arbel.h:371
enum arbel_queue_pair_state state
Queue state.
Definition: arbel.h:412
unsigned int uint32_t
Definition: stdint.h:12
unsigned long physaddr_t
Definition: stdint.h:20
static unsigned int arbel_recv_doorbell_idx(struct arbel *arbel, struct ib_queue_pair *qp)
Get receive work request doorbell index.
Definition: arbel.h:639
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
struct arbel_recv_work_queue recv
Receive work queue.
Definition: arbel.h:410
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
unsigned int reserved_uars
Number of reserved UARs.
Definition: arbel.h:328
#define DBG(...)
Print a debugging message.
Definition: compiler.h:498
#define fls(x)
Find last (i.e.
Definition: strings.h:166
static int arbel_cmd_2rst_qpee(struct arbel *arbel, unsigned long qpn)
Definition: arbel.c:385
struct arbel_send_work_queue send
Send work queue.
Definition: arbel.h:408
An Arbel device.
Definition: arbel.h:469
static __always_inline void ib_qp_set_drvdata(struct ib_queue_pair *qp, void *priv)
Set Infiniband queue pair driver-private data.
Definition: infiniband.h:631
void * memset(void *dest, int character, size_t len) __nonnull
#define ARBEL_UAR_RES_RQ
Definition: arbel.h:44

References arbel_alloc_qpn(), arbel_cmd_2rst_qpee(), arbel_cmd_rst2init_qpee(), arbel_create_recv_wq(), arbel_create_send_wq(), arbel_free_qpn(), ARBEL_GLOBAL_PD, ARBEL_PM_STATE_MIGRATED, arbel_qp_st, ARBEL_QP_ST_INIT, arbel_recv_doorbell_idx(), arbel_send_doorbell_idx(), ARBEL_UAR_RES_NONE, ARBEL_UAR_RES_RQ, ARBEL_UAR_RES_SQ, arbel::db_rec, DBG, DBGC, arbel_send_work_queue::doorbell_idx, arbel_recv_work_queue::doorbell_idx, ENOMEM, ENOTSUP, fls, free, free_phys(), arbel_recv_work_queue::grh, arbel_recv_work_queue::grh_size, ib_get_drvdata(), ib_qp_set_drvdata(), IB_QPT_RC, arbel::ibdev, arbel::limits, arbel::lkey, memset(), MLX_FILL_1, MLX_FILL_2, MLX_FILL_3, MLX_FILL_4, MLX_FILL_H, ib_device::port, arbelprm_doorbell_record::qp, qp, rc, arbel_queue_pair::recv, arbel_dev_limits::reserved_uars, arbel_queue_pair::send, arbel_queue_pair::state, strerror(), virt_to_bus(), virt_to_phys(), arbel_send_work_queue::wqe, arbel_recv_work_queue::wqe, arbel_send_work_queue::wqe_size, arbel_recv_work_queue::wqe_size, and zalloc().

◆ arbel_modify_qp()

static int arbel_modify_qp ( struct ib_device ibdev,
struct ib_queue_pair qp 
)
static

Modify queue pair.

Parameters
ibdevInfiniband device
qpQueue pair
Return values
rcReturn status code

Definition at line 1124 of file arbel.c.

1125  {
1126  struct arbel *arbel = ib_get_drvdata ( ibdev );
1127  struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
1128  struct arbelprm_qp_ee_state_transitions qpctx;
1129  int rc;
1130 
1131  /* Transition queue to RTR state, if applicable */
1132  if ( arbel_qp->state < ARBEL_QP_ST_RTR ) {
1133  memset ( &qpctx, 0, sizeof ( qpctx ) );
1134  MLX_FILL_2 ( &qpctx, 4,
1135  qpc_eec_data.mtu, ARBEL_MTU_2048,
1136  qpc_eec_data.msg_max, 31 );
1137  MLX_FILL_1 ( &qpctx, 7,
1138  qpc_eec_data.remote_qpn_een, qp->av.qpn );
1139  MLX_FILL_2 ( &qpctx, 11,
1140  qpc_eec_data.primary_address_path.rnr_retry,
1142  qpc_eec_data.primary_address_path.rlid,
1143  qp->av.lid );
1144  MLX_FILL_2 ( &qpctx, 12,
1145  qpc_eec_data.primary_address_path.ack_timeout,
1146  14 /* 4.096us * 2^(14) = 67ms */,
1147  qpc_eec_data.primary_address_path.max_stat_rate,
1148  arbel_rate ( &qp->av ) );
1149  memcpy ( &qpctx.u.dwords[14], &qp->av.gid,
1150  sizeof ( qp->av.gid ) );
1151  MLX_FILL_1 ( &qpctx, 30,
1152  qpc_eec_data.retry_count, ARBEL_RETRY_MAX );
1153  MLX_FILL_1 ( &qpctx, 39,
1154  qpc_eec_data.next_rcv_psn, qp->recv.psn );
1155  MLX_FILL_1 ( &qpctx, 40,
1156  qpc_eec_data.ra_buff_indx,
1158  ( ( qp->qpn & ~ARBEL_QPN_RANDOM_MASK ) -
1159  arbel->special_qpn_base ) ) );
1160  if ( ( rc = arbel_cmd_init2rtr_qpee ( arbel, qp->qpn,
1161  &qpctx ) ) != 0 ) {
1162  DBGC ( arbel, "Arbel %p QPN %#lx INIT2RTR_QPEE failed:"
1163  " %s\n", arbel, qp->qpn, strerror ( rc ) );
1164  return rc;
1165  }
1166  arbel_qp->state = ARBEL_QP_ST_RTR;
1167  }
1168 
1169  /* Transition queue to RTS state, if applicable */
1170  if ( arbel_qp->state < ARBEL_QP_ST_RTS ) {
1171  memset ( &qpctx, 0, sizeof ( qpctx ) );
1172  MLX_FILL_1 ( &qpctx, 11,
1173  qpc_eec_data.primary_address_path.rnr_retry,
1174  ARBEL_RETRY_MAX );
1175  MLX_FILL_1 ( &qpctx, 12,
1176  qpc_eec_data.primary_address_path.ack_timeout,
1177  14 /* 4.096us * 2^(14) = 67ms */ );
1178  MLX_FILL_2 ( &qpctx, 30,
1179  qpc_eec_data.retry_count, ARBEL_RETRY_MAX,
1180  qpc_eec_data.sic, 1 );
1181  MLX_FILL_1 ( &qpctx, 32,
1182  qpc_eec_data.next_send_psn, qp->send.psn );
1183  if ( ( rc = arbel_cmd_rtr2rts_qpee ( arbel, qp->qpn,
1184  &qpctx ) ) != 0 ) {
1185  DBGC ( arbel, "Arbel %p QPN %#lx RTR2RTS_QPEE failed: "
1186  "%s\n", arbel, qp->qpn, strerror ( rc ) );
1187  return rc;
1188  }
1189  arbel_qp->state = ARBEL_QP_ST_RTS;
1190  }
1191 
1192  /* Update parameters in RTS state */
1193  memset ( &qpctx, 0, sizeof ( qpctx ) );
1195  MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
1196  if ( ( rc = arbel_cmd_rts2rts_qpee ( arbel, qp->qpn, &qpctx ) ) != 0 ){
1197  DBGC ( arbel, "Arbel %p QPN %#lx RTS2RTS_QPEE failed: %s\n",
1198  arbel, qp->qpn, strerror ( rc ) );
1199  return rc;
1200  }
1201 
1202  return 0;
1203 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
static __always_inline void * ib_qp_get_drvdata(struct ib_queue_pair *qp)
Get Infiniband queue pair driver-private data.
Definition: infiniband.h:642
static unsigned int arbel_rate(struct ib_address_vector *av)
Calculate transmission rate.
Definition: arbel.c:822
unsigned int reserved_rdbs
Number of reserved RDBs.
Definition: arbel.h:324
#define ARBEL_QPN_RANDOM_MASK
Queue pair number randomisation mask.
Definition: arbel.h:395
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
#define ARBEL_QPEE_OPT_PARAM_QKEY
Definition: arbel.h:107
static int arbel_cmd_rtr2rts_qpee(struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
Definition: arbel.c:367
#define ARBEL_MTU_2048
Definition: arbel.h:94
An Arbel queue pair.
Definition: arbel.h:406
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
enum arbel_queue_pair_state state
Queue state.
Definition: arbel.h:412
#define ARBEL_RETRY_MAX
Definition: arbel.h:120
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
unsigned long special_qpn_base
Special QPN base.
Definition: arbel.h:527
static int arbel_cmd_rts2rts_qpee(struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
Definition: arbel.c:376
__be32 opt_param_mask
Definition: CIB_PRM.h:28
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull
static int arbel_cmd_init2rtr_qpee(struct arbel *arbel, unsigned long qpn, const struct arbelprm_qp_ee_state_transitions *ctx)
Definition: arbel.c:358

References arbel_cmd_init2rtr_qpee(), arbel_cmd_rtr2rts_qpee(), arbel_cmd_rts2rts_qpee(), ARBEL_MTU_2048, ARBEL_QP_ST_RTR, ARBEL_QP_ST_RTS, ARBEL_QPEE_OPT_PARAM_QKEY, ARBEL_QPN_RANDOM_MASK, arbel_rate(), ARBEL_RETRY_MAX, DBGC, ib_get_drvdata(), ib_qp_get_drvdata(), arbel::ibdev, arbel::limits, memcpy(), memset(), MLX_FILL_1, MLX_FILL_2, opt_param_mask, qp, rc, arbel_dev_limits::reserved_rdbs, arbel::special_qpn_base, arbel_queue_pair::state, and strerror().

◆ arbel_destroy_qp()

static void arbel_destroy_qp ( struct ib_device ibdev,
struct ib_queue_pair qp 
)
static

Destroy queue pair.

Parameters
ibdevInfiniband device
qpQueue pair

Definition at line 1211 of file arbel.c.

1212  {
1213  struct arbel *arbel = ib_get_drvdata ( ibdev );
1214  struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
1215  struct arbelprm_qp_db_record *send_db_rec;
1216  struct arbelprm_qp_db_record *recv_db_rec;
1217  int rc;
1218 
1219  /* Take ownership back from hardware */
1220  if ( ( rc = arbel_cmd_2rst_qpee ( arbel, qp->qpn ) ) != 0 ) {
1221  DBGC ( arbel, "Arbel %p QPN %#lx FATAL 2RST_QPEE failed: "
1222  "%s\n", arbel, qp->qpn, strerror ( rc ) );
1223  /* Leak memory and return; at least we avoid corruption */
1224  return;
1225  }
1226 
1227  /* Clear doorbell records */
1228  send_db_rec = &arbel->db_rec[arbel_qp->send.doorbell_idx].qp;
1229  recv_db_rec = &arbel->db_rec[arbel_qp->recv.doorbell_idx].qp;
1230  MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
1231  MLX_FILL_1 ( recv_db_rec, 1, res, ARBEL_UAR_RES_NONE );
1232 
1233  /* Free memory */
1234  free_phys ( arbel_qp->recv.grh, arbel_qp->recv.grh_size );
1235  free_phys ( arbel_qp->recv.wqe, arbel_qp->recv.wqe_size );
1236  free_phys ( arbel_qp->send.wqe, arbel_qp->send.wqe_size );
1237  free ( arbel_qp );
1238 
1239  /* Mark queue number as free */
1240  arbel_free_qpn ( ibdev, qp );
1241 
1242  ib_qp_set_drvdata ( qp, NULL );
1243 }
struct ib_global_route_header * grh
GRH buffers (if applicable)
Definition: arbel.h:373
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
union arbel_recv_wqe * wqe
Work queue entries.
Definition: arbel.h:369
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define ARBEL_UAR_RES_NONE
Definition: arbel.h:40
unsigned int doorbell_idx
Doorbell record number.
Definition: arbel.h:367
#define DBGC(...)
Definition: compiler.h:505
size_t wqe_size
Size of work queue.
Definition: arbel.h:352
static __always_inline void * ib_qp_get_drvdata(struct ib_queue_pair *qp)
Get Infiniband queue pair driver-private data.
Definition: infiniband.h:642
unsigned int doorbell_idx
Doorbell record number.
Definition: arbel.h:348
union arbel_send_wqe * wqe
Work queue entries.
Definition: arbel.h:350
struct arbelprm_qp_db_record qp
Definition: arbel.h:268
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
size_t grh_size
Size of GRB buffers.
Definition: arbel.h:375
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static void arbel_free_qpn(struct ib_device *ibdev, struct ib_queue_pair *qp)
Free queue pair number.
Definition: arbel.c:806
An Arbel queue pair.
Definition: arbel.h:406
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
size_t wqe_size
Size of work queue.
Definition: arbel.h:371
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
struct arbel_recv_work_queue recv
Receive work queue.
Definition: arbel.h:410
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
static int arbel_cmd_2rst_qpee(struct arbel *arbel, unsigned long qpn)
Definition: arbel.c:385
struct arbel_send_work_queue send
Send work queue.
Definition: arbel.h:408
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
static __always_inline void ib_qp_set_drvdata(struct ib_queue_pair *qp, void *priv)
Set Infiniband queue pair driver-private data.
Definition: infiniband.h:631

References arbel_cmd_2rst_qpee(), arbel_free_qpn(), ARBEL_UAR_RES_NONE, arbel::db_rec, DBGC, arbel_send_work_queue::doorbell_idx, arbel_recv_work_queue::doorbell_idx, free, free_phys(), arbel_recv_work_queue::grh, arbel_recv_work_queue::grh_size, ib_get_drvdata(), ib_qp_get_drvdata(), ib_qp_set_drvdata(), arbel::ibdev, MLX_FILL_1, NULL, qp, arbelprm_doorbell_record::qp, rc, arbel_queue_pair::recv, arbel_queue_pair::send, strerror(), arbel_send_work_queue::wqe, arbel_recv_work_queue::wqe, arbel_send_work_queue::wqe_size, and arbel_recv_work_queue::wqe_size.

◆ arbel_ring_doorbell()

static void arbel_ring_doorbell ( struct arbel arbel,
union arbelprm_doorbell_register db_reg,
unsigned int  offset 
)
static

Ring doorbell register in UAR.

Parameters
arbelArbel device
db_regDoorbell register structure
offsetAddress of doorbell

Definition at line 1259 of file arbel.c.

1261  {
1262 
1263  DBGC2 ( arbel, "Arbel %p ringing doorbell %08x:%08x at %lx\n",
1264  arbel, ntohl ( db_reg->dword[0] ), ntohl ( db_reg->dword[1] ),
1265  virt_to_phys ( arbel->uar + offset ) );
1266 
1267  barrier();
1268  writel ( db_reg->dword[0], ( arbel->uar + offset + 0 ) );
1269  barrier();
1270  writel ( db_reg->dword[1], ( arbel->uar + offset + 4 ) );
1271 }
#define ntohl(value)
Definition: byteswap.h:134
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
void * uar
PCI user Access Region.
Definition: arbel.h:475
#define DBGC2(...)
Definition: compiler.h:522
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
An Arbel device.
Definition: arbel.h:469

References barrier, DBGC2, arbelprm_doorbell_register::dword, ntohl, offset, arbel::uar, virt_to_phys(), and writel().

Referenced by arbel_post_send().

◆ arbel_fill_ud_send_wqe()

static size_t arbel_fill_ud_send_wqe ( struct ib_device ibdev,
struct ib_queue_pair *qp  __unused,
struct ib_address_vector dest,
struct io_buffer iobuf,
union arbel_send_wqe wqe 
)
static

Construct UD send work queue entry.

Parameters
ibdevInfiniband device
qpQueue pair
destDestination address vector
iobufI/O buffer
wqeSend work queue entry
Return values
ndsWork queue entry size

Definition at line 1288 of file arbel.c.

1292  {
1293  struct arbel *arbel = ib_get_drvdata ( ibdev );
1294  const union ib_gid *gid;
1295 
1296  /* Construct this work queue entry */
1297  MLX_FILL_1 ( &wqe->ud.ctrl, 0, always1, 1 );
1298  MLX_FILL_2 ( &wqe->ud.ud, 0,
1299  ud_address_vector.pd, ARBEL_GLOBAL_PD,
1300  ud_address_vector.port_number, ibdev->port );
1301  MLX_FILL_2 ( &wqe->ud.ud, 1,
1302  ud_address_vector.rlid, dest->lid,
1303  ud_address_vector.g, dest->gid_present );
1304  MLX_FILL_2 ( &wqe->ud.ud, 2,
1305  ud_address_vector.max_stat_rate, arbel_rate ( dest ),
1306  ud_address_vector.msg, 3 );
1307  MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, dest->sl );
1308  gid = ( dest->gid_present ? &dest->gid : &arbel_no_gid );
1309  memcpy ( &wqe->ud.ud.u.dwords[4], gid, sizeof ( *gid ) );
1310  MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, dest->qpn );
1311  MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, dest->qkey );
1312  MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) );
1313  MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, arbel->lkey );
1314  MLX_FILL_H ( &wqe->ud.data[0], 2,
1315  local_address_h, virt_to_bus ( iobuf->data ) );
1316  MLX_FILL_1 ( &wqe->ud.data[0], 3,
1317  local_address_l, virt_to_bus ( iobuf->data ) );
1318 
1319  return ( offsetof ( typeof ( wqe->ud ), data[1] ) >> 4 );
1320 }
struct arbelprm_wqe_segment_ud ud
Definition: arbel.h:223
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define ARBEL_GLOBAL_PD
Global protection domain.
Definition: arbel.h:536
__be32 byte_count
Definition: CIB_PRM.h:28
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
#define offsetof(type, field)
Get offset of a field within a structure.
Definition: stddef.h:24
An Infiniband Global Identifier.
Definition: ib_packet.h:33
static const union ib_gid arbel_no_gid
GID used for GID-less send work queue entries.
Definition: arbel.c:1274
static unsigned int arbel_rate(struct ib_address_vector *av)
Calculate transmission rate.
Definition: arbel.c:822
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
static void * dest
Definition: strings.h:176
unsigned int port
Port number.
Definition: infiniband.h:418
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
struct arbelprm_ud_send_wqe ud
Definition: arbel.h:339
void * data
Start of data.
Definition: iobuf.h:48
uint8_t data[48]
Additional event data.
Definition: ena.h:22
typeof(acpi_finder=acpi_find)
ACPI table finder.
Definition: acpi.c:45
u8 gid[16]
Definition: CIB_PRM.h:31
struct arbelprm_wqe_segment_ctrl_send ctrl
Definition: arbel.h:222
struct arbelprm_wqe_segment_data_ptr data[ARBEL_MAX_GATHER]
Definition: arbel.h:224
An Arbel device.
Definition: arbel.h:469

References ARBEL_GLOBAL_PD, arbel_no_gid, arbel_rate(), byte_count, arbelprm_ud_send_wqe::ctrl, arbelprm_ud_send_wqe::data, data, io_buffer::data, dest, gid, ib_get_drvdata(), arbel::ibdev, iob_len(), arbel::lkey, memcpy(), MLX_FILL_1, MLX_FILL_2, MLX_FILL_H, offsetof, ib_device::port, typeof(), arbelprm_ud_send_wqe::ud, arbel_send_wqe::ud, and virt_to_bus().

◆ arbel_fill_mlx_send_wqe()

static size_t arbel_fill_mlx_send_wqe ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct ib_address_vector dest,
struct io_buffer iobuf,
union arbel_send_wqe wqe 
)
static

Construct MLX send work queue entry.

Parameters
ibdevInfiniband device
qpQueue pair
destDestination address vector
iobufI/O buffer
wqeSend work queue entry
Return values
ndsWork queue entry size

Definition at line 1332 of file arbel.c.

1336  {
1337  struct arbel *arbel = ib_get_drvdata ( ibdev );
1338  struct io_buffer headers;
1339 
1340  /* Construct IB headers */
1341  iob_populate ( &headers, &wqe->mlx.headers, 0,
1342  sizeof ( wqe->mlx.headers ) );
1343  iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) );
1344  ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), dest );
1345 
1346  /* Construct this work queue entry */
1347  MLX_FILL_5 ( &wqe->mlx.ctrl, 0,
1348  c, 1 /* generate completion */,
1349  icrc, 0 /* generate ICRC */,
1350  max_statrate, arbel_rate ( dest ),
1351  slr, 0,
1352  v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) );
1353  MLX_FILL_1 ( &wqe->mlx.ctrl, 1, rlid, dest->lid );
1354  MLX_FILL_1 ( &wqe->mlx.data[0], 0,
1355  byte_count, iob_len ( &headers ) );
1356  MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, arbel->lkey );
1357  MLX_FILL_H ( &wqe->mlx.data[0], 2,
1358  local_address_h, virt_to_bus ( headers.data ) );
1359  MLX_FILL_1 ( &wqe->mlx.data[0], 3,
1360  local_address_l, virt_to_bus ( headers.data ) );
1361  MLX_FILL_1 ( &wqe->mlx.data[1], 0,
1362  byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) );
1363  MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, arbel->lkey );
1364  MLX_FILL_H ( &wqe->mlx.data[1], 2,
1365  local_address_h, virt_to_bus ( iobuf->data ) );
1366  MLX_FILL_1 ( &wqe->mlx.data[1], 3,
1367  local_address_l, virt_to_bus ( iobuf->data ) );
1368 
1369  return ( offsetof ( typeof ( wqe->mlx ), data[2] ) >> 4 );
1370 }
uint32_t c
Definition: md4.c:30
#define IB_QPN_SMI
Subnet management interface QPN.
Definition: infiniband.h:21
uint8_t headers[IB_MAX_HEADER_SIZE]
Definition: arbel.h:231
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
__be32 byte_count
Definition: CIB_PRM.h:28
struct arbelprm_wqe_segment_data_ptr data[ARBEL_MAX_GATHER]
Definition: arbel.h:230
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
#define offsetof(type, field)
Get offset of a field within a structure.
Definition: stddef.h:24
static void iob_populate(struct io_buffer *iobuf, void *data, size_t len, size_t max_len)
Create a temporary I/O buffer.
Definition: iobuf.h:190
static unsigned int arbel_rate(struct ib_address_vector *av)
Calculate transmission rate.
Definition: arbel.c:822
struct arbelprm_mlx_send_wqe mlx
Definition: arbel.h:340
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
static void * dest
Definition: strings.h:176
__be16 rlid
Definition: CIB_PRM.h:38
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
uint8_t headers[IB_MAX_HEADER_SIZE]
Definition: arbel.h:14
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
#define iob_reserve(iobuf, len)
Definition: iobuf.h:67
struct arbelprm_wqe_segment_ctrl_mlx ctrl
Definition: arbel.h:229
void * data
Start of data.
Definition: iobuf.h:48
uint8_t data[48]
Additional event data.
Definition: ena.h:22
#define MLX_FILL_5(_ptr, _index,...)
Definition: mlx_bitops.h:183
typeof(acpi_finder=acpi_find)
ACPI table finder.
Definition: acpi.c:45
An Arbel device.
Definition: arbel.h:469
int ib_push(struct ib_device *ibdev, struct io_buffer *iobuf, struct ib_queue_pair *qp, size_t payload_len, const struct ib_address_vector *dest)
Add IB headers.
Definition: ib_packet.c:52
A persistent I/O buffer.
Definition: iobuf.h:33

References arbel_rate(), byte_count, c, arbelprm_mlx_send_wqe::ctrl, arbelprm_mlx_send_wqe::data, data, io_buffer::data, dest, headers, arbelprm_mlx_send_wqe::headers, ib_get_drvdata(), ib_push(), IB_QPN_SMI, arbel::ibdev, iob_len(), iob_populate(), iob_reserve, arbel::lkey, arbel_send_wqe::mlx, MLX_FILL_1, MLX_FILL_5, MLX_FILL_H, offsetof, qp, rlid, typeof(), and virt_to_bus().

◆ arbel_fill_rc_send_wqe()

static size_t arbel_fill_rc_send_wqe ( struct ib_device ibdev,
struct ib_queue_pair *qp  __unused,
struct ib_address_vector *dest  __unused,
struct io_buffer iobuf,
union arbel_send_wqe wqe 
)
static

Construct RC send work queue entry.

Parameters
ibdevInfiniband device
qpQueue pair
destDestination address vector
iobufI/O buffer
wqeSend work queue entry
Return values
ndsWork queue entry size

Definition at line 1382 of file arbel.c.

1386  {
1387  struct arbel *arbel = ib_get_drvdata ( ibdev );
1388 
1389  /* Construct this work queue entry */
1390  MLX_FILL_1 ( &wqe->rc.ctrl, 0, always1, 1 );
1391  MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) );
1392  MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, arbel->lkey );
1393  MLX_FILL_H ( &wqe->rc.data[0], 2,
1394  local_address_h, virt_to_bus ( iobuf->data ) );
1395  MLX_FILL_1 ( &wqe->rc.data[0], 3,
1396  local_address_l, virt_to_bus ( iobuf->data ) );
1397 
1398  return ( offsetof ( typeof ( wqe->rc ), data[1] ) >> 4 );
1399 }
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
__be32 byte_count
Definition: CIB_PRM.h:28
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
#define offsetof(type, field)
Get offset of a field within a structure.
Definition: stddef.h:24
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
struct arbelprm_wqe_segment_ctrl_send ctrl
Definition: arbel.h:236
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:341
void * data
Start of data.
Definition: iobuf.h:48
uint8_t data[48]
Additional event data.
Definition: ena.h:22
typeof(acpi_finder=acpi_find)
ACPI table finder.
Definition: acpi.c:45
struct arbelprm_wqe_segment_data_ptr data[ARBEL_MAX_GATHER]
Definition: arbel.h:237
An Arbel device.
Definition: arbel.h:469

References byte_count, arbelprm_rc_send_wqe::ctrl, arbelprm_rc_send_wqe::data, data, io_buffer::data, ib_get_drvdata(), arbel::ibdev, iob_len(), arbel::lkey, MLX_FILL_1, MLX_FILL_H, offsetof, arbel_send_wqe::rc, typeof(), and virt_to_bus().

◆ arbel_post_send()

static int arbel_post_send ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct ib_address_vector dest,
struct io_buffer iobuf 
)
static

Post send work queue entry.

Parameters
ibdevInfiniband device
qpQueue pair
destDestination address vector
iobufI/O buffer
Return values
rcReturn status code

Definition at line 1423 of file arbel.c.

1426  {
1427  struct arbel *arbel = ib_get_drvdata ( ibdev );
1428  struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
1429  struct ib_work_queue *wq = &qp->send;
1430  struct arbel_send_work_queue *arbel_send_wq = &arbel_qp->send;
1431  union arbel_send_wqe *prev_wqe;
1432  union arbel_send_wqe *wqe;
1433  struct arbelprm_qp_db_record *qp_db_rec;
1434  union arbelprm_doorbell_register db_reg;
1435  unsigned long wqe_idx_mask;
1436  size_t nds;
1437 
1438  /* Allocate work queue entry */
1439  wqe_idx_mask = ( wq->num_wqes - 1 );
1440  if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
1441  DBGC ( arbel, "Arbel %p QPN %#lx send queue full",
1442  arbel, qp->qpn );
1443  return -ENOBUFS;
1444  }
1445  wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
1446  prev_wqe = &arbel_send_wq->wqe[(wq->next_idx - 1) & wqe_idx_mask];
1447  wqe = &arbel_send_wq->wqe[wq->next_idx & wqe_idx_mask];
1448 
1449  /* Construct work queue entry */
1450  memset ( ( ( ( void * ) wqe ) + sizeof ( wqe->next ) ), 0,
1451  ( sizeof ( *wqe ) - sizeof ( wqe->next ) ) );
1452  assert ( qp->type < ( sizeof ( arbel_fill_send_wqe ) /
1453  sizeof ( arbel_fill_send_wqe[0] ) ) );
1454  assert ( arbel_fill_send_wqe[qp->type] != NULL );
1455  nds = arbel_fill_send_wqe[qp->type] ( ibdev, qp, dest, iobuf, wqe );
1456  DBGCP ( arbel, "Arbel %p QPN %#lx posting send WQE %#lx:\n",
1457  arbel, qp->qpn, ( wq->next_idx & wqe_idx_mask ) );
1458  DBGCP_HDA ( arbel, virt_to_phys ( wqe ), wqe, sizeof ( *wqe ) );
1459 
1460  /* Update previous work queue entry's "next" field */
1461  MLX_SET ( &prev_wqe->next, nopcode, ARBEL_OPCODE_SEND );
1462  MLX_FILL_3 ( &prev_wqe->next, 1,
1463  nds, nds,
1464  f, 0,
1465  always1, 1 );
1466 
1467  /* Update doorbell record */
1468  barrier();
1469  qp_db_rec = &arbel->db_rec[arbel_send_wq->doorbell_idx].qp;
1470  MLX_FILL_1 ( qp_db_rec, 0,
1471  counter, ( ( wq->next_idx + 1 ) & 0xffff ) );
1472 
1473  /* Ring doorbell register */
1474  MLX_FILL_4 ( &db_reg.send, 0,
1475  nopcode, ARBEL_OPCODE_SEND,
1476  f, 0,
1477  wqe_counter, ( wq->next_idx & 0xffff ),
1478  wqe_cnt, 1 );
1479  MLX_FILL_2 ( &db_reg.send, 1,
1480  nds, nds,
1481  qpn, qp->qpn );
1483 
1484  /* Update work queue's index */
1485  wq->next_idx++;
1486 
1487  return 0;
1488 }
An Arbel send work queue entry.
Definition: arbel.h:337
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
#define MLX_FILL_4(_ptr, _index,...)
Definition: mlx_bitops.h:179
An Arbel send work queue.
Definition: arbel.h:346
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
static __always_inline void * ib_qp_get_drvdata(struct ib_queue_pair *qp)
Get Infiniband queue pair driver-private data.
Definition: infiniband.h:642
unsigned int doorbell_idx
Doorbell record number.
Definition: arbel.h:348
union arbel_send_wqe * wqe
Work queue entries.
Definition: arbel.h:350
struct arbelprm_qp_db_record qp
Definition: arbel.h:268
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
__be32 qpn
Definition: CIB_PRM.h:29
#define ARBEL_OPCODE_SEND
Definition: arbel.h:48
static void arbel_ring_doorbell(struct arbel *arbel, union arbelprm_doorbell_register *db_reg, unsigned int offset)
Ring doorbell register in UAR.
Definition: arbel.c:1259
#define DBGCP_HDA(...)
Definition: compiler.h:540
#define MLX_FILL_3(_ptr, _index,...)
Definition: mlx_bitops.h:175
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
#define ARBEL_DB_POST_SND_OFFSET
Definition: arbel.h:104
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
An Infiniband Work Queue.
Definition: infiniband.h:100
__be16 wqe_counter
Definition: CIB_PRM.h:36
unsigned int num_wqes
Number of work queue entries.
Definition: infiniband.h:112
static void * dest
Definition: strings.h:176
An Arbel queue pair.
Definition: arbel.h:406
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
struct arbelprm_wqe_segment_next next
Definition: arbel.h:338
unsigned long next_idx
Next work queue entry index.
Definition: infiniband.h:122
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
static size_t(* arbel_fill_send_wqe[])(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
Work queue entry constructors.
Definition: arbel.c:1403
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
#define DBGCP(...)
Definition: compiler.h:539
uint32_t f
Definition: sha256.c:33
#define MLX_SET(_ptr, _field, _value)
Definition: mlx_bitops.h:204
struct arbel_send_work_queue send
Send work queue.
Definition: arbel.h:408
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull
struct io_buffer ** iobufs
I/O buffers assigned to work queue.
Definition: infiniband.h:124

References ARBEL_DB_POST_SND_OFFSET, arbel_fill_send_wqe, ARBEL_OPCODE_SEND, arbel_ring_doorbell(), assert(), barrier, arbel::db_rec, DBGC, DBGCP, DBGCP_HDA, dest, arbel_send_work_queue::doorbell_idx, ENOBUFS, f, ib_get_drvdata(), ib_qp_get_drvdata(), arbel::ibdev, ib_work_queue::iobufs, memset(), MLX_FILL_1, MLX_FILL_2, MLX_FILL_3, MLX_FILL_4, MLX_SET, arbel_send_wqe::next, ib_work_queue::next_idx, NULL, ib_work_queue::num_wqes, qp, arbelprm_doorbell_record::qp, qpn, arbelprm_doorbell_register::send, arbel_queue_pair::send, virt_to_phys(), arbel_send_work_queue::wqe, and wqe_counter.

◆ arbel_post_recv()

static int arbel_post_recv ( struct ib_device ibdev,
struct ib_queue_pair qp,
struct io_buffer iobuf 
)
static

Post receive work queue entry.

Parameters
ibdevInfiniband device
qpQueue pair
iobufI/O buffer
Return values
rcReturn status code

Definition at line 1498 of file arbel.c.

1500  {
1501  struct arbel *arbel = ib_get_drvdata ( ibdev );
1502  struct arbel_queue_pair *arbel_qp = ib_qp_get_drvdata ( qp );
1503  struct ib_work_queue *wq = &qp->recv;
1504  struct arbel_recv_work_queue *arbel_recv_wq = &arbel_qp->recv;
1505  struct arbelprm_recv_wqe *wqe;
1506  struct arbelprm_wqe_segment_data_ptr *data;
1507  struct ib_global_route_header *grh;
1508  union arbelprm_doorbell_record *db_rec;
1509  unsigned int wqe_idx_mask;
1510 
1511  /* Allocate work queue entry */
1512  wqe_idx_mask = ( wq->num_wqes - 1 );
1513  if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
1514  DBGC ( arbel, "Arbel %p QPN %#lx receive queue full\n",
1515  arbel, qp->qpn );
1516  return -ENOBUFS;
1517  }
1518  wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
1519  wqe = &arbel_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv;
1520 
1521  /* Construct work queue entry */
1522  data = &wqe->data[0];
1523  if ( arbel_recv_wq->grh ) {
1524  grh = &arbel_recv_wq->grh[wq->next_idx & wqe_idx_mask];
1525  MLX_FILL_1 ( data, 0, byte_count, sizeof ( *grh ) );
1526  MLX_FILL_1 ( data, 1, l_key, arbel->lkey );
1527  MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( grh ) );
1528  MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( grh ) );
1529  data++;
1530  }
1531  MLX_FILL_1 ( data, 0, byte_count, iob_tailroom ( iobuf ) );
1532  MLX_FILL_1 ( data, 1, l_key, arbel->lkey );
1533  MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( iobuf->data ) );
1534  MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( iobuf->data ) );
1535 
1536  /* Update doorbell record */
1537  barrier();
1538  db_rec = &arbel->db_rec[arbel_recv_wq->doorbell_idx];
1539  MLX_FILL_1 ( &db_rec->qp, 0,
1540  counter, ( ( wq->next_idx + 1 ) & 0xffff ) );
1541 
1542  /* Update work queue's index */
1543  wq->next_idx++;
1544 
1545  return 0;
1546 }
struct ib_global_route_header * grh
GRH buffers (if applicable)
Definition: arbel.h:373
union arbel_recv_wqe * wqe
Work queue entries.
Definition: arbel.h:369
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
struct arbelprm_wqe_segment_data_ptr data[ARBEL_MAX_SCATTER]
Definition: arbel.h:252
unsigned int doorbell_idx
Doorbell record number.
Definition: arbel.h:367
#define DBGC(...)
Definition: compiler.h:505
__be32 byte_count
Definition: CIB_PRM.h:28
static __always_inline void * ib_qp_get_drvdata(struct ib_queue_pair *qp)
Get Infiniband queue pair driver-private data.
Definition: infiniband.h:642
struct ib_global_route_header grh
Definition: ib_packet.h:16
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
struct arbelprm_qp_db_record qp
Definition: arbel.h:268
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
An Infiniband Work Queue.
Definition: infiniband.h:100
An Infiniband Global Route Header.
Definition: ib_packet.h:89
unsigned int num_wqes
Number of work queue entries.
Definition: infiniband.h:112
An Arbel receive work queue.
Definition: arbel.h:365
An Arbel queue pair.
Definition: arbel.h:406
static size_t iob_tailroom(struct io_buffer *iobuf)
Calculate available space at end of an I/O buffer.
Definition: iobuf.h:175
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
unsigned long next_idx
Next work queue entry index.
Definition: infiniband.h:122
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
void * data
Start of data.
Definition: iobuf.h:48
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
struct arbelprm_recv_wqe recv
Definition: arbel.h:360
uint8_t data[48]
Additional event data.
Definition: ena.h:22
struct arbel_recv_work_queue recv
Receive work queue.
Definition: arbel.h:410
An Arbel device.
Definition: arbel.h:469
struct io_buffer ** iobufs
I/O buffers assigned to work queue.
Definition: infiniband.h:124

References barrier, byte_count, data, io_buffer::data, arbelprm_recv_wqe::data, arbel::db_rec, DBGC, arbel_recv_work_queue::doorbell_idx, ENOBUFS, grh, arbel_recv_work_queue::grh, ib_get_drvdata(), ib_qp_get_drvdata(), arbel::ibdev, iob_tailroom(), ib_work_queue::iobufs, arbel::lkey, MLX_FILL_1, MLX_FILL_H, ib_work_queue::next_idx, ib_work_queue::num_wqes, arbelprm_doorbell_record::qp, qp, arbel_recv_wqe::recv, arbel_queue_pair::recv, virt_to_bus(), and arbel_recv_work_queue::wqe.

◆ arbel_complete()

static int arbel_complete ( struct ib_device ibdev,
struct ib_completion_queue cq,
union arbelprm_completion_entry cqe 
)
static

Handle completion.

Parameters
ibdevInfiniband device
cqCompletion queue
cqeHardware completion queue entry
Return values
rcReturn status code

Definition at line 1556 of file arbel.c.

1558  {
1559  struct arbel *arbel = ib_get_drvdata ( ibdev );
1560  struct ib_work_queue *wq;
1561  struct ib_queue_pair *qp;
1562  struct arbel_queue_pair *arbel_qp;
1563  struct arbel_send_work_queue *arbel_send_wq;
1564  struct arbel_recv_work_queue *arbel_recv_wq;
1565  struct arbelprm_recv_wqe *recv_wqe;
1566  struct io_buffer *iobuf;
1567  struct ib_address_vector recv_dest;
1568  struct ib_address_vector recv_source;
1569  struct ib_global_route_header *grh;
1570  struct ib_address_vector *source;
1571  unsigned int opcode;
1572  unsigned long qpn;
1573  int is_send;
1574  unsigned long wqe_adr;
1575  unsigned long wqe_idx;
1576  size_t len;
1577  int rc = 0;
1578 
1579  /* Parse completion */
1580  qpn = MLX_GET ( &cqe->normal, my_qpn );
1581  is_send = MLX_GET ( &cqe->normal, s );
1582  wqe_adr = ( MLX_GET ( &cqe->normal, wqe_adr ) << 6 );
1583  opcode = MLX_GET ( &cqe->normal, opcode );
1584  if ( opcode >= ARBEL_OPCODE_RECV_ERROR ) {
1585  /* "s" field is not valid for error opcodes */
1586  is_send = ( opcode == ARBEL_OPCODE_SEND_ERROR );
1587  DBGC ( arbel, "Arbel %p CQN %#lx %s QPN %#lx syndrome %#x "
1588  "vendor %#x\n", arbel, cq->cqn,
1589  ( is_send ? "send" : "recv" ), qpn,
1590  MLX_GET ( &cqe->error, syndrome ),
1591  MLX_GET ( &cqe->error, vendor_code ) );
1592  DBGC_HDA ( arbel, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
1593  rc = -EIO;
1594  /* Don't return immediately; propagate error to completer */
1595  }
1596 
1597  /* Identify work queue */
1598  wq = ib_find_wq ( cq, qpn, is_send );
1599  if ( ! wq ) {
1600  DBGC ( arbel, "Arbel %p CQN %#lx unknown %s QPN %#lx\n",
1601  arbel, cq->cqn, ( is_send ? "send" : "recv" ), qpn );
1602  return -EIO;
1603  }
1604  qp = wq->qp;
1605  arbel_qp = ib_qp_get_drvdata ( qp );
1606  arbel_send_wq = &arbel_qp->send;
1607  arbel_recv_wq = &arbel_qp->recv;
1608 
1609  /* Identify work queue entry index */
1610  if ( is_send ) {
1611  wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_send_wq->wqe ) ) /
1612  sizeof ( arbel_send_wq->wqe[0] ) );
1613  assert ( wqe_idx < qp->send.num_wqes );
1614  } else {
1615  wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_recv_wq->wqe ) ) /
1616  sizeof ( arbel_recv_wq->wqe[0] ) );
1617  assert ( wqe_idx < qp->recv.num_wqes );
1618  }
1619 
1620  DBGCP ( arbel, "Arbel %p CQN %#lx QPN %#lx %s WQE %#lx completed:\n",
1621  arbel, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
1622  wqe_idx );
1623  DBGCP_HDA ( arbel, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
1624 
1625  /* Identify I/O buffer */
1626  iobuf = wq->iobufs[wqe_idx];
1627  if ( ! iobuf ) {
1628  DBGC ( arbel, "Arbel %p CQN %#lx QPN %#lx empty %s WQE %#lx\n",
1629  arbel, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
1630  wqe_idx );
1631  return -EIO;
1632  }
1633  wq->iobufs[wqe_idx] = NULL;
1634 
1635  if ( is_send ) {
1636  /* Hand off to completion handler */
1637  ib_complete_send ( ibdev, qp, iobuf, rc );
1638  } else {
1639  /* Set received length */
1640  len = MLX_GET ( &cqe->normal, byte_cnt );
1641  recv_wqe = &arbel_recv_wq->wqe[wqe_idx].recv;
1642  assert ( MLX_GET ( &recv_wqe->data[0], local_address_l ) ==
1643  virt_to_bus ( iobuf->data ) );
1644  assert ( MLX_GET ( &recv_wqe->data[0], byte_count ) ==
1645  iob_tailroom ( iobuf ) );
1646  MLX_FILL_1 ( &recv_wqe->data[0], 0, byte_count, 0 );
1647  MLX_FILL_1 ( &recv_wqe->data[0], 1,
1648  l_key, ARBEL_INVALID_LKEY );
1649  memset ( &recv_dest, 0, sizeof ( recv_dest ) );
1650  recv_dest.qpn = qpn;
1651  switch ( qp->type ) {
1652  case IB_QPT_SMI:
1653  case IB_QPT_GSI:
1654  case IB_QPT_UD:
1655  /* Locate corresponding GRH */
1656  assert ( arbel_recv_wq->grh != NULL );
1657  grh = &arbel_recv_wq->grh[wqe_idx];
1658  len -= sizeof ( *grh );
1659  /* Construct address vector */
1660  source = &recv_source;
1661  memset ( source, 0, sizeof ( *source ) );
1662  source->qpn = MLX_GET ( &cqe->normal, rqpn );
1663  source->lid = MLX_GET ( &cqe->normal, rlid );
1664  source->sl = MLX_GET ( &cqe->normal, sl );
1665  recv_dest.gid_present = source->gid_present =
1666  MLX_GET ( &cqe->normal, g );
1667  memcpy ( &recv_dest.gid, &grh->dgid,
1668  sizeof ( recv_dest.gid ) );
1669  memcpy ( &source->gid, &grh->sgid,
1670  sizeof ( source->gid ) );
1671  break;
1672  case IB_QPT_RC:
1673  source = &qp->av;
1674  break;
1675  default:
1676  assert ( 0 );
1677  return -EINVAL;
1678  }
1679  assert ( len <= iob_tailroom ( iobuf ) );
1680  iob_put ( iobuf, len );
1681  /* Hand off to completion handler */
1682  ib_complete_recv ( ibdev, qp, &recv_dest, source, iobuf, rc );
1683  }
1684 
1685  return rc;
1686 }
struct ib_global_route_header * grh
GRH buffers (if applicable)
Definition: arbel.h:373
#define EINVAL
Invalid argument.
Definition: errno.h:428
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define ARBEL_OPCODE_SEND_ERROR
Definition: arbel.h:50
#define iob_put(iobuf, len)
Definition: iobuf.h:120
union arbel_recv_wqe * wqe
Work queue entries.
Definition: arbel.h:369
An Arbel send work queue.
Definition: arbel.h:346
uint8_t opcode
Opcode.
Definition: ena.h:16
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
uint32_t g
Definition: sha256.c:34
struct arbelprm_wqe_segment_data_ptr data[ARBEL_MAX_SCATTER]
Definition: arbel.h:252
#define DBGC(...)
Definition: compiler.h:505
__be32 byte_count
Definition: CIB_PRM.h:28
static __always_inline void * ib_qp_get_drvdata(struct ib_queue_pair *qp)
Get Infiniband queue pair driver-private data.
Definition: infiniband.h:642
struct ib_global_route_header grh
Definition: ib_packet.h:16
unsigned int gid_present
GID is present.
Definition: infiniband.h:90
union arbel_send_wqe * wqe
Work queue entries.
Definition: arbel.h:350
struct arbelprm_recv_wqe recv
Definition: arbel.h:11
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
__be32 qpn
Definition: CIB_PRM.h:29
union ib_gid dgid
Destiniation GID.
Definition: ib_packet.h:106
#define DBGCP_HDA(...)
Definition: compiler.h:540
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
An Infiniband Work Queue.
Definition: infiniband.h:100
#define DBGC_HDA(...)
Definition: compiler.h:506
void ib_complete_send(struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf, int rc)
Complete send work queue entry.
Definition: infiniband.c:515
#define ARBEL_INVALID_LKEY
Definition: arbel.h:98
An Infiniband Global Route Header.
Definition: ib_packet.h:89
struct arbelprm_completion_with_error error
Definition: arbel.h:257
struct ib_work_queue * ib_find_wq(struct ib_completion_queue *cq, unsigned long qpn, int is_send)
Find work queue belonging to completion queue.
Definition: infiniband.c:396
An Arbel receive work queue.
Definition: arbel.h:365
An Arbel queue pair.
Definition: arbel.h:406
union ib_gid sgid
Source GID.
Definition: ib_packet.h:104
struct arbelprm_completion_queue_entry normal
Definition: arbel.h:256
__be16 rlid
Definition: CIB_PRM.h:38
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
u8 sl
Definition: CIB_PRM.h:42
static size_t iob_tailroom(struct io_buffer *iobuf)
Calculate available space at end of an I/O buffer.
Definition: iobuf.h:175
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
unsigned long qpn
Queue Pair Number.
Definition: infiniband.h:74
struct ib_queue_pair * qp
Containing queue pair.
Definition: infiniband.h:102
uint16_t syndrome
ID of event.
Definition: ena.h:14
An Infiniband Queue Pair.
Definition: infiniband.h:157
unsigned int sl
Service level.
Definition: infiniband.h:88
#define ARBEL_OPCODE_RECV_ERROR
Definition: arbel.h:49
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
uint32_t len
Length.
Definition: ena.h:14
void * data
Start of data.
Definition: iobuf.h:48
#define EIO
Input/output error.
Definition: errno.h:433
struct arbelprm_recv_wqe recv
Definition: arbel.h:360
union ib_gid gid
GID, if present.
Definition: infiniband.h:92
unsigned long cqn
Completion queue number.
Definition: infiniband.h:230
#define DBGCP(...)
Definition: compiler.h:539
void ib_complete_recv(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct ib_address_vector *source, struct io_buffer *iobuf, int rc)
Complete receive work queue entry.
Definition: infiniband.c:536
struct arbel_recv_work_queue recv
Receive work queue.
Definition: arbel.h:410
An Infiniband Address Vector.
Definition: infiniband.h:72
unsigned int lid
Local ID.
Definition: infiniband.h:81
struct arbel_send_work_queue send
Send work queue.
Definition: arbel.h:408
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
__be32 byte_cnt
Definition: CIB_PRM.h:37
void * memset(void *dest, int character, size_t len) __nonnull
A persistent I/O buffer.
Definition: iobuf.h:33
struct arbelprm_send_doorbell send
Definition: arbel.h:11
struct io_buffer ** iobufs
I/O buffers assigned to work queue.
Definition: infiniband.h:124

References ARBEL_INVALID_LKEY, ARBEL_OPCODE_RECV_ERROR, ARBEL_OPCODE_SEND_ERROR, assert(), byte_cnt, byte_count, ib_completion_queue::cqn, io_buffer::data, arbelprm_recv_wqe::data, DBGC, DBGC_HDA, DBGCP, DBGCP_HDA, ib_global_route_header::dgid, EINVAL, EIO, arbelprm_completion_entry::error, g, ib_address_vector::gid, ib_address_vector::gid_present, grh, arbel_recv_work_queue::grh, ib_complete_recv(), ib_complete_send(), ib_find_wq(), ib_get_drvdata(), ib_qp_get_drvdata(), IB_QPT_GSI, IB_QPT_RC, IB_QPT_SMI, IB_QPT_UD, arbel::ibdev, iob_put, iob_tailroom(), ib_work_queue::iobufs, len, ib_address_vector::lid, memcpy(), memset(), MLX_FILL_1, MLX_GET, arbelprm_completion_entry::normal, NULL, opcode, qp, ib_work_queue::qp, qpn, ib_address_vector::qpn, rc, arbel_recv_wqe::recv, recv, arbel_queue_pair::recv, rlid, send, arbel_queue_pair::send, ib_global_route_header::sgid, sl, ib_address_vector::sl, syndrome, virt_to_bus(), virt_to_phys(), arbel_send_work_queue::wqe, and arbel_recv_work_queue::wqe.

Referenced by arbel_poll_cq().

◆ arbel_poll_cq()

static void arbel_poll_cq ( struct ib_device ibdev,
struct ib_completion_queue cq 
)
static

Poll completion queue.

Parameters
ibdevInfiniband device
cqCompletion queue

Definition at line 1694 of file arbel.c.

1695  {
1696  struct arbel *arbel = ib_get_drvdata ( ibdev );
1697  struct arbel_completion_queue *arbel_cq = ib_cq_get_drvdata ( cq );
1698  struct arbelprm_cq_ci_db_record *ci_db_rec;
1699  union arbelprm_completion_entry *cqe;
1700  unsigned int cqe_idx_mask;
1701  int rc;
1702 
1703  while ( 1 ) {
1704  /* Look for completion entry */
1705  cqe_idx_mask = ( cq->num_cqes - 1 );
1706  cqe = &arbel_cq->cqe[cq->next_idx & cqe_idx_mask];
1707  if ( MLX_GET ( &cqe->normal, owner ) != 0 ) {
1708  /* Entry still owned by hardware; end of poll */
1709  break;
1710  }
1711 
1712  /* Handle completion */
1713  if ( ( rc = arbel_complete ( ibdev, cq, cqe ) ) != 0 ) {
1714  DBGC ( arbel, "Arbel %p CQN %#lx failed to complete: "
1715  "%s\n", arbel, cq->cqn, strerror ( rc ) );
1716  DBGC_HD ( arbel, cqe, sizeof ( *cqe ) );
1717  }
1718 
1719  /* Return ownership to hardware */
1720  MLX_FILL_1 ( &cqe->normal, 7, owner, 1 );
1721  barrier();
1722  /* Update completion queue's index */
1723  cq->next_idx++;
1724  /* Update doorbell record */
1725  ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
1726  MLX_FILL_1 ( ci_db_rec, 0,
1727  counter, ( cq->next_idx & 0xffffffffUL ) );
1728  }
1729 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
u8 owner
Definition: CIB_PRM.h:36
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
unsigned int ci_doorbell_idx
Consumer counter doorbell record number.
Definition: arbel.h:424
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
union arbelprm_completion_entry * cqe
Completion queue entries.
Definition: arbel.h:428
struct arbelprm_cq_ci_db_record cq_ci
Definition: arbel.h:267
static int arbel_complete(struct ib_device *ibdev, struct ib_completion_queue *cq, union arbelprm_completion_entry *cqe)
Handle completion.
Definition: arbel.c:1556
An Arbel completion queue.
Definition: arbel.h:422
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
struct arbelprm_completion_queue_entry normal
Definition: arbel.h:256
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
#define DBGC_HD(...)
Definition: compiler.h:507
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
unsigned long next_idx
Next completion queue entry index.
Definition: infiniband.h:240
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
unsigned long cqn
Completion queue number.
Definition: infiniband.h:230
unsigned int num_cqes
Number of completion queue entries.
Definition: infiniband.h:232
Definition: arbel.h:255
static __always_inline void * ib_cq_get_drvdata(struct ib_completion_queue *cq)
Get Infiniband completion queue driver-private data.
Definition: infiniband.h:686
An Arbel device.
Definition: arbel.h:469

References arbel_complete(), barrier, arbel_completion_queue::ci_doorbell_idx, arbelprm_doorbell_record::cq_ci, arbel_completion_queue::cqe, ib_completion_queue::cqn, arbel::db_rec, DBGC, DBGC_HD, ib_cq_get_drvdata(), ib_get_drvdata(), arbel::ibdev, MLX_FILL_1, MLX_GET, ib_completion_queue::next_idx, arbelprm_completion_entry::normal, ib_completion_queue::num_cqes, owner, rc, and strerror().

◆ arbel_create_eq()

static int arbel_create_eq ( struct arbel arbel)
static

Create event queue.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 1744 of file arbel.c.

1744  {
1745  struct arbel_event_queue *arbel_eq = &arbel->eq;
1746  struct arbelprm_eqc eqctx;
1747  struct arbelprm_event_mask mask;
1748  unsigned int i;
1749  int rc;
1750 
1751  /* Select event queue number */
1752  arbel_eq->eqn = arbel->limits.reserved_eqs;
1753 
1754  /* Calculate doorbell address */
1755  arbel_eq->doorbell = ( arbel->eq_ci_doorbells +
1756  ARBEL_DB_EQ_OFFSET ( arbel_eq->eqn ) );
1757 
1758  /* Allocate event queue itself */
1759  arbel_eq->eqe_size =
1760  ( ARBEL_NUM_EQES * sizeof ( arbel_eq->eqe[0] ) );
1761  arbel_eq->eqe = malloc_phys ( arbel_eq->eqe_size,
1762  sizeof ( arbel_eq->eqe[0] ) );
1763  if ( ! arbel_eq->eqe ) {
1764  rc = -ENOMEM;
1765  goto err_eqe;
1766  }
1767  memset ( arbel_eq->eqe, 0, arbel_eq->eqe_size );
1768  for ( i = 0 ; i < ARBEL_NUM_EQES ; i++ ) {
1769  MLX_FILL_1 ( &arbel_eq->eqe[i].generic, 7, owner, 1 );
1770  }
1771  barrier();
1772 
1773  /* Hand queue over to hardware */
1774  memset ( &eqctx, 0, sizeof ( eqctx ) );
1775  MLX_FILL_1 ( &eqctx, 0, st, 0xa /* "Fired" */ );
1776  MLX_FILL_H ( &eqctx, 1,
1777  start_address_h, virt_to_phys ( arbel_eq->eqe ) );
1778  MLX_FILL_1 ( &eqctx, 2,
1779  start_address_l, virt_to_phys ( arbel_eq->eqe ) );
1780  MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( ARBEL_NUM_EQES - 1 ) );
1781  MLX_FILL_1 ( &eqctx, 6, pd, ARBEL_GLOBAL_PD );
1782  MLX_FILL_1 ( &eqctx, 7, lkey, arbel->lkey );
1783  if ( ( rc = arbel_cmd_sw2hw_eq ( arbel, arbel_eq->eqn,
1784  &eqctx ) ) != 0 ) {
1785  DBGC ( arbel, "Arbel %p EQN %#lx SW2HW_EQ failed: %s\n",
1786  arbel, arbel_eq->eqn, strerror ( rc ) );
1787  goto err_sw2hw_eq;
1788  }
1789 
1790  /* Map events to this event queue */
1791  memset ( &mask, 0xff, sizeof ( mask ) );
1792  if ( ( rc = arbel_cmd_map_eq ( arbel,
1793  ( ARBEL_MAP_EQ | arbel_eq->eqn ),
1794  &mask ) ) != 0 ) {
1795  DBGC ( arbel, "Arbel %p EQN %#lx MAP_EQ failed: %s\n",
1796  arbel, arbel_eq->eqn, strerror ( rc ) );
1797  goto err_map_eq;
1798  }
1799 
1800  DBGC ( arbel, "Arbel %p EQN %#lx ring [%08lx,%08lx), doorbell %08lx\n",
1801  arbel, arbel_eq->eqn, virt_to_phys ( arbel_eq->eqe ),
1802  ( virt_to_phys ( arbel_eq->eqe ) + arbel_eq->eqe_size ),
1803  virt_to_phys ( arbel_eq->doorbell ) );
1804  return 0;
1805 
1806  err_map_eq:
1807  arbel_cmd_hw2sw_eq ( arbel, arbel_eq->eqn, &eqctx );
1808  err_sw2hw_eq:
1809  free_phys ( arbel_eq->eqe, arbel_eq->eqe_size );
1810  err_eqe:
1811  memset ( arbel_eq, 0, sizeof ( *arbel_eq ) );
1812  return rc;
1813 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_cmd_map_eq(struct arbel *arbel, unsigned long index_map, const struct arbelprm_event_mask *mask)
Definition: arbel.c:295
void * eq_ci_doorbells
Event queue consumer index doorbells.
Definition: arbel.h:477
__be32 lkey
Definition: CIB_PRM.h:29
u8 owner
Definition: CIB_PRM.h:36
#define DBGC(...)
Definition: compiler.h:505
#define ARBEL_GLOBAL_PD
Global protection domain.
Definition: arbel.h:536
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
unsigned int reserved_eqs
Number of reserved EQs.
Definition: arbel.h:314
unsigned long eqn
Event queue number.
Definition: arbel.h:446
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
struct arbel_event_queue eq
Event queue.
Definition: arbel.h:512
union arbelprm_event_entry * eqe
Event queue entries.
Definition: arbel.h:442
struct sockaddr_tcpip st
Definition: syslog.c:56
static int arbel_cmd_sw2hw_eq(struct arbel *arbel, unsigned int index, const struct arbelprm_eqc *eqctx)
Definition: arbel.c:304
#define ENOMEM
Not enough space.
Definition: errno.h:534
static int arbel_cmd_hw2sw_eq(struct arbel *arbel, unsigned int index, struct arbelprm_eqc *eqctx)
Definition: arbel.c:313
#define ARBEL_NUM_EQES
Number of event queue entries.
Definition: arbel.h:457
struct arbelprm_event_queue_entry generic
Definition: arbel.h:261
#define ARBEL_DB_EQ_OFFSET(_eqn)
Definition: arbel.h:105
size_t eqe_size
Size of event queue.
Definition: arbel.h:444
#define ARBEL_MAP_EQ
Definition: arbel.h:109
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
void * doorbell
Doorbell register.
Definition: arbel.h:450
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
#define fls(x)
Find last (i.e.
Definition: strings.h:166
An Arbel device.
Definition: arbel.h:469
A Arbel event queue.
Definition: arbel.h:440
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_hw2sw_eq(), arbel_cmd_map_eq(), arbel_cmd_sw2hw_eq(), ARBEL_DB_EQ_OFFSET, ARBEL_GLOBAL_PD, ARBEL_MAP_EQ, ARBEL_NUM_EQES, barrier, DBGC, arbel_event_queue::doorbell, ENOMEM, arbel::eq, arbel::eq_ci_doorbells, arbel_event_queue::eqe, arbel_event_queue::eqe_size, arbel_event_queue::eqn, fls, free_phys(), arbelprm_event_entry::generic, arbel::limits, lkey, arbel::lkey, malloc_phys(), memset(), MLX_FILL_1, MLX_FILL_H, owner, rc, arbel_dev_limits::reserved_eqs, st, strerror(), and virt_to_phys().

Referenced by arbel_start().

◆ arbel_destroy_eq()

static void arbel_destroy_eq ( struct arbel arbel)
static

Destroy event queue.

Parameters
arbelArbel device

Definition at line 1820 of file arbel.c.

1820  {
1821  struct arbel_event_queue *arbel_eq = &arbel->eq;
1822  struct arbelprm_eqc eqctx;
1823  struct arbelprm_event_mask mask;
1824  int rc;
1825 
1826  /* Unmap events from event queue */
1827  memset ( &mask, 0, sizeof ( mask ) );
1828  MLX_FILL_1 ( &mask, 1, port_state_change, 1 );
1829  if ( ( rc = arbel_cmd_map_eq ( arbel,
1830  ( ARBEL_UNMAP_EQ | arbel_eq->eqn ),
1831  &mask ) ) != 0 ) {
1832  DBGC ( arbel, "Arbel %p EQN %#lx FATAL MAP_EQ failed to "
1833  "unmap: %s\n", arbel, arbel_eq->eqn, strerror ( rc ) );
1834  /* Continue; HCA may die but system should survive */
1835  }
1836 
1837  /* Take ownership back from hardware */
1838  if ( ( rc = arbel_cmd_hw2sw_eq ( arbel, arbel_eq->eqn,
1839  &eqctx ) ) != 0 ) {
1840  DBGC ( arbel, "Arbel %p EQN %#lx FATAL HW2SW_EQ failed: %s\n",
1841  arbel, arbel_eq->eqn, strerror ( rc ) );
1842  /* Leak memory and return; at least we avoid corruption */
1843  return;
1844  }
1845 
1846  /* Free memory */
1847  free_phys ( arbel_eq->eqe, arbel_eq->eqe_size );
1848  memset ( arbel_eq, 0, sizeof ( *arbel_eq ) );
1849 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_cmd_map_eq(struct arbel *arbel, unsigned long index_map, const struct arbelprm_event_mask *mask)
Definition: arbel.c:295
#define DBGC(...)
Definition: compiler.h:505
unsigned long eqn
Event queue number.
Definition: arbel.h:446
struct arbel_event_queue eq
Event queue.
Definition: arbel.h:512
union arbelprm_event_entry * eqe
Event queue entries.
Definition: arbel.h:442
static int arbel_cmd_hw2sw_eq(struct arbel *arbel, unsigned int index, struct arbelprm_eqc *eqctx)
Definition: arbel.c:313
#define ARBEL_UNMAP_EQ
Definition: arbel.h:110
size_t eqe_size
Size of event queue.
Definition: arbel.h:444
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
pseudo_bit_t port_state_change[0x00001]
Definition: arbel.h:22
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
An Arbel device.
Definition: arbel.h:469
A Arbel event queue.
Definition: arbel.h:440
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_hw2sw_eq(), arbel_cmd_map_eq(), ARBEL_UNMAP_EQ, DBGC, arbel::eq, arbel_event_queue::eqe, arbel_event_queue::eqe_size, arbel_event_queue::eqn, free_phys(), memset(), MLX_FILL_1, port_state_change, rc, and strerror().

Referenced by arbel_start(), and arbel_stop().

◆ arbel_event_port_state_change()

static void arbel_event_port_state_change ( struct arbel arbel,
union arbelprm_event_entry eqe 
)
static

Handle port state event.

Parameters
arbelArbel device
eqePort state change event queue entry

Definition at line 1857 of file arbel.c.

1858  {
1859  unsigned int port;
1860  int link_up;
1861 
1862  /* Get port and link status */
1863  port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 );
1864  link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 );
1865  DBGC ( arbel, "Arbel %p port %d link %s\n", arbel, ( port + 1 ),
1866  ( link_up ? "up" : "down" ) );
1867 
1868  /* Sanity check */
1869  if ( port >= ARBEL_NUM_PORTS ) {
1870  DBGC ( arbel, "Arbel %p port %d does not exist!\n",
1871  arbel, ( port + 1 ) );
1872  return;
1873  }
1874 
1875  /* Update MAD parameters */
1877 }
static int arbel_mad(struct ib_device *ibdev, union ib_mad *mad)
Issue management datagram.
Definition: arbel.c:543
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
u8 port
Port number.
Definition: CIB_PRM.h:31
struct arbelprm_event_queue_entry generic
Definition: arbel.h:261
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
int ib_smc_update(struct ib_device *ibdev, ib_local_mad_t local_mad)
Update Infiniband parameters using SMC.
Definition: ib_smc.c:249
struct arbelprm_port_state_change_event port_state_change
Definition: arbel.h:262
#define ARBEL_NUM_PORTS
Definition: arbel.h:24
uint8_t data[48]
Additional event data.
Definition: ena.h:22
An Arbel device.
Definition: arbel.h:469

References arbel_mad(), ARBEL_NUM_PORTS, data, DBGC, arbelprm_event_entry::generic, ib_smc_update(), arbel::ibdev, MLX_GET, port, and arbelprm_event_entry::port_state_change.

Referenced by arbel_poll_eq().

◆ arbel_poll_eq()

static void arbel_poll_eq ( struct ib_device ibdev)
static

Poll event queue.

Parameters
ibdevInfiniband device

Definition at line 1884 of file arbel.c.

1884  {
1885  struct arbel *arbel = ib_get_drvdata ( ibdev );
1886  struct arbel_event_queue *arbel_eq = &arbel->eq;
1887  union arbelprm_event_entry *eqe;
1888  union arbelprm_eq_doorbell_register db_reg;
1889  unsigned int eqe_idx_mask;
1890  unsigned int event_type;
1891 
1892  /* No event is generated upon reaching INIT, so we must poll
1893  * separately for link state changes while we remain DOWN.
1894  */
1895  if ( ib_is_open ( ibdev ) &&
1896  ( ibdev->port_state == IB_PORT_STATE_DOWN ) ) {
1897  ib_smc_update ( ibdev, arbel_mad );
1898  }
1899 
1900  /* Poll event queue */
1901  while ( 1 ) {
1902  /* Look for event entry */
1903  eqe_idx_mask = ( ARBEL_NUM_EQES - 1 );
1904  eqe = &arbel_eq->eqe[arbel_eq->next_idx & eqe_idx_mask];
1905  if ( MLX_GET ( &eqe->generic, owner ) != 0 ) {
1906  /* Entry still owned by hardware; end of poll */
1907  break;
1908  }
1909  DBGCP ( arbel, "Arbel %p EQN %#lx event:\n",
1910  arbel, arbel_eq->eqn );
1911  DBGCP_HDA ( arbel, virt_to_phys ( eqe ),
1912  eqe, sizeof ( *eqe ) );
1913 
1914  /* Handle event */
1915  event_type = MLX_GET ( &eqe->generic, event_type );
1916  switch ( event_type ) {
1919  break;
1920  default:
1921  DBGC ( arbel, "Arbel %p EQN %#lx unrecognised event "
1922  "type %#x:\n",
1923  arbel, arbel_eq->eqn, event_type );
1924  DBGC_HDA ( arbel, virt_to_phys ( eqe ),
1925  eqe, sizeof ( *eqe ) );
1926  break;
1927  }
1928 
1929  /* Return ownership to hardware */
1930  MLX_FILL_1 ( &eqe->generic, 7, owner, 1 );
1931  barrier();
1932 
1933  /* Update event queue's index */
1934  arbel_eq->next_idx++;
1935 
1936  /* Ring doorbell */
1937  MLX_FILL_1 ( &db_reg.ci, 0, ci, arbel_eq->next_idx );
1938  writel ( db_reg.dword[0], arbel_eq->doorbell );
1939  }
1940 }
static int arbel_mad(struct ib_device *ibdev, union ib_mad *mad)
Issue management datagram.
Definition: arbel.c:543
static int ib_is_open(struct ib_device *ibdev)
Check whether or not Infiniband device is open.
Definition: infiniband.h:576
u8 owner
Definition: CIB_PRM.h:36
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
unsigned long eqn
Event queue number.
Definition: arbel.h:446
Definition: arbel.h:260
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
struct arbel_event_queue eq
Event queue.
Definition: arbel.h:512
union arbelprm_event_entry * eqe
Event queue entries.
Definition: arbel.h:442
pseudo_bit_t ci[0x00020]
Definition: arbel.h:11
#define DBGCP_HDA(...)
Definition: compiler.h:540
#define ARBEL_NUM_EQES
Number of event queue entries.
Definition: arbel.h:457
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
struct arbelprm_event_queue_entry generic
Definition: arbel.h:261
#define DBGC_HDA(...)
Definition: compiler.h:506
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
#define ARBEL_EV_PORT_STATE_CHANGE
Definition: arbel.h:112
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
void * doorbell
Doorbell register.
Definition: arbel.h:450
int ib_smc_update(struct ib_device *ibdev, ib_local_mad_t local_mad)
Update Infiniband parameters using SMC.
Definition: ib_smc.c:249
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define IB_PORT_STATE_DOWN
Definition: ib_mad.h:151
#define barrier()
Optimisation barrier.
Definition: compiler.h:655
uint8_t port_state
Port state.
Definition: infiniband.h:425
#define DBGCP(...)
Definition: compiler.h:539
unsigned long next_idx
Next event queue entry index.
Definition: arbel.h:448
static void arbel_event_port_state_change(struct arbel *arbel, union arbelprm_event_entry *eqe)
Handle port state event.
Definition: arbel.c:1857
An Arbel device.
Definition: arbel.h:469
A Arbel event queue.
Definition: arbel.h:440

References ARBEL_EV_PORT_STATE_CHANGE, arbel_event_port_state_change(), arbel_mad(), ARBEL_NUM_EQES, barrier, ci, arbelprm_eq_doorbell_register::ci, DBGC, DBGC_HDA, DBGCP, DBGCP_HDA, arbel_event_queue::doorbell, arbelprm_eq_doorbell_register::dword, arbel::eq, arbel_event_queue::eqe, arbel_event_queue::eqn, arbelprm_event_entry::generic, ib_get_drvdata(), ib_is_open(), IB_PORT_STATE_DOWN, ib_smc_update(), arbel::ibdev, MLX_FILL_1, MLX_GET, arbel_event_queue::next_idx, owner, ib_device::port_state, virt_to_phys(), and writel().

◆ arbel_map_vpm()

static int arbel_map_vpm ( struct arbel arbel,
int(*)(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *)  map,
uint64_t  va,
physaddr_t  pa,
size_t  len 
)
static

Map virtual to physical address for firmware usage.

Parameters
arbelArbel device
mapMapping function
vaVirtual address
paPhysical address
lenLength of region
Return values
rcReturn status code

Definition at line 1959 of file arbel.c.

1962  {
1963  struct arbelprm_virtual_physical_mapping mapping;
1964  physaddr_t start;
1965  physaddr_t low;
1966  physaddr_t high;
1967  physaddr_t end;
1968  size_t size;
1969  int rc;
1970 
1971  /* Sanity checks */
1972  assert ( ( va & ( ARBEL_PAGE_SIZE - 1 ) ) == 0 );
1973  assert ( ( pa & ( ARBEL_PAGE_SIZE - 1 ) ) == 0 );
1974  assert ( ( len & ( ARBEL_PAGE_SIZE - 1 ) ) == 0 );
1975  assert ( len != 0 );
1976 
1977  /* Calculate starting points */
1978  start = pa;
1979  end = ( start + len );
1980  size = ( 1UL << ( fls ( start ^ end ) - 1 ) );
1981  low = high = ( end & ~( size - 1 ) );
1982  assert ( start < low );
1983  assert ( high <= end );
1984 
1985  /* These mappings tend to generate huge volumes of
1986  * uninteresting debug data, which basically makes it
1987  * impossible to use debugging otherwise.
1988  */
1990 
1991  /* Map blocks in descending order of size */
1992  while ( size >= ARBEL_PAGE_SIZE ) {
1993 
1994  /* Find the next candidate block */
1995  if ( ( low - size ) >= start ) {
1996  low -= size;
1997  pa = low;
1998  } else if ( high <= ( end - size ) ) {
1999  pa = high;
2000  high += size;
2001  } else {
2002  size >>= 1;
2003  continue;
2004  }
2005  assert ( ( va & ( size - 1 ) ) == 0 );
2006  assert ( ( pa & ( size - 1 ) ) == 0 );
2007 
2008  /* Map this block */
2009  memset ( &mapping, 0, sizeof ( mapping ) );
2010  MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) );
2011  MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) );
2012  MLX_FILL_H ( &mapping, 2, pa_h, pa );
2013  MLX_FILL_2 ( &mapping, 3,
2014  log2size, ( ( fls ( size ) - 1 ) - 12 ),
2015  pa_l, ( pa >> 12 ) );
2016  if ( ( rc = map ( arbel, &mapping ) ) != 0 ) {
2018  DBGC ( arbel, "Arbel %p could not map %08llx+%zx to "
2019  "%08lx: %s\n",
2020  arbel, va, size, pa, strerror ( rc ) );
2021  return rc;
2022  }
2023  va += size;
2024  }
2025  assert ( low == start );
2026  assert ( high == end );
2027 
2029  return 0;
2030 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
uint32_t low
Low 16 bits of address.
Definition: myson.h:19
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
#define DBG_ENABLE(level)
Definition: compiler.h:313
#define DBGC(...)
Definition: compiler.h:505
#define DBG_DISABLE(level)
Definition: compiler.h:312
uint32_t start
Starting offset.
Definition: netvsc.h:12
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
#define DBGLVL_LOG
Definition: compiler.h:316
uint32_t high
High 32 bits of address.
Definition: myson.h:20
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define MLX_FILL_H(_structure_st, _index, _field, _address)
Definition: mlx_bitops.h:240
static __always_inline int struct dma_mapping * map
Definition: dma.h:181
unsigned long physaddr_t
Definition: stdint.h:20
uint32_t len
Length.
Definition: ena.h:14
#define ARBEL_PAGE_SIZE
Definition: arbel.h:100
uint32_t end
Ending offset.
Definition: netvsc.h:18
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:16
#define fls(x)
Find last (i.e.
Definition: strings.h:166
#define DBGLVL_EXTRA
Definition: compiler.h:318
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References ARBEL_PAGE_SIZE, assert(), DBG_DISABLE, DBG_ENABLE, DBGC, DBGLVL_EXTRA, DBGLVL_LOG, end, fls, high, len, low, map, memset(), MLX_FILL_1, MLX_FILL_2, MLX_FILL_H, rc, size, start, and strerror().

Referenced by arbel_alloc_icm(), and arbel_start_firmware().

◆ arbel_start_firmware()

static int arbel_start_firmware ( struct arbel arbel)
static

Start firmware running.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 2038 of file arbel.c.

2038  {
2039  struct arbelprm_query_fw fw;
2040  struct arbelprm_access_lam lam;
2041  unsigned int fw_pages;
2042  size_t fw_len;
2043  physaddr_t fw_base;
2044  uint64_t eq_set_ci_base_addr;
2045  int rc;
2046 
2047  /* Get firmware parameters */
2048  if ( ( rc = arbel_cmd_query_fw ( arbel, &fw ) ) != 0 ) {
2049  DBGC ( arbel, "Arbel %p could not query firmware: %s\n",
2050  arbel, strerror ( rc ) );
2051  goto err_query_fw;
2052  }
2053  DBGC ( arbel, "Arbel %p firmware version %d.%d.%d\n", arbel,
2054  MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
2055  MLX_GET ( &fw, fw_rev_subminor ) );
2056  fw_pages = MLX_GET ( &fw, fw_pages );
2057  DBGC ( arbel, "Arbel %p requires %d kB for firmware\n",
2058  arbel, ( fw_pages * 4 ) );
2059  eq_set_ci_base_addr =
2060  ( ( (uint64_t) MLX_GET ( &fw, eq_set_ci_base_addr_h ) << 32 ) |
2061  ( (uint64_t) MLX_GET ( &fw, eq_set_ci_base_addr_l ) ) );
2062  arbel->eq_ci_doorbells = pci_ioremap ( arbel->pci, eq_set_ci_base_addr,
2063  0x200 );
2064 
2065  /* Enable locally-attached memory. Ignore failure; there may
2066  * be no attached memory.
2067  */
2068  arbel_cmd_enable_lam ( arbel, &lam );
2069 
2070  /* Allocate firmware pages and map firmware area */
2071  fw_len = ( fw_pages * ARBEL_PAGE_SIZE );
2072  if ( ! arbel->firmware_area ) {
2073  arbel->firmware_len = fw_len;
2075  if ( ! arbel->firmware_area ) {
2076  rc = -ENOMEM;
2077  goto err_alloc_fa;
2078  }
2079  } else {
2080  assert ( arbel->firmware_len == fw_len );
2081  }
2082  fw_base = user_to_phys ( arbel->firmware_area, 0 );
2083  DBGC ( arbel, "Arbel %p firmware area at [%08lx,%08lx)\n",
2084  arbel, fw_base, ( fw_base + fw_len ) );
2085  if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_fa,
2086  0, fw_base, fw_len ) ) != 0 ) {
2087  DBGC ( arbel, "Arbel %p could not map firmware: %s\n",
2088  arbel, strerror ( rc ) );
2089  goto err_map_fa;
2090  }
2091 
2092  /* Start firmware */
2093  if ( ( rc = arbel_cmd_run_fw ( arbel ) ) != 0 ) {
2094  DBGC ( arbel, "Arbel %p could not run firmware: %s\n",
2095  arbel, strerror ( rc ) );
2096  goto err_run_fw;
2097  }
2098 
2099  DBGC ( arbel, "Arbel %p firmware started\n", arbel );
2100  return 0;
2101 
2102  err_run_fw:
2104  err_map_fa:
2105  err_alloc_fa:
2106  err_query_fw:
2107  return rc;
2108 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_cmd_map_fa(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *map)
Definition: arbel.c:521
void * eq_ci_doorbells
Event queue consumer index doorbells.
Definition: arbel.h:477
unsigned long user_to_phys(userptr_t userptr, off_t offset)
Convert user pointer to physical address.
#define DBGC(...)
Definition: compiler.h:505
unsigned long long uint64_t
Definition: stdint.h:13
#define ENOMEM
Not enough space.
Definition: errno.h:534
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
userptr_t firmware_area
Firmware area in external memory.
Definition: arbel.h:495
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
unsigned long physaddr_t
Definition: stdint.h:20
static int arbel_cmd_enable_lam(struct arbel *arbel, struct arbelprm_access_lam *lam)
Definition: arbel.c:461
size_t firmware_len
Firmware size.
Definition: arbel.h:488
static __always_inline userptr_t umalloc(size_t size)
Allocate external memory.
Definition: umalloc.h:54
#define ARBEL_PAGE_SIZE
Definition: arbel.h:100
static int arbel_cmd_unmap_fa(struct arbel *arbel)
Definition: arbel.c:514
struct pci_device * pci
PCI device.
Definition: arbel.h:471
static int arbel_cmd_run_fw(struct arbel *arbel)
Definition: arbel.c:447
static int arbel_map_vpm(struct arbel *arbel, int(*map)(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *), uint64_t va, physaddr_t pa, size_t len)
Map virtual to physical address for firmware usage.
Definition: arbel.c:1959
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
An Arbel device.
Definition: arbel.h:469
static int arbel_cmd_query_fw(struct arbel *arbel, struct arbelprm_query_fw *fw)
Definition: arbel.c:246

References arbel_cmd_enable_lam(), arbel_cmd_map_fa(), arbel_cmd_query_fw(), arbel_cmd_run_fw(), arbel_cmd_unmap_fa(), arbel_map_vpm(), ARBEL_PAGE_SIZE, assert(), DBGC, ENOMEM, arbel::eq_ci_doorbells, arbel::firmware_area, arbel::firmware_len, MLX_GET, arbel::pci, pci_ioremap(), rc, strerror(), umalloc(), and user_to_phys().

Referenced by arbel_probe(), and arbel_start().

◆ arbel_stop_firmware()

static void arbel_stop_firmware ( struct arbel arbel)
static

Stop firmware running.

Parameters
arbelArbel device

Definition at line 2115 of file arbel.c.

2115  {
2116  int rc;
2117 
2118  if ( ( rc = arbel_cmd_unmap_fa ( arbel ) ) != 0 ) {
2119  DBGC ( arbel, "Arbel %p FATAL could not stop firmware: %s\n",
2120  arbel, strerror ( rc ) );
2121  /* Leak memory and return; at least we avoid corruption */
2123  return;
2124  }
2125 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define DBGC(...)
Definition: compiler.h:505
userptr_t firmware_area
Firmware area in external memory.
Definition: arbel.h:495
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define UNULL
Equivalent of NULL for user pointers.
Definition: uaccess.h:36
static int arbel_cmd_unmap_fa(struct arbel *arbel)
Definition: arbel.c:514
An Arbel device.
Definition: arbel.h:469

References arbel_cmd_unmap_fa(), DBGC, arbel::firmware_area, rc, strerror(), and UNULL.

Referenced by arbel_probe(), arbel_start(), and arbel_stop().

◆ arbel_get_limits()

static int arbel_get_limits ( struct arbel arbel)
static

Get device limits.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 2140 of file arbel.c.

2140  {
2141  struct arbelprm_query_dev_lim dev_lim;
2142  int rc;
2143 
2144  if ( ( rc = arbel_cmd_query_dev_lim ( arbel, &dev_lim ) ) != 0 ) {
2145  DBGC ( arbel, "Arbel %p could not get device limits: %s\n",
2146  arbel, strerror ( rc ) );
2147  return rc;
2148  }
2149 
2151  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_qps ) );
2152  arbel->limits.qpc_entry_size = MLX_GET ( &dev_lim, qpc_entry_sz );
2153  arbel->limits.eqpc_entry_size = MLX_GET ( &dev_lim, eqpc_entry_sz );
2155  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_srqs ) );
2156  arbel->limits.srqc_entry_size = MLX_GET ( &dev_lim, srq_entry_sz );
2158  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_ees ) );
2159  arbel->limits.eec_entry_size = MLX_GET ( &dev_lim, eec_entry_sz );
2160  arbel->limits.eeec_entry_size = MLX_GET ( &dev_lim, eeec_entry_sz );
2162  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_cqs ) );
2163  arbel->limits.cqc_entry_size = MLX_GET ( &dev_lim, cqc_entry_sz );
2165  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_mtts ) );
2166  arbel->limits.mtt_entry_size = MLX_GET ( &dev_lim, mtt_entry_sz );
2168  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_mrws ) );
2169  arbel->limits.mpt_entry_size = MLX_GET ( &dev_lim, mpt_entry_sz );
2171  ( 1 << MLX_GET ( &dev_lim, log2_rsvd_rdbs ) );
2172  arbel->limits.reserved_eqs = MLX_GET ( &dev_lim, num_rsvd_eqs );
2173  arbel->limits.eqc_entry_size = MLX_GET ( &dev_lim, eqc_entry_sz );
2174  arbel->limits.reserved_uars = MLX_GET ( &dev_lim, num_rsvd_uars );
2176  MLX_GET ( &dev_lim, uar_scratch_entry_sz );
2177 
2178  DBGC ( arbel, "Arbel %p reserves %d x %#zx QPC, %d x %#zx EQPC, "
2179  "%d x %#zx SRQC\n", arbel,
2183  DBGC ( arbel, "Arbel %p reserves %d x %#zx EEC, %d x %#zx EEEC, "
2184  "%d x %#zx CQC\n", arbel,
2188  DBGC ( arbel, "Arbel %p reserves %d x %#zx EQC, %d x %#zx MTT, "
2189  "%d x %#zx MPT\n", arbel,
2193  DBGC ( arbel, "Arbel %p reserves %d x %#zx RDB, %d x %#zx UAR, "
2194  "%d x %#zx UAR scratchpad\n", arbel,
2199 
2200  return 0;
2201 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
unsigned int reserved_srqs
Number of reserved SRQs.
Definition: arbel.h:300
unsigned int reserved_mtts
Number of reserved MTTs.
Definition: arbel.h:316
size_t eeec_entry_size
Extended EE context entry size.
Definition: arbel.h:308
size_t eqc_entry_size
EQ context entry size.
Definition: arbel.h:326
#define DBGC(...)
Definition: compiler.h:505
unsigned int reserved_eqs
Number of reserved EQs.
Definition: arbel.h:314
size_t cqc_entry_size
CQ context entry size.
Definition: arbel.h:312
size_t eqpc_entry_size
Extended QP context entry size.
Definition: arbel.h:298
unsigned int reserved_rdbs
Number of reserved RDBs.
Definition: arbel.h:324
size_t srqc_entry_size
SRQ context entry size.
Definition: arbel.h:302
#define ARBEL_RDB_ENTRY_SIZE
Definition: arbel.h:102
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
size_t mtt_entry_size
MTT entry size.
Definition: arbel.h:318
unsigned int reserved_cqs
Number of reserved CQs.
Definition: arbel.h:310
static int arbel_cmd_query_dev_lim(struct arbel *arbel, struct arbelprm_query_dev_lim *dev_lim)
Definition: arbel.c:237
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
size_t eec_entry_size
EE context entry size.
Definition: arbel.h:306
unsigned int reserved_mrws
Number of reserved MRWs.
Definition: arbel.h:320
unsigned int reserved_ees
Number of reserved EEs.
Definition: arbel.h:304
unsigned int reserved_qps
Number of reserved QPs.
Definition: arbel.h:294
size_t qpc_entry_size
QP context entry size.
Definition: arbel.h:296
#define ARBEL_PAGE_SIZE
Definition: arbel.h:100
size_t uar_scratch_entry_size
UAR scratchpad entry size.
Definition: arbel.h:330
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
unsigned int reserved_uars
Number of reserved UARs.
Definition: arbel.h:328
An Arbel device.
Definition: arbel.h:469
size_t mpt_entry_size
MPT entry size.
Definition: arbel.h:322

References arbel_cmd_query_dev_lim(), ARBEL_PAGE_SIZE, ARBEL_RDB_ENTRY_SIZE, arbel_dev_limits::cqc_entry_size, DBGC, arbel_dev_limits::eec_entry_size, arbel_dev_limits::eeec_entry_size, arbel_dev_limits::eqc_entry_size, arbel_dev_limits::eqpc_entry_size, arbel::limits, MLX_GET, arbel_dev_limits::mpt_entry_size, arbel_dev_limits::mtt_entry_size, arbel_dev_limits::qpc_entry_size, rc, arbel_dev_limits::reserved_cqs, arbel_dev_limits::reserved_ees, arbel_dev_limits::reserved_eqs, arbel_dev_limits::reserved_mrws, arbel_dev_limits::reserved_mtts, arbel_dev_limits::reserved_qps, arbel_dev_limits::reserved_rdbs, arbel_dev_limits::reserved_srqs, arbel_dev_limits::reserved_uars, arbel_dev_limits::srqc_entry_size, strerror(), and arbel_dev_limits::uar_scratch_entry_size.

Referenced by arbel_probe().

◆ icm_align()

static size_t icm_align ( size_t  icm_offset,
size_t  len 
)
static

Align ICM table.

Parameters
icm_offsetCurrent ICM offset
lenICM table length
Return values
icm_offsetICM offset

Definition at line 2210 of file arbel.c.

2210  {
2211 
2212  /* Round up to a multiple of the table size */
2213  assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) );
2214  return ( ( icm_offset + len - 1 ) & ~( len - 1 ) );
2215 }
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
uint32_t len
Length.
Definition: ena.h:14
#define fls(x)
Find last (i.e.
Definition: strings.h:166

References assert(), fls, and len.

Referenced by arbel_alloc_icm().

◆ arbel_alloc_icm()

static int arbel_alloc_icm ( struct arbel arbel,
struct arbelprm_init_hca *  init_hca 
)
static

Allocate ICM.

Parameters
arbelArbel device
init_hcaINIT_HCA structure to fill in
Return values
rcReturn status code

Definition at line 2224 of file arbel.c.

2225  {
2226  struct arbelprm_scalar_parameter icm_size;
2227  struct arbelprm_scalar_parameter icm_aux_size;
2228  struct arbelprm_scalar_parameter unmap_icm;
2229  union arbelprm_doorbell_record *db_rec;
2230  size_t icm_offset = 0;
2231  unsigned int log_num_uars, log_num_qps, log_num_srqs, log_num_ees;
2232  unsigned int log_num_cqs, log_num_mtts, log_num_mpts, log_num_rdbs;
2233  unsigned int log_num_eqs, log_num_mcs;
2234  size_t icm_len, icm_aux_len;
2235  size_t len;
2236  physaddr_t icm_phys;
2237  int rc;
2238 
2239  /* Calculate number of each object type within ICM */
2240  log_num_qps = fls ( arbel->limits.reserved_qps +
2242  log_num_srqs = fls ( arbel->limits.reserved_srqs - 1 );
2243  log_num_ees = fls ( arbel->limits.reserved_ees - 1 );
2244  log_num_cqs = fls ( arbel->limits.reserved_cqs + ARBEL_MAX_CQS - 1 );
2245  log_num_eqs = fls ( arbel->limits.reserved_eqs + ARBEL_MAX_EQS - 1 );
2246  log_num_mtts = fls ( arbel->limits.reserved_mtts - 1 );
2247  log_num_mpts = fls ( arbel->limits.reserved_mrws + 1 - 1 );
2248  log_num_rdbs = fls ( arbel->limits.reserved_rdbs +
2250  log_num_uars = fls ( arbel->limits.reserved_uars +
2251  1 /* single UAR used */ - 1 );
2252  log_num_mcs = ARBEL_LOG_MULTICAST_HASH_SIZE;
2253 
2254  /* Queue pair contexts */
2255  len = ( ( 1 << log_num_qps ) * arbel->limits.qpc_entry_size );
2256  icm_offset = icm_align ( icm_offset, len );
2257  MLX_FILL_2 ( init_hca, 13,
2258  qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l,
2259  ( icm_offset >> 7 ),
2260  qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp,
2261  log_num_qps );
2262  DBGC ( arbel, "Arbel %p ICM QPC is %d x %#zx at [%zx,%zx)\n",
2263  arbel, ( 1 << log_num_qps ), arbel->limits.qpc_entry_size,
2264  icm_offset, ( icm_offset + len ) );
2265  icm_offset += len;
2266 
2267  /* Extended queue pair contexts */
2268  len = ( ( 1 << log_num_qps ) * arbel->limits.eqpc_entry_size );
2269  icm_offset = icm_align ( icm_offset, len );
2270  MLX_FILL_1 ( init_hca, 25,
2271  qpc_eec_cqc_eqc_rdb_parameters.eqpc_base_addr_l,
2272  icm_offset );
2273  DBGC ( arbel, "Arbel %p ICM EQPC is %d x %#zx at [%zx,%zx)\n",
2274  arbel, ( 1 << log_num_qps ), arbel->limits.eqpc_entry_size,
2275  icm_offset, ( icm_offset + len ) );
2276  icm_offset += len;
2277 
2278  /* Completion queue contexts */
2279  len = ( ( 1 << log_num_cqs ) * arbel->limits.cqc_entry_size );
2280  icm_offset = icm_align ( icm_offset, len );
2281  MLX_FILL_2 ( init_hca, 21,
2282  qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l,
2283  ( icm_offset >> 6 ),
2284  qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq,
2285  log_num_cqs );
2286  DBGC ( arbel, "Arbel %p ICM CQC is %d x %#zx at [%zx,%zx)\n",
2287  arbel, ( 1 << log_num_cqs ), arbel->limits.cqc_entry_size,
2288  icm_offset, ( icm_offset + len ) );
2289  icm_offset += len;
2290 
2291  /* Event queue contexts */
2292  len = ( ( 1 << log_num_eqs ) * arbel->limits.eqc_entry_size );
2293  icm_offset = icm_align ( icm_offset, len );
2294  MLX_FILL_2 ( init_hca, 33,
2295  qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l,
2296  ( icm_offset >> 6 ),
2297  qpc_eec_cqc_eqc_rdb_parameters.log_num_eq,
2298  log_num_eqs );
2299  DBGC ( arbel, "Arbel %p ICM EQC is %d x %#zx at [%zx,%zx)\n",
2300  arbel, ( 1 << log_num_eqs ), arbel->limits.eqc_entry_size,
2301  icm_offset, ( icm_offset + len ) );
2302  icm_offset += len;
2303 
2304  /* End-to-end contexts */
2305  len = ( ( 1 << log_num_ees ) * arbel->limits.eec_entry_size );
2306  icm_offset = icm_align ( icm_offset, len );
2307  MLX_FILL_2 ( init_hca, 17,
2308  qpc_eec_cqc_eqc_rdb_parameters.eec_base_addr_l,
2309  ( icm_offset >> 7 ),
2310  qpc_eec_cqc_eqc_rdb_parameters.log_num_of_ee,
2311  log_num_ees );
2312  DBGC ( arbel, "Arbel %p ICM EEC is %d x %#zx at [%zx,%zx)\n",
2313  arbel, ( 1 << log_num_ees ), arbel->limits.eec_entry_size,
2314  icm_offset, ( icm_offset + len ) );
2315  icm_offset += len;
2316 
2317  /* Shared receive queue contexts */
2318  len = ( ( 1 << log_num_srqs ) * arbel->limits.srqc_entry_size );
2319  icm_offset = icm_align ( icm_offset, len );
2320  MLX_FILL_2 ( init_hca, 19,
2321  qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l,
2322  ( icm_offset >> 5 ),
2323  qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq,
2324  log_num_srqs );
2325  DBGC ( arbel, "Arbel %p ICM SRQC is %d x %#zx at [%zx,%zx)\n",
2326  arbel, ( 1 << log_num_srqs ), arbel->limits.srqc_entry_size,
2327  icm_offset, ( icm_offset + len ) );
2328  icm_offset += len;
2329 
2330  /* Memory protection table */
2331  len = ( ( 1 << log_num_mpts ) * arbel->limits.mpt_entry_size );
2332  icm_offset = icm_align ( icm_offset, len );
2333  MLX_FILL_1 ( init_hca, 61,
2334  tpt_parameters.mpt_base_adr_l, icm_offset );
2335  MLX_FILL_1 ( init_hca, 62,
2336  tpt_parameters.log_mpt_sz, log_num_mpts );
2337  DBGC ( arbel, "Arbel %p ICM MPT is %d x %#zx at [%zx,%zx)\n",
2338  arbel, ( 1 << log_num_mpts ), arbel->limits.mpt_entry_size,
2339  icm_offset, ( icm_offset + len ) );
2340  icm_offset += len;
2341 
2342  /* Remote read data base table */
2343  len = ( ( 1 << log_num_rdbs ) * ARBEL_RDB_ENTRY_SIZE );
2344  icm_offset = icm_align ( icm_offset, len );
2345  MLX_FILL_1 ( init_hca, 37,
2346  qpc_eec_cqc_eqc_rdb_parameters.rdb_base_addr_l,
2347  icm_offset );
2348  DBGC ( arbel, "Arbel %p ICM RDB is %d x %#zx at [%zx,%zx)\n",
2349  arbel, ( 1 << log_num_rdbs ), ARBEL_RDB_ENTRY_SIZE,
2350  icm_offset, ( icm_offset + len ) );
2351  icm_offset += len;
2352 
2353  /* Extended end-to-end contexts */
2354  len = ( ( 1 << log_num_ees ) * arbel->limits.eeec_entry_size );
2355  icm_offset = icm_align ( icm_offset, len );
2356  MLX_FILL_1 ( init_hca, 29,
2357  qpc_eec_cqc_eqc_rdb_parameters.eeec_base_addr_l,
2358  icm_offset );
2359  DBGC ( arbel, "Arbel %p ICM EEEC is %d x %#zx at [%zx,%zx)\n",
2360  arbel, ( 1 << log_num_ees ), arbel->limits.eeec_entry_size,
2361  icm_offset, ( icm_offset + len ) );
2362  icm_offset += len;
2363 
2364  /* Multicast table */
2365  len = ( ( 1 << log_num_mcs ) * sizeof ( struct arbelprm_mgm_entry ) );
2366  icm_offset = icm_align ( icm_offset, len );
2367  MLX_FILL_1 ( init_hca, 49,
2368  multicast_parameters.mc_base_addr_l, icm_offset );
2369  MLX_FILL_1 ( init_hca, 52,
2370  multicast_parameters.log_mc_table_entry_sz,
2371  fls ( sizeof ( struct arbelprm_mgm_entry ) - 1 ) );
2372  MLX_FILL_1 ( init_hca, 53,
2373  multicast_parameters.mc_table_hash_sz,
2374  ( 1 << log_num_mcs ) );
2375  MLX_FILL_1 ( init_hca, 54,
2376  multicast_parameters.log_mc_table_sz,
2377  log_num_mcs /* Only one entry per hash */ );
2378  DBGC ( arbel, "Arbel %p ICM MC is %d x %#zx at [%zx,%zx)\n", arbel,
2379  ( 1 << log_num_mcs ), sizeof ( struct arbelprm_mgm_entry ),
2380  icm_offset, ( icm_offset + len ) );
2381  icm_offset += len;
2382 
2383  /* Memory translation table */
2384  len = ( ( 1 << log_num_mtts ) * arbel->limits.mtt_entry_size );
2385  icm_offset = icm_align ( icm_offset, len );
2386  MLX_FILL_1 ( init_hca, 65,
2387  tpt_parameters.mtt_base_addr_l, icm_offset );
2388  DBGC ( arbel, "Arbel %p ICM MTT is %d x %#zx at [%zx,%zx)\n",
2389  arbel, ( 1 << log_num_mtts ), arbel->limits.mtt_entry_size,
2390  icm_offset, ( icm_offset + len ) );
2391  icm_offset += len;
2392 
2393  /* User access region scratchpads */
2394  len = ( ( 1 << log_num_uars ) * arbel->limits.uar_scratch_entry_size );
2395  icm_offset = icm_align ( icm_offset, len );
2396  MLX_FILL_1 ( init_hca, 77,
2397  uar_parameters.uar_scratch_base_addr_l, icm_offset );
2398  DBGC ( arbel, "Arbel %p UAR scratchpad is %d x %#zx at [%zx,%zx)\n",
2399  arbel, ( 1 << log_num_uars ),
2401  icm_offset, ( icm_offset + len ) );
2402  icm_offset += len;
2403 
2404  /* Record amount of ICM to be allocated */
2405  icm_offset = icm_align ( icm_offset, ARBEL_PAGE_SIZE );
2406  icm_len = icm_offset;
2407 
2408  /* User access region contexts
2409  *
2410  * The reserved UAR(s) do not need to be backed by physical
2411  * memory, and our UAR is allocated separately; neither are
2412  * part of the umalloc()ed ICM block, but both contribute to
2413  * the total length of ICM virtual address space.
2414  */
2415  len = ( ( 1 << log_num_uars ) * ARBEL_PAGE_SIZE );
2416  icm_offset = icm_align ( icm_offset, len );
2417  MLX_FILL_1 ( init_hca, 74, uar_parameters.log_max_uars, log_num_uars );
2418  MLX_FILL_1 ( init_hca, 79,
2419  uar_parameters.uar_context_base_addr_l, icm_offset );
2420  arbel->db_rec_offset =
2421  ( icm_offset +
2423  DBGC ( arbel, "Arbel %p UAR is %d x %#zx at [%zx,%zx), doorbells "
2424  "[%zx,%zx)\n", arbel, ( 1 << log_num_uars ), ARBEL_PAGE_SIZE,
2425  icm_offset, ( icm_offset + len ), arbel->db_rec_offset,
2427  icm_offset += len;
2428 
2429  /* Get ICM auxiliary area size */
2430  memset ( &icm_size, 0, sizeof ( icm_size ) );
2431  MLX_FILL_1 ( &icm_size, 1, value, icm_len );
2432  if ( ( rc = arbel_cmd_set_icm_size ( arbel, &icm_size,
2433  &icm_aux_size ) ) != 0 ) {
2434  DBGC ( arbel, "Arbel %p could not set ICM size: %s\n",
2435  arbel, strerror ( rc ) );
2436  goto err_set_icm_size;
2437  }
2438  icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * ARBEL_PAGE_SIZE );
2439 
2440  /* Allocate ICM data and auxiliary area */
2441  DBGC ( arbel, "Arbel %p requires %zd kB ICM and %zd kB AUX ICM\n",
2442  arbel, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) );
2443  if ( ! arbel->icm ) {
2444  arbel->icm_len = icm_len;
2445  arbel->icm_aux_len = icm_aux_len;
2447  if ( ! arbel->icm ) {
2448  rc = -ENOMEM;
2449  goto err_alloc_icm;
2450  }
2451  } else {
2452  assert ( arbel->icm_len == icm_len );
2453  assert ( arbel->icm_aux_len == icm_aux_len );
2454  }
2455  icm_phys = user_to_phys ( arbel->icm, 0 );
2456 
2457  /* Allocate doorbell UAR */
2459  if ( ! arbel->db_rec ) {
2460  rc = -ENOMEM;
2461  goto err_alloc_doorbell;
2462  }
2463 
2464  /* Map ICM auxiliary area */
2465  DBGC ( arbel, "Arbel %p ICM AUX at [%08lx,%08lx)\n",
2466  arbel, icm_phys, ( icm_phys + arbel->icm_aux_len ) );
2468  0, icm_phys, arbel->icm_aux_len ) ) != 0 ){
2469  DBGC ( arbel, "Arbel %p could not map AUX ICM: %s\n",
2470  arbel, strerror ( rc ) );
2471  goto err_map_icm_aux;
2472  }
2473  icm_phys += arbel->icm_aux_len;
2474 
2475  /* Map ICM area */
2476  DBGC ( arbel, "Arbel %p ICM at [%08lx,%08lx)\n",
2477  arbel, icm_phys, ( icm_phys + arbel->icm_len ) );
2479  0, icm_phys, arbel->icm_len ) ) != 0 ) {
2480  DBGC ( arbel, "Arbel %p could not map ICM: %s\n",
2481  arbel, strerror ( rc ) );
2482  goto err_map_icm;
2483  }
2484  icm_phys += arbel->icm_len;
2485 
2486  /* Map doorbell UAR */
2487  DBGC ( arbel, "Arbel %p UAR at [%08lx,%08lx)\n",
2492  virt_to_phys ( arbel->db_rec ),
2493  ARBEL_PAGE_SIZE ) ) != 0 ) {
2494  DBGC ( arbel, "Arbel %p could not map doorbell UAR: %s\n",
2495  arbel, strerror ( rc ) );
2496  goto err_map_doorbell;
2497  }
2498 
2499  /* Initialise doorbell records */
2502  MLX_FILL_1 ( &db_rec->qp, 1, res, ARBEL_UAR_RES_GROUP_SEP );
2503 
2504  return 0;
2505 
2506  memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
2507  MLX_FILL_1 ( &unmap_icm, 1, value, arbel->db_rec_offset );
2508  arbel_cmd_unmap_icm ( arbel, 1, &unmap_icm );
2509  err_map_doorbell:
2510  memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
2512  &unmap_icm );
2513  err_map_icm:
2515  err_map_icm_aux:
2517  arbel->db_rec= NULL;
2518  err_alloc_doorbell:
2519  err_alloc_icm:
2520  err_set_icm_size:
2521  return rc;
2522 }
static int arbel_cmd_unmap_icm_aux(struct arbel *arbel)
Definition: arbel.c:487
static size_t icm_align(size_t icm_offset, size_t len)
Align ICM table.
Definition: arbel.c:2210
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
#define ARBEL_GROUP_SEPARATOR_DOORBELL
Definition: arbel.h:601
unsigned int reserved_srqs
Number of reserved SRQs.
Definition: arbel.h:300
unsigned int reserved_mtts
Number of reserved MTTs.
Definition: arbel.h:316
userptr_t icm
ICM area.
Definition: arbel.h:506
size_t eeec_entry_size
Extended EE context entry size.
Definition: arbel.h:308
size_t eqc_entry_size
EQ context entry size.
Definition: arbel.h:326
unsigned long user_to_phys(userptr_t userptr, off_t offset)
Convert user pointer to physical address.
#define DBGC(...)
Definition: compiler.h:505
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
unsigned int reserved_eqs
Number of reserved EQs.
Definition: arbel.h:314
static int arbel_cmd_unmap_icm(struct arbel *arbel, unsigned int page_count, const struct arbelprm_scalar_parameter *offset)
Definition: arbel.c:469
struct arbelprm_qp_db_record qp
Definition: arbel.h:268
size_t cqc_entry_size
CQ context entry size.
Definition: arbel.h:312
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
size_t eqpc_entry_size
Extended QP context entry size.
Definition: arbel.h:298
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
#define ARBEL_MAX_EQS
Maximum number of allocatable event queues.
Definition: arbel.h:437
static int arbel_cmd_map_icm_aux(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *map)
Definition: arbel.c:494
unsigned int reserved_rdbs
Number of reserved RDBs.
Definition: arbel.h:324
#define ARBEL_LOG_MULTICAST_HASH_SIZE
Definition: arbel.h:114
size_t srqc_entry_size
SRQ context entry size.
Definition: arbel.h:302
static int arbel_cmd_set_icm_size(struct arbel *arbel, const struct arbelprm_scalar_parameter *icm_size, struct arbelprm_scalar_parameter *icm_aux_size)
Definition: arbel.c:503
static int arbel_cmd_map_icm(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *map)
Definition: arbel.c:478
#define ENOMEM
Not enough space.
Definition: errno.h:534
#define ARBEL_RDB_ENTRY_SIZE
Definition: arbel.h:102
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
size_t db_rec_offset
Offset within ICM of doorbell records.
Definition: arbel.h:508
#define ARBEL_RSVD_SPECIAL_QPS
Number of queue pairs reserved for the "special QP" block.
Definition: arbel.h:386
pseudo_bit_t value[0x00020]
Definition: arbel.h:13
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
size_t mtt_entry_size
MTT entry size.
Definition: arbel.h:318
#define ARBEL_MAX_CQS
Maximum number of allocatable completion queues.
Definition: arbel.h:419
unsigned int reserved_cqs
Number of reserved CQs.
Definition: arbel.h:310
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
size_t eec_entry_size
EE context entry size.
Definition: arbel.h:306
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
unsigned int reserved_mrws
Number of reserved MRWs.
Definition: arbel.h:320
unsigned int reserved_ees
Number of reserved EEs.
Definition: arbel.h:304
size_t icm_aux_len
ICM AUX size.
Definition: arbel.h:499
unsigned long physaddr_t
Definition: stdint.h:20
unsigned int reserved_qps
Number of reserved QPs.
Definition: arbel.h:294
size_t qpc_entry_size
QP context entry size.
Definition: arbel.h:296
uint32_t len
Length.
Definition: ena.h:14
static __always_inline userptr_t umalloc(size_t size)
Allocate external memory.
Definition: umalloc.h:54
#define ARBEL_PAGE_SIZE
Definition: arbel.h:100
size_t uar_scratch_entry_size
UAR scratchpad entry size.
Definition: arbel.h:330
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
size_t icm_len
ICM size.
Definition: arbel.h:497
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
unsigned int reserved_uars
Number of reserved UARs.
Definition: arbel.h:328
static int arbel_map_vpm(struct arbel *arbel, int(*map)(struct arbel *arbel, const struct arbelprm_virtual_physical_mapping *), uint64_t va, physaddr_t pa, size_t len)
Map virtual to physical address for firmware usage.
Definition: arbel.c:1959
#define fls(x)
Find last (i.e.
Definition: strings.h:166
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
size_t mpt_entry_size
MPT entry size.
Definition: arbel.h:322
#define ARBEL_MAX_QPS
Maximum number of allocatable queue pairs.
Definition: arbel.h:392
void * memset(void *dest, int character, size_t len) __nonnull
#define ARBEL_UAR_RES_GROUP_SEP
Definition: arbel.h:45

References arbel_cmd_map_icm(), arbel_cmd_map_icm_aux(), arbel_cmd_set_icm_size(), arbel_cmd_unmap_icm(), arbel_cmd_unmap_icm_aux(), ARBEL_GROUP_SEPARATOR_DOORBELL, ARBEL_LOG_MULTICAST_HASH_SIZE, arbel_map_vpm(), ARBEL_MAX_CQS, ARBEL_MAX_EQS, ARBEL_MAX_QPS, ARBEL_PAGE_SIZE, ARBEL_RDB_ENTRY_SIZE, ARBEL_RSVD_SPECIAL_QPS, ARBEL_UAR_RES_GROUP_SEP, assert(), arbel_dev_limits::cqc_entry_size, arbel::db_rec, arbel::db_rec_offset, DBGC, arbel_dev_limits::eec_entry_size, arbel_dev_limits::eeec_entry_size, ENOMEM, arbel_dev_limits::eqc_entry_size, arbel_dev_limits::eqpc_entry_size, fls, free_phys(), arbel::icm, icm_align(), arbel::icm_aux_len, arbel::icm_len, len, arbel::limits, malloc_phys(), memset(), MLX_FILL_1, MLX_FILL_2, MLX_GET, arbel_dev_limits::mpt_entry_size, arbel_dev_limits::mtt_entry_size, NULL, arbelprm_doorbell_record::qp, arbel_dev_limits::qpc_entry_size, rc, arbel_dev_limits::reserved_cqs, arbel_dev_limits::reserved_ees, arbel_dev_limits::reserved_eqs, arbel_dev_limits::reserved_mrws, arbel_dev_limits::reserved_mtts, arbel_dev_limits::reserved_qps, arbel_dev_limits::reserved_rdbs, arbel_dev_limits::reserved_srqs, arbel_dev_limits::reserved_uars, arbel_dev_limits::srqc_entry_size, strerror(), arbel_dev_limits::uar_scratch_entry_size, umalloc(), user_to_phys(), value, and virt_to_phys().

Referenced by arbel_start().

◆ arbel_free_icm()

static void arbel_free_icm ( struct arbel arbel)
static

Free ICM.

Parameters
arbelArbel device

Definition at line 2529 of file arbel.c.

2529  {
2530  struct arbelprm_scalar_parameter unmap_icm;
2531 
2532  memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
2533  MLX_FILL_1 ( &unmap_icm, 1, value, arbel->db_rec_offset );
2534  arbel_cmd_unmap_icm ( arbel, 1, &unmap_icm );
2535  memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
2537  &unmap_icm );
2540  arbel->db_rec = NULL;
2541 }
static int arbel_cmd_unmap_icm_aux(struct arbel *arbel)
Definition: arbel.c:487
static int arbel_cmd_unmap_icm(struct arbel *arbel, unsigned int page_count, const struct arbelprm_scalar_parameter *offset)
Definition: arbel.c:469
union arbelprm_doorbell_record * db_rec
Doorbell records.
Definition: arbel.h:510
size_t db_rec_offset
Offset within ICM of doorbell records.
Definition: arbel.h:508
pseudo_bit_t value[0x00020]
Definition: arbel.h:13
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
#define ARBEL_PAGE_SIZE
Definition: arbel.h:100
size_t icm_len
ICM size.
Definition: arbel.h:497
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_unmap_icm(), arbel_cmd_unmap_icm_aux(), ARBEL_PAGE_SIZE, arbel::db_rec, arbel::db_rec_offset, free_phys(), arbel::icm_len, memset(), MLX_FILL_1, NULL, and value.

Referenced by arbel_start(), and arbel_stop().

◆ arbel_reset()

static void arbel_reset ( struct arbel arbel)
static

Reset device.

Parameters
arbelArbel device

Definition at line 2555 of file arbel.c.

2555  {
2556  struct pci_device *pci = arbel->pci;
2557  struct pci_config_backup backup;
2558  static const uint8_t backup_exclude[] =
2559  PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c );
2560  uint16_t vendor;
2561  unsigned int i;
2562 
2563  /* Perform device reset and preserve PCI configuration */
2564  pci_backup ( pci, &backup, PCI_CONFIG_BACKUP_ALL, backup_exclude );
2566  ( arbel->config + ARBEL_RESET_OFFSET ) );
2567  for ( i = 0 ; i < ARBEL_RESET_WAIT_TIME_MS ; i++ ) {
2568  mdelay ( 1 );
2570  if ( vendor != 0xffff )
2571  break;
2572  }
2573  pci_restore ( pci, &backup, PCI_CONFIG_BACKUP_ALL, backup_exclude );
2574 }
void pci_restore(struct pci_device *pci, struct pci_config_backup *backup, unsigned int limit, const uint8_t *exclude)
Restore PCI configuration space.
Definition: pcibackup.c:87
unsigned short uint16_t
Definition: stdint.h:11
static unsigned short vendor
Definition: davicom.c:128
#define ARBEL_RESET_MAGIC
Definition: arbel.h:36
void * config
PCI configuration registers.
Definition: arbel.h:473
int pci_read_config_word(struct pci_device *pci, unsigned int where, uint16_t *value)
Read 16-bit word from PCI configuration space.
#define PCI_CONFIG_BACKUP_ALL
Limit of PCI configuration space.
Definition: pcibackup.h:15
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
A PCI device.
Definition: pci.h:206
A PCI configuration space backup.
Definition: pcibackup.h:21
#define PCI_VENDOR_ID
PCI vendor ID.
Definition: pci.h:19
unsigned char uint8_t
Definition: stdint.h:10
#define ARBEL_RESET_OFFSET
Definition: arbel.h:35
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
void pci_backup(struct pci_device *pci, struct pci_config_backup *backup, unsigned int limit, const uint8_t *exclude)
Back up PCI configuration space.
Definition: pcibackup.c:67
#define PCI_CONFIG_BACKUP_EXCLUDE(...)
Define a PCI configuration space backup exclusion list.
Definition: pcibackup.h:29
#define ARBEL_RESET_WAIT_TIME_MS
Definition: arbel.h:37
struct pci_device * pci
PCI device.
Definition: arbel.h:471
An Arbel device.
Definition: arbel.h:469

References ARBEL_RESET_MAGIC, ARBEL_RESET_OFFSET, ARBEL_RESET_WAIT_TIME_MS, arbel::config, mdelay(), arbel::pci, pci_backup(), PCI_CONFIG_BACKUP_ALL, PCI_CONFIG_BACKUP_EXCLUDE, pci_read_config_word(), pci_restore(), PCI_VENDOR_ID, vendor, and writel().

Referenced by arbel_probe(), and arbel_stop().

◆ arbel_setup_mpt()

static int arbel_setup_mpt ( struct arbel arbel)
static

Set up memory protection table.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 2582 of file arbel.c.

2582  {
2583  struct arbelprm_mpt mpt;
2584  uint32_t key;
2585  int rc;
2586 
2587  /* Derive key */
2589  arbel->lkey = ( ( key << 8 ) | ( key >> 24 ) );
2590 
2591  /* Initialise memory protection table */
2592  memset ( &mpt, 0, sizeof ( mpt ) );
2593  MLX_FILL_7 ( &mpt, 0,
2594  a, 1,
2595  rw, 1,
2596  rr, 1,
2597  lw, 1,
2598  lr, 1,
2599  pa, 1,
2600  r_w, 1 );
2601  MLX_FILL_1 ( &mpt, 2, mem_key, key );
2602  MLX_FILL_2 ( &mpt, 3,
2603  pd, ARBEL_GLOBAL_PD,
2604  rae, 1 );
2605  MLX_FILL_1 ( &mpt, 6, reg_wnd_len_h, 0xffffffffUL );
2606  MLX_FILL_1 ( &mpt, 7, reg_wnd_len_l, 0xffffffffUL );
2608  &mpt ) ) != 0 ) {
2609  DBGC ( arbel, "Arbel %p could not set up MPT: %s\n",
2610  arbel, strerror ( rc ) );
2611  return rc;
2612  }
2613 
2614  return 0;
2615 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define MLX_FILL_7(_ptr, _index,...)
Definition: mlx_bitops.h:191
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
static int arbel_cmd_sw2hw_mpt(struct arbel *arbel, unsigned int index, const struct arbelprm_mpt *mpt)
Definition: arbel.c:286
#define DBGC(...)
Definition: compiler.h:505
#define ARBEL_GLOBAL_PD
Global protection domain.
Definition: arbel.h:536
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
uint32_t a
Definition: md4.c:28
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
unsigned int uint32_t
Definition: stdint.h:12
unsigned int reserved_mrws
Number of reserved MRWs.
Definition: arbel.h:320
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
#define ARBEL_MKEY_PREFIX
Memory key prefix.
Definition: arbel.h:539
An Arbel device.
Definition: arbel.h:469
union @382 key
Sense key.
Definition: crypto.h:284
void * memset(void *dest, int character, size_t len) __nonnull

References a, arbel_cmd_sw2hw_mpt(), ARBEL_GLOBAL_PD, ARBEL_MKEY_PREFIX, DBGC, key, arbel::limits, arbel::lkey, memset(), MLX_FILL_1, MLX_FILL_2, MLX_FILL_7, rc, arbel_dev_limits::reserved_mrws, and strerror().

Referenced by arbel_start().

◆ arbel_configure_special_qps()

static int arbel_configure_special_qps ( struct arbel arbel)
static

Configure special queue pairs.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 2623 of file arbel.c.

2623  {
2624  unsigned int smi_qpn_base;
2625  unsigned int gsi_qpn_base;
2626  int rc;
2627 
2628  /* Special QP block must be aligned on an even number */
2629  arbel->special_qpn_base = ( ( arbel->limits.reserved_qps + 1 ) & ~1 );
2632  DBGC ( arbel, "Arbel %p special QPs at [%lx,%lx]\n", arbel,
2633  arbel->special_qpn_base, ( arbel->qpn_base - 1 ) );
2634  smi_qpn_base = arbel->special_qpn_base;
2635  gsi_qpn_base = ( smi_qpn_base + 2 );
2636 
2637  /* Issue commands to configure special QPs */
2638  if ( ( rc = arbel_cmd_conf_special_qp ( arbel, 0,
2639  smi_qpn_base ) ) != 0 ) {
2640  DBGC ( arbel, "Arbel %p could not configure SMI QPs: %s\n",
2641  arbel, strerror ( rc ) );
2642  return rc;
2643  }
2644  if ( ( rc = arbel_cmd_conf_special_qp ( arbel, 1,
2645  gsi_qpn_base ) ) != 0 ) {
2646  DBGC ( arbel, "Arbel %p could not configure GSI QPs: %s\n",
2647  arbel, strerror ( rc ) );
2648  return rc;
2649  }
2650 
2651  return 0;
2652 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define DBGC(...)
Definition: compiler.h:505
#define ARBEL_NUM_SPECIAL_QPS
Number of special queue pairs.
Definition: arbel.h:379
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
unsigned long qpn_base
QPN base.
Definition: arbel.h:529
static int arbel_cmd_conf_special_qp(struct arbel *arbel, unsigned int qp_type, unsigned long base_qpn)
Definition: arbel.c:401
unsigned int reserved_qps
Number of reserved QPs.
Definition: arbel.h:294
struct arbel_dev_limits limits
Device limits.
Definition: arbel.h:525
unsigned long special_qpn_base
Special QPN base.
Definition: arbel.h:527
An Arbel device.
Definition: arbel.h:469

References arbel_cmd_conf_special_qp(), ARBEL_NUM_SPECIAL_QPS, DBGC, arbel::limits, arbel::qpn_base, rc, arbel_dev_limits::reserved_qps, arbel::special_qpn_base, and strerror().

Referenced by arbel_start().

◆ arbel_start()

static int arbel_start ( struct arbel arbel,
int  running 
)
static

Start Arbel device.

Parameters
arbelArbel device
runningFirmware is already running
Return values
rcReturn status code

Definition at line 2661 of file arbel.c.

2661  {
2662  struct arbelprm_init_hca init_hca;
2663  unsigned int i;
2664  int rc;
2665 
2666  /* Start firmware if not already running */
2667  if ( ! running ) {
2668  if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
2669  goto err_start_firmware;
2670  }
2671 
2672  /* Allocate ICM */
2673  memset ( &init_hca, 0, sizeof ( init_hca ) );
2674  if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
2675  goto err_alloc_icm;
2676 
2677  /* Initialise HCA */
2678  if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
2679  DBGC ( arbel, "Arbel %p could not initialise HCA: %s\n",
2680  arbel, strerror ( rc ) );
2681  goto err_init_hca;
2682  }
2683 
2684  /* Set up memory protection */
2685  if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
2686  goto err_setup_mpt;
2687  for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
2688  arbel->ibdev[i]->rdma_key = arbel->lkey;
2689 
2690  /* Set up event queue */
2691  if ( ( rc = arbel_create_eq ( arbel ) ) != 0 )
2692  goto err_create_eq;
2693 
2694  /* Configure special QPs */
2695  if ( ( rc = arbel_configure_special_qps ( arbel ) ) != 0 )
2696  goto err_conf_special_qps;
2697 
2698  return 0;
2699 
2700  err_conf_special_qps:
2701  arbel_destroy_eq ( arbel );
2702  err_create_eq:
2703  err_setup_mpt:
2705  err_init_hca:
2706  arbel_free_icm ( arbel );
2707  err_alloc_icm:
2709  err_start_firmware:
2710  return rc;
2711 }
static void arbel_free_icm(struct arbel *arbel)
Free ICM.
Definition: arbel.c:2529
static void arbel_destroy_eq(struct arbel *arbel)
Destroy event queue.
Definition: arbel.c:1820
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_alloc_icm(struct arbel *arbel, struct arbelprm_init_hca *init_hca)
Allocate ICM.
Definition: arbel.c:2224
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
unsigned long lkey
Unrestricted LKey.
Definition: arbel.h:517
static int arbel_start_firmware(struct arbel *arbel)
Start firmware running.
Definition: arbel.c:2038
static int arbel_create_eq(struct arbel *arbel)
Create event queue.
Definition: arbel.c:1744
static int arbel_cmd_init_hca(struct arbel *arbel, const struct arbelprm_init_hca *init_hca)
Definition: arbel.c:254
uint32_t rdma_key
RDMA key.
Definition: infiniband.h:456
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static int arbel_setup_mpt(struct arbel *arbel)
Set up memory protection table.
Definition: arbel.c:2582
#define ARBEL_NUM_PORTS
Definition: arbel.h:24
static void arbel_stop_firmware(struct arbel *arbel)
Stop firmware running.
Definition: arbel.c:2115
static int arbel_configure_special_qps(struct arbel *arbel)
Configure special queue pairs.
Definition: arbel.c:2623
An Arbel device.
Definition: arbel.h:469
static int arbel_cmd_close_hca(struct arbel *arbel)
Definition: arbel.c:263
if(natsemi->flags &NATSEMI_64BIT) return 1
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_alloc_icm(), arbel_cmd_close_hca(), arbel_cmd_init_hca(), arbel_configure_special_qps(), arbel_create_eq(), arbel_destroy_eq(), arbel_free_icm(), ARBEL_NUM_PORTS, arbel_setup_mpt(), arbel_start_firmware(), arbel_stop_firmware(), DBGC, arbel::ibdev, if(), arbel::lkey, memset(), rc, ib_device::rdma_key, and strerror().

Referenced by arbel_open(), and arbel_probe().

◆ arbel_stop()

static void arbel_stop ( struct arbel arbel)
static

Stop Arbel device.

Parameters
arbelArbel device

Definition at line 2718 of file arbel.c.

2718  {
2719  arbel_destroy_eq ( arbel );
2721  arbel_free_icm ( arbel );
2723  arbel_reset ( arbel );
2724 }
static void arbel_free_icm(struct arbel *arbel)
Free ICM.
Definition: arbel.c:2529
static void arbel_destroy_eq(struct arbel *arbel)
Destroy event queue.
Definition: arbel.c:1820
static void arbel_reset(struct arbel *arbel)
Reset device.
Definition: arbel.c:2555
static void arbel_stop_firmware(struct arbel *arbel)
Stop firmware running.
Definition: arbel.c:2115
An Arbel device.
Definition: arbel.h:469
static int arbel_cmd_close_hca(struct arbel *arbel)
Definition: arbel.c:263

References arbel_cmd_close_hca(), arbel_destroy_eq(), arbel_free_icm(), arbel_reset(), and arbel_stop_firmware().

Referenced by arbel_close(), and arbel_probe().

◆ arbel_open()

static int arbel_open ( struct arbel arbel)
static

Open Arbel device.

Parameters
arbelArbel device
Return values
rcReturn status code

Definition at line 2732 of file arbel.c.

2732  {
2733  int rc;
2734 
2735  /* Start device if applicable */
2736  if ( arbel->open_count == 0 ) {
2737  if ( ( rc = arbel_start ( arbel, 0 ) ) != 0 )
2738  return rc;
2739  }
2740 
2741  /* Increment open counter */
2742  arbel->open_count++;
2743 
2744  return 0;
2745 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_start(struct arbel *arbel, int running)
Start Arbel device.
Definition: arbel.c:2661
unsigned int open_count
Device open request counter.
Definition: arbel.h:485
An Arbel device.
Definition: arbel.h:469

References arbel_start(), arbel::open_count, and rc.

Referenced by arbel_ib_open().

◆ arbel_close()

static void arbel_close ( struct arbel arbel)
static

Close Arbel device.

Parameters
arbelArbel device

Definition at line 2752 of file arbel.c.

2752  {
2753 
2754  /* Decrement open counter */
2755  assert ( arbel->open_count != 0 );
2756  arbel->open_count--;
2757 
2758  /* Stop device if applicable */
2759  if ( arbel->open_count == 0 )
2760  arbel_stop ( arbel );
2761 }
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
static void arbel_stop(struct arbel *arbel)
Stop Arbel device.
Definition: arbel.c:2718
unsigned int open_count
Device open request counter.
Definition: arbel.h:485
An Arbel device.
Definition: arbel.h:469

References arbel_stop(), assert(), and arbel::open_count.

Referenced by arbel_ib_close(), and arbel_ib_open().

◆ arbel_ib_open()

static int arbel_ib_open ( struct ib_device ibdev)
static

Initialise Infiniband link.

Parameters
ibdevInfiniband device
Return values
rcReturn status code

Definition at line 2776 of file arbel.c.

2776  {
2777  struct arbel *arbel = ib_get_drvdata ( ibdev );
2778  struct arbelprm_init_ib init_ib;
2779  int rc;
2780 
2781  /* Open hardware */
2782  if ( ( rc = arbel_open ( arbel ) ) != 0 )
2783  goto err_open;
2784 
2785  /* Initialise IB */
2786  memset ( &init_ib, 0, sizeof ( init_ib ) );
2787  MLX_FILL_3 ( &init_ib, 0,
2788  mtu_cap, ARBEL_MTU_2048,
2789  port_width_cap, 3,
2790  vl_cap, 1 );
2791  MLX_FILL_1 ( &init_ib, 1, max_gid, 1 );
2792  MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
2793  if ( ( rc = arbel_cmd_init_ib ( arbel, ibdev->port,
2794  &init_ib ) ) != 0 ) {
2795  DBGC ( arbel, "Arbel %p port %d could not intialise IB: %s\n",
2796  arbel, ibdev->port, strerror ( rc ) );
2797  goto err_init_ib;
2798  }
2799 
2800  /* Update MAD parameters */
2801  ib_smc_update ( ibdev, arbel_mad );
2802 
2803  return 0;
2804 
2805  err_init_ib:
2806  arbel_close ( arbel );
2807  err_open:
2808  return rc;
2809 }
static int arbel_mad(struct ib_device *ibdev, union ib_mad *mad)
Issue management datagram.
Definition: arbel.c:543
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
static void arbel_close(struct arbel *arbel)
Close Arbel device.
Definition: arbel.c:2752
#define MLX_FILL_3(_ptr, _index,...)
Definition: mlx_bitops.h:175
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static int arbel_open(struct arbel *arbel)
Open Arbel device.
Definition: arbel.c:2732
#define ARBEL_MTU_2048
Definition: arbel.h:94
unsigned int port
Port number.
Definition: infiniband.h:418
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
int ib_smc_update(struct ib_device *ibdev, ib_local_mad_t local_mad)
Update Infiniband parameters using SMC.
Definition: ib_smc.c:249
#define MLX_FILL_1(_ptr, _index,...)
Definition: mlx_bitops.h:167
static int arbel_cmd_init_ib(struct arbel *arbel, unsigned int port, const struct arbelprm_init_ib *init_ib)
Definition: arbel.c:270
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_close(), arbel_cmd_init_ib(), arbel_mad(), ARBEL_MTU_2048, arbel_open(), DBGC, ib_get_drvdata(), ib_smc_update(), arbel::ibdev, memset(), MLX_FILL_1, MLX_FILL_3, ib_device::port, rc, and strerror().

◆ arbel_ib_close()

static void arbel_ib_close ( struct ib_device ibdev)
static

Close Infiniband link.

Parameters
ibdevInfiniband device

Definition at line 2816 of file arbel.c.

2816  {
2817  struct arbel *arbel = ib_get_drvdata ( ibdev );
2818  int rc;
2819 
2820  /* Close IB */
2821  if ( ( rc = arbel_cmd_close_ib ( arbel, ibdev->port ) ) != 0 ) {
2822  DBGC ( arbel, "Arbel %p port %d could not close IB: %s\n",
2823  arbel, ibdev->port, strerror ( rc ) );
2824  /* Nothing we can do about this */
2825  }
2826 
2827  /* Close hardware */
2828  arbel_close ( arbel );
2829 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
static void arbel_close(struct arbel *arbel)
Close Arbel device.
Definition: arbel.c:2752
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
unsigned int port
Port number.
Definition: infiniband.h:418
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
An Arbel device.
Definition: arbel.h:469
static int arbel_cmd_close_ib(struct arbel *arbel, unsigned int port)
Definition: arbel.c:279

References arbel_close(), arbel_cmd_close_ib(), DBGC, ib_get_drvdata(), arbel::ibdev, ib_device::port, rc, and strerror().

◆ arbel_inform_sma()

static int arbel_inform_sma ( struct ib_device ibdev,
union ib_mad mad 
)
static

Inform embedded subnet management agent of a received MAD.

Parameters
ibdevInfiniband device
madMAD
Return values
rcReturn status code

Definition at line 2838 of file arbel.c.

2838  {
2839  int rc;
2840 
2841  /* Send the MAD to the embedded SMA */
2842  if ( ( rc = arbel_mad ( ibdev, mad ) ) != 0 )
2843  return rc;
2844 
2845  /* Update parameters held in software */
2847 
2848  return 0;
2849 }
static int arbel_mad(struct ib_device *ibdev, union ib_mad *mad)
Issue management datagram.
Definition: arbel.c:543
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
int ib_smc_update(struct ib_device *ibdev, ib_local_mad_t local_mad)
Update Infiniband parameters using SMC.
Definition: ib_smc.c:249
union ib_mad mad
Definition: arbel.h:12

References arbel_mad(), ib_smc_update(), arbel::ibdev, mad, and rc.

◆ arbel_mcast_attach()

static int arbel_mcast_attach ( struct ib_device ibdev,
struct ib_queue_pair qp,
union ib_gid gid 
)
static

Attach to multicast group.

Parameters
ibdevInfiniband device
qpQueue pair
gidMulticast GID
Return values
rcReturn status code

Definition at line 2866 of file arbel.c.

2868  {
2869  struct arbel *arbel = ib_get_drvdata ( ibdev );
2870  struct arbelprm_mgm_hash hash;
2871  struct arbelprm_mgm_entry mgm;
2872  unsigned int index;
2873  int rc;
2874 
2875  /* Generate hash table index */
2876  if ( ( rc = arbel_cmd_mgid_hash ( arbel, gid, &hash ) ) != 0 ) {
2877  DBGC ( arbel, "Arbel %p could not hash GID: %s\n",
2878  arbel, strerror ( rc ) );
2879  return rc;
2880  }
2881  index = MLX_GET ( &hash, hash );
2882 
2883  /* Check for existing hash table entry */
2884  if ( ( rc = arbel_cmd_read_mgm ( arbel, index, &mgm ) ) != 0 ) {
2885  DBGC ( arbel, "Arbel %p could not read MGM %#x: %s\n",
2886  arbel, index, strerror ( rc ) );
2887  return rc;
2888  }
2889  if ( MLX_GET ( &mgm, mgmqp_0.qi ) != 0 ) {
2890  /* FIXME: this implementation allows only a single QP
2891  * per multicast group, and doesn't handle hash
2892  * collisions. Sufficient for IPoIB but may need to
2893  * be extended in future.
2894  */
2895  DBGC ( arbel, "Arbel %p MGID index %#x already in use\n",
2896  arbel, index );
2897  return -EBUSY;
2898  }
2899 
2900  /* Update hash table entry */
2901  MLX_FILL_2 ( &mgm, 8,
2902  mgmqp_0.qpn_i, qp->qpn,
2903  mgmqp_0.qi, 1 );
2904  memcpy ( &mgm.u.dwords[4], gid, sizeof ( *gid ) );
2905  if ( ( rc = arbel_cmd_write_mgm ( arbel, index, &mgm ) ) != 0 ) {
2906  DBGC ( arbel, "Arbel %p could not write MGM %#x: %s\n",
2907  arbel, index, strerror ( rc ) );
2908  return rc;
2909  }
2910 
2911  return 0;
2912 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define MLX_FILL_2(_ptr, _index,...)
Definition: mlx_bitops.h:171
#define EBUSY
Device or resource busy.
Definition: errno.h:338
static int arbel_cmd_write_mgm(struct arbel *arbel, unsigned int index, const struct arbelprm_mgm_entry *mgm)
Definition: arbel.c:428
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static int arbel_cmd_mgid_hash(struct arbel *arbel, const union ib_gid *gid, struct arbelprm_mgm_hash *hash)
Definition: arbel.c:437
pseudo_bit_t hash[0x00010]
Hash algorithm.
Definition: arbel.h:13
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
static int arbel_cmd_read_mgm(struct arbel *arbel, unsigned int index, struct arbelprm_mgm_entry *mgm)
Definition: arbel.c:419
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
u8 gid[16]
Definition: CIB_PRM.h:31
An Arbel device.
Definition: arbel.h:469

References arbel_cmd_mgid_hash(), arbel_cmd_read_mgm(), arbel_cmd_write_mgm(), DBGC, EBUSY, gid, hash, ib_get_drvdata(), arbel::ibdev, index, memcpy(), MLX_FILL_2, MLX_GET, qp, rc, and strerror().

◆ arbel_mcast_detach()

static void arbel_mcast_detach ( struct ib_device ibdev,
struct ib_queue_pair *qp  __unused,
union ib_gid gid 
)
static

Detach from multicast group.

Parameters
ibdevInfiniband device
qpQueue pair
gidMulticast GID

Definition at line 2921 of file arbel.c.

2923  {
2924  struct arbel *arbel = ib_get_drvdata ( ibdev );
2925  struct arbelprm_mgm_hash hash;
2926  struct arbelprm_mgm_entry mgm;
2927  unsigned int index;
2928  int rc;
2929 
2930  /* Generate hash table index */
2931  if ( ( rc = arbel_cmd_mgid_hash ( arbel, gid, &hash ) ) != 0 ) {
2932  DBGC ( arbel, "Arbel %p could not hash GID: %s\n",
2933  arbel, strerror ( rc ) );
2934  return;
2935  }
2936  index = MLX_GET ( &hash, hash );
2937 
2938  /* Clear hash table entry */
2939  memset ( &mgm, 0, sizeof ( mgm ) );
2940  if ( ( rc = arbel_cmd_write_mgm ( arbel, index, &mgm ) ) != 0 ) {
2941  DBGC ( arbel, "Arbel %p could not write MGM %#x: %s\n",
2942  arbel, index, strerror ( rc ) );
2943  return;
2944  }
2945 }
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static int arbel_cmd_write_mgm(struct arbel *arbel, unsigned int index, const struct arbelprm_mgm_entry *mgm)
Definition: arbel.c:428
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
static __always_inline void * ib_get_drvdata(struct ib_device *ibdev)
Get Infiniband device driver-private data.
Definition: infiniband.h:708
static int arbel_cmd_mgid_hash(struct arbel *arbel, const union ib_gid *gid, struct arbelprm_mgm_hash *hash)
Definition: arbel.c:437
pseudo_bit_t hash[0x00010]
Hash algorithm.
Definition: arbel.h:13
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define MLX_GET(_ptr, _field)
Definition: mlx_bitops.h:222
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
u8 gid[16]
Definition: CIB_PRM.h:31
An Arbel device.
Definition: arbel.h:469
void * memset(void *dest, int character, size_t len) __nonnull

References arbel_cmd_mgid_hash(), arbel_cmd_write_mgm(), DBGC, gid, hash, ib_get_drvdata(), arbel::ibdev, index, memset(), MLX_GET, rc, and strerror().

◆ arbel_alloc()

static struct arbel* arbel_alloc ( void  )
static

Allocate Arbel device.

Return values
arbelArbel device

Definition at line 2978 of file arbel.c.

2978  {
2979  struct arbel *arbel;
2980 
2981  /* Allocate Arbel device */
2982  arbel = zalloc ( sizeof ( *arbel ) );
2983  if ( ! arbel )
2984  goto err_arbel;
2985 
2986  /* Allocate space for mailboxes */
2988  if ( ! arbel->mailbox_in )
2989  goto err_mailbox_in;
2991  if ( ! arbel->mailbox_out )
2992  goto err_mailbox_out;
2993 
2994  return arbel;
2995 
2997  err_mailbox_out:
2999  err_mailbox_in:
3000  free ( arbel );
3001  err_arbel:
3002  return NULL;
3003 }
#define ARBEL_MBOX_SIZE
Definition: arbel.h:550
void * mailbox_out
Command output mailbox.
Definition: arbel.h:482
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:624
#define ARBEL_MBOX_ALIGN
Definition: arbel.h:549
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
An Arbel device.
Definition: arbel.h:469
void * mailbox_in
Command input mailbox.
Definition: arbel.h:480

References ARBEL_MBOX_ALIGN, ARBEL_MBOX_SIZE, free, free_phys(), arbel::mailbox_in, arbel::mailbox_out, malloc_phys(), NULL, and zalloc().

Referenced by arbel_probe().

◆ arbel_free()

static void arbel_free ( struct arbel arbel)
static

Free Arbel device.

Parameters
arbelArbel device

Definition at line 3010 of file arbel.c.

3010  {
3011 
3012  ufree ( arbel->icm );
3013  ufree ( arbel->firmware_area );
3016  free ( arbel );
3017 }
userptr_t icm
ICM area.
Definition: arbel.h:506
#define ARBEL_MBOX_SIZE
Definition: arbel.h:550
void * mailbox_out
Command output mailbox.
Definition: arbel.h:482
userptr_t firmware_area
Firmware area in external memory.
Definition: arbel.h:495
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
static __always_inline void ufree(userptr_t userptr)
Free external memory.
Definition: umalloc.h:65
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
An Arbel device.
Definition: arbel.h:469
void * mailbox_in
Command input mailbox.
Definition: arbel.h:480

References ARBEL_MBOX_SIZE, arbel::firmware_area, free, free_phys(), arbel::icm, arbel::mailbox_in, arbel::mailbox_out, and ufree().

Referenced by arbel_probe(), and arbel_remove().

◆ arbel_probe()

static int arbel_probe ( struct pci_device pci)
static

Probe PCI device.

Parameters
pciPCI device
idPCI ID
Return values
rcReturn status code

Definition at line 3026 of file arbel.c.

3026  {
3027  struct arbel *arbel;
3028  struct ib_device *ibdev;
3029  unsigned long config;
3030  unsigned long uar;
3031  int i;
3032  int rc;
3033 
3034  /* Allocate Arbel device */
3035  arbel = arbel_alloc();
3036  if ( ! arbel ) {
3037  rc = -ENOMEM;
3038  goto err_alloc;
3039  }
3040  pci_set_drvdata ( pci, arbel );
3041  arbel->pci = pci;
3042 
3043  /* Fix up PCI device */
3044  adjust_pci_device ( pci );
3045 
3046  /* Map PCI BARs */
3047  config = pci_bar_start ( pci, ARBEL_PCI_CONFIG_BAR );
3048  arbel->config = pci_ioremap ( pci, config, ARBEL_PCI_CONFIG_BAR_SIZE );
3049  uar = ( pci_bar_start ( pci, ARBEL_PCI_UAR_BAR ) +
3051  arbel->uar = pci_ioremap ( pci, uar, ARBEL_PCI_UAR_SIZE );
3052 
3053  /* Allocate Infiniband devices */
3054  for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) {
3055  ibdev = alloc_ibdev ( 0 );
3056  if ( ! ibdev ) {
3057  rc = -ENOMEM;
3058  goto err_alloc_ibdev;
3059  }
3060  arbel->ibdev[i] = ibdev;
3061  ibdev->op = &arbel_ib_operations;
3062  ibdev->dev = &pci->dev;
3063  ibdev->port = ( ARBEL_PORT_BASE + i );
3064  ibdev->ports = ARBEL_NUM_PORTS;
3065  ib_set_drvdata ( ibdev, arbel );
3066  }
3067 
3068  /* Reset device */
3069  arbel_reset ( arbel );
3070 
3071  /* Start firmware */
3072  if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
3073  goto err_start_firmware;
3074 
3075  /* Get device limits */
3076  if ( ( rc = arbel_get_limits ( arbel ) ) != 0 )
3077  goto err_get_limits;
3078 
3079  /* Start device */
3080  if ( ( rc = arbel_start ( arbel, 1 ) ) != 0 )
3081  goto err_start;
3082 
3083  /* Initialise parameters using SMC */
3084  for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
3085  ib_smc_init ( arbel->ibdev[i], arbel_mad );
3086 
3087  /* Register Infiniband devices */
3088  for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) {
3089  if ( ( rc = register_ibdev ( arbel->ibdev[i] ) ) != 0 ) {
3090  DBGC ( arbel, "Arbel %p port %d could not register IB "
3091  "device: %s\n", arbel,
3092  arbel->ibdev[i]->port, strerror ( rc ) );
3093  goto err_register_ibdev;
3094  }
3095  }
3096 
3097  /* Leave device quiescent until opened */
3098  if ( arbel->open_count == 0 )
3099  arbel_stop ( arbel );
3100 
3101  return 0;
3102 
3103  i = ARBEL_NUM_PORTS;
3104  err_register_ibdev:
3105  for ( i-- ; i >= 0 ; i-- )
3106  unregister_ibdev ( arbel->ibdev[i] );
3107  arbel_stop ( arbel );
3108  err_start:
3109  err_get_limits:
3111  err_start_firmware:
3112  i = ARBEL_NUM_PORTS;
3113  err_alloc_ibdev:
3114  for ( i-- ; i >= 0 ; i-- )
3115  ibdev_put ( arbel->ibdev[i] );
3116  iounmap ( arbel->uar );
3117  iounmap ( arbel->config );
3118  arbel_free ( arbel );
3119  err_alloc:
3120  return rc;
3121 }
void unregister_ibdev(struct ib_device *ibdev)
Unregister Infiniband device.
Definition: infiniband.c:985
static __always_inline void ib_set_drvdata(struct ib_device *ibdev, void *priv)
Set Infiniband device driver-private data.
Definition: infiniband.h:697
static int arbel_mad(struct ib_device *ibdev, union ib_mad *mad)
Issue management datagram.
Definition: arbel.c:543
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
#define DBGC(...)
Definition: compiler.h:505
struct device * dev
Underlying device.
Definition: infiniband.h:410
void * config
PCI configuration registers.
Definition: arbel.h:473
static int arbel_start_firmware(struct arbel *arbel)
Start firmware running.
Definition: arbel.c:2038
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:154
struct device dev
Generic device.
Definition: pci.h:208
An Infiniband device.
Definition: infiniband.h:398
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:359
#define ENOMEM
Not enough space.
Definition: errno.h:534
int ib_smc_init(struct ib_device *ibdev, ib_local_mad_t local_mad)
Initialise Infiniband parameters using SMC.
Definition: ib_smc.c:232
struct ib_device_operations * op
Infiniband operations.
Definition: infiniband.h:416
#define ARBEL_PCI_UAR_SIZE
Definition: arbel.h:32
unsigned long pci_bar_start(struct pci_device *pci, unsigned int reg)
Find the start of a PCI BAR.
Definition: pci.c:96
static void arbel_stop(struct arbel *arbel)
Stop Arbel device.
Definition: arbel.c:2718
static void arbel_free(struct arbel *arbel)
Free Arbel device.
Definition: arbel.c:3010
unsigned int port
Port number.
Definition: infiniband.h:418
static __always_inline void ibdev_put(struct ib_device *ibdev)
Drop reference to Infiniband device.
Definition: infiniband.h:598
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
#define ARBEL_PCI_UAR_IDX
Definition: arbel.h:31
struct ib_device * alloc_ibdev(size_t priv_size)
Allocate Infiniband device.
Definition: infiniband.c:917
static int arbel_start(struct arbel *arbel, int running)
Start Arbel device.
Definition: arbel.c:2661
static void arbel_reset(struct arbel *arbel)
Reset device.
Definition: arbel.c:2555
void * uar
PCI user Access Region.
Definition: arbel.h:475
static struct arbel * arbel_alloc(void)
Allocate Arbel device.
Definition: arbel.c:2978
#define ARBEL_PCI_UAR_BAR
Definition: arbel.h:30
int register_ibdev(struct ib_device *ibdev)
Register Infiniband device.
Definition: infiniband.c:944
unsigned int ports
Total ports on device.
Definition: infiniband.h:420
#define ARBEL_PORT_BASE
Definition: arbel.h:25
#define ARBEL_NUM_PORTS
Definition: arbel.h:24
#define ARBEL_PCI_CONFIG_BAR_SIZE
Definition: arbel.h:29
#define ARBEL_PCI_CONFIG_BAR
Definition: arbel.h:28
void iounmap(volatile const void *io_addr)
Unmap I/O address.
static int arbel_get_limits(struct arbel *arbel)
Get device limits.
Definition: arbel.c:2140
unsigned int open_count
Device open request counter.
Definition: arbel.h:485
static void arbel_stop_firmware(struct arbel *arbel)
Stop firmware running.
Definition: arbel.c:2115
static struct ib_device_operations arbel_ib_operations
Arbel Infiniband operations.
Definition: arbel.c:2948
struct pci_device * pci
PCI device.
Definition: arbel.h:471
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
An Arbel device.
Definition: arbel.h:469

References adjust_pci_device(), alloc_ibdev(), arbel_alloc(), arbel_free(), arbel_get_limits(), arbel_ib_operations, arbel_mad(), ARBEL_NUM_PORTS, ARBEL_PCI_CONFIG_BAR, ARBEL_PCI_CONFIG_BAR_SIZE, ARBEL_PCI_UAR_BAR, ARBEL_PCI_UAR_IDX, ARBEL_PCI_UAR_SIZE, ARBEL_PORT_BASE, arbel_reset(), arbel_start(), arbel_start_firmware(), arbel_stop(), arbel_stop_firmware(), arbel::config, DBGC, pci_device::dev, ib_device::dev, ENOMEM, ib_set_drvdata(), ib_smc_init(), arbel::ibdev, ibdev_put(), iounmap(), ib_device::op, arbel::open_count, arbel::pci, pci_bar_start(), pci_ioremap(), pci_set_drvdata(), ib_device::port, ib_device::ports, rc, register_ibdev(), strerror(), arbel::uar, and unregister_ibdev().

◆ arbel_remove()

static void arbel_remove ( struct pci_device pci)
static

Remove PCI device.

Parameters
pciPCI device

Definition at line 3128 of file arbel.c.

3128  {
3129  struct arbel *arbel = pci_get_drvdata ( pci );
3130  int i;
3131 
3132  for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
3133  unregister_ibdev ( arbel->ibdev[i] );
3134  for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
3135  ibdev_put ( arbel->ibdev[i] );
3136  iounmap ( arbel->uar );
3137  iounmap ( arbel->config );
3138  arbel_free ( arbel );
3139 }
void unregister_ibdev(struct ib_device *ibdev)
Unregister Infiniband device.
Definition: infiniband.c:985
struct ib_device * ibdev[ARBEL_NUM_PORTS]
Infiniband devices.
Definition: arbel.h:532
void * config
PCI configuration registers.
Definition: arbel.h:473
static void arbel_free(struct arbel *arbel)
Free Arbel device.
Definition: arbel.c:3010
static __always_inline void ibdev_put(struct ib_device *ibdev)
Drop reference to Infiniband device.
Definition: infiniband.h:598
void * uar
PCI user Access Region.
Definition: arbel.h:475
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:369
#define ARBEL_NUM_PORTS
Definition: arbel.h:24
void iounmap(volatile const void *io_addr)
Unmap I/O address.
struct pci_device * pci
PCI device.
Definition: arbel.h:471
An Arbel device.
Definition: arbel.h:469

References arbel_free(), ARBEL_NUM_PORTS, arbel::config, arbel::ibdev, ibdev_put(), iounmap(), arbel::pci, pci_get_drvdata(), arbel::uar, and unregister_ibdev().

Variable Documentation

◆ arbel_qp_st

uint8_t arbel_qp_st[]
static
Initial value:
= {
}
#define ARBEL_ST_MLX
Definition: arbel.h:91
#define ARBEL_ST_RC
Definition: arbel.h:89
#define ARBEL_ST_UD
Definition: arbel.h:90

Queue pair transport service type map.

Definition at line 828 of file arbel.c.

Referenced by arbel_create_qp().

◆ arbel_no_gid

const union ib_gid arbel_no_gid
static
Initial value:
= {
.bytes = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0 },
}

GID used for GID-less send work queue entries.

Definition at line 1274 of file arbel.c.

Referenced by arbel_fill_ud_send_wqe().

◆ arbel_fill_send_wqe

size_t( * arbel_fill_send_wqe[])(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
static
Initial value:
= {
}
static size_t arbel_fill_rc_send_wqe(struct ib_device *ibdev, struct ib_queue_pair *qp __unused, struct ib_address_vector *dest __unused, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
Construct RC send work queue entry.
Definition: arbel.c:1382
static size_t arbel_fill_ud_send_wqe(struct ib_device *ibdev, struct ib_queue_pair *qp __unused, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
Construct UD send work queue entry.
Definition: arbel.c:1288
static size_t arbel_fill_mlx_send_wqe(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf, union arbel_send_wqe *wqe)
Construct MLX send work queue entry.
Definition: arbel.c:1332

Work queue entry constructors.

Definition at line 1403 of file arbel.c.

Referenced by arbel_post_send().

◆ arbel_ib_operations

struct ib_device_operations arbel_ib_operations
static
Initial value:
= {
.create_cq = arbel_create_cq,
.destroy_cq = arbel_destroy_cq,
.create_qp = arbel_create_qp,
.modify_qp = arbel_modify_qp,
.destroy_qp = arbel_destroy_qp,
.post_send = arbel_post_send,
.post_recv = arbel_post_recv,
.poll_cq = arbel_poll_cq,
.poll_eq = arbel_poll_eq,
.open = arbel_ib_open,
.close = arbel_ib_close,
.mcast_attach = arbel_mcast_attach,
.mcast_detach = arbel_mcast_detach,
.set_port_info = arbel_inform_sma,
.set_pkey_table = arbel_inform_sma,
}
static void arbel_poll_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Poll completion queue.
Definition: arbel.c:1694
static void arbel_poll_eq(struct ib_device *ibdev)
Poll event queue.
Definition: arbel.c:1884
static void arbel_ib_close(struct ib_device *ibdev)
Close Infiniband link.
Definition: arbel.c:2816
static void arbel_destroy_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Destroy queue pair.
Definition: arbel.c:1211
static int arbel_create_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Create completion queue.
Definition: arbel.c:611
static void arbel_mcast_detach(struct ib_device *ibdev, struct ib_queue_pair *qp __unused, union ib_gid *gid)
Detach from multicast group.
Definition: arbel.c:2921
static int arbel_ib_open(struct ib_device *ibdev)
Initialise Infiniband link.
Definition: arbel.c:2776
static int arbel_post_recv(struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf)
Post receive work queue entry.
Definition: arbel.c:1498
static int arbel_create_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Create queue pair.
Definition: arbel.c:971
static int arbel_mcast_attach(struct ib_device *ibdev, struct ib_queue_pair *qp, union ib_gid *gid)
Attach to multicast group.
Definition: arbel.c:2866
static int arbel_post_send(struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *dest, struct io_buffer *iobuf)
Post send work queue entry.
Definition: arbel.c:1423
static int arbel_modify_qp(struct ib_device *ibdev, struct ib_queue_pair *qp)
Modify queue pair.
Definition: arbel.c:1124
static void arbel_destroy_cq(struct ib_device *ibdev, struct ib_completion_queue *cq)
Destroy completion queue.
Definition: arbel.c:715
static int arbel_inform_sma(struct ib_device *ibdev, union ib_mad *mad)
Inform embedded subnet management agent of a received MAD.
Definition: arbel.c:2838

Arbel Infiniband operations.

Definition at line 2948 of file arbel.c.

Referenced by arbel_probe().

◆ arbel_nics

struct pci_device_id arbel_nics[]
static
Initial value:
= {
PCI_ROM ( 0x15b3, 0x6274, "mt25204", "MT25204 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x6282, "mt25218", "MT25218 HCA driver", 0 ),
}
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:303

Definition at line 3141 of file arbel.c.

◆ __pci_driver

struct pci_driver arbel_driver __pci_driver
Initial value:
= {
.ids = arbel_nics,
.id_count = ( sizeof ( arbel_nics ) / sizeof ( arbel_nics[0] ) ),
.probe = arbel_probe,
}
static void arbel_remove(struct pci_device *pci)
Remove PCI device.
Definition: arbel.c:3128
static struct pci_device_id arbel_nics[]
Definition: arbel.c:3141
static struct xen_remove_from_physmap * remove
Definition: xenmem.h:39
static int arbel_probe(struct pci_device *pci)
Probe PCI device.
Definition: arbel.c:3026

Definition at line 3146 of file arbel.c.