iPXE
ena.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2018 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdint.h>
27 #include <string.h>
28 #include <unistd.h>
29 #include <errno.h>
30 #include <byteswap.h>
31 #include <ipxe/netdevice.h>
32 #include <ipxe/ethernet.h>
33 #include <ipxe/if_ether.h>
34 #include <ipxe/iobuf.h>
35 #include <ipxe/malloc.h>
36 #include <ipxe/pci.h>
37 #include "ena.h"
38 
39 /** @file
40  *
41  * Amazon ENA network driver
42  *
43  */
44 
45 /**
46  * Get direction name (for debugging)
47  *
48  * @v direction Direction
49  * @ret name Direction name
50  */
51 static const char * ena_direction ( unsigned int direction ) {
52 
53  switch ( direction ) {
54  case ENA_SQ_TX: return "TX";
55  case ENA_SQ_RX: return "RX";
56  default: return "<UNKNOWN>";
57  }
58 }
59 
60 /******************************************************************************
61  *
62  * Device reset
63  *
64  ******************************************************************************
65  */
66 
67 /**
68  * Wait for reset operation to be acknowledged
69  *
70  * @v ena ENA device
71  * @v expected Expected reset state
72  * @ret rc Return status code
73  */
74 static int ena_reset_wait ( struct ena_nic *ena, uint32_t expected ) {
75  uint32_t stat;
76  unsigned int i;
77 
78  /* Wait for reset to complete */
79  for ( i = 0 ; i < ENA_RESET_MAX_WAIT_MS ; i++ ) {
80 
81  /* Check if device is ready */
82  stat = readl ( ena->regs + ENA_STAT );
83  if ( ( stat & ENA_STAT_RESET ) == expected )
84  return 0;
85 
86  /* Delay */
87  mdelay ( 1 );
88  }
89 
90  DBGC ( ena, "ENA %p timed out waiting for reset status %#08x "
91  "(got %#08x)\n", ena, expected, stat );
92  return -ETIMEDOUT;
93 }
94 
95 /**
96  * Reset hardware
97  *
98  * @v ena ENA device
99  * @ret rc Return status code
100  */
101 static int ena_reset ( struct ena_nic *ena ) {
102  int rc;
103 
104  /* Trigger reset */
105  writel ( ENA_CTRL_RESET, ( ena->regs + ENA_CTRL ) );
106 
107  /* Wait for reset to take effect */
108  if ( ( rc = ena_reset_wait ( ena, ENA_STAT_RESET ) ) != 0 )
109  return rc;
110 
111  /* Clear reset */
112  writel ( 0, ( ena->regs + ENA_CTRL ) );
113 
114  /* Wait for reset to clear */
115  if ( ( rc = ena_reset_wait ( ena, 0 ) ) != 0 )
116  return rc;
117 
118  return 0;
119 }
120 
121 /******************************************************************************
122  *
123  * Admin queue
124  *
125  ******************************************************************************
126  */
127 
128 /**
129  * Set queue base address
130  *
131  * @v ena ENA device
132  * @v offset Register offset
133  * @v address Base address
134  */
135 static inline void ena_set_base ( struct ena_nic *ena, unsigned int offset,
136  void *base ) {
138 
139  /* Program base address registers */
140  writel ( ( phys & 0xffffffffUL ),
141  ( ena->regs + offset + ENA_BASE_LO ) );
142  if ( sizeof ( phys ) > sizeof ( uint32_t ) ) {
143  writel ( ( ( ( uint64_t ) phys ) >> 32 ),
144  ( ena->regs + offset + ENA_BASE_HI ) );
145  } else {
146  writel ( 0, ( ena->regs + offset + ENA_BASE_HI ) );
147  }
148 }
149 
150 /**
151  * Set queue capabilities
152  *
153  * @v ena ENA device
154  * @v offset Register offset
155  * @v count Number of entries
156  * @v size Size of each entry
157  */
158 static inline __attribute__ (( always_inline )) void
159 ena_set_caps ( struct ena_nic *ena, unsigned int offset, unsigned int count,
160  size_t size ) {
161 
162  /* Program capabilities register */
163  writel ( ENA_CAPS ( count, size ), ( ena->regs + offset ) );
164 }
165 
166 /**
167  * Clear queue capabilities
168  *
169  * @v ena ENA device
170  * @v offset Register offset
171  */
172 static inline __attribute__ (( always_inline )) void
173 ena_clear_caps ( struct ena_nic *ena, unsigned int offset ) {
174 
175  /* Clear capabilities register */
176  writel ( 0, ( ena->regs + offset ) );
177 }
178 
179 /**
180  * Create admin queues
181  *
182  * @v ena ENA device
183  * @ret rc Return status code
184  */
185 static int ena_create_admin ( struct ena_nic *ena ) {
186  size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
187  size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
188  int rc;
189 
190  /* Allocate admin completion queue */
191  ena->acq.rsp = malloc_phys ( acq_len, acq_len );
192  if ( ! ena->acq.rsp ) {
193  rc = -ENOMEM;
194  goto err_alloc_acq;
195  }
196  memset ( ena->acq.rsp, 0, acq_len );
197 
198  /* Allocate admin queue */
199  ena->aq.req = malloc_phys ( aq_len, aq_len );
200  if ( ! ena->aq.req ) {
201  rc = -ENOMEM;
202  goto err_alloc_aq;
203  }
204  memset ( ena->aq.req, 0, aq_len );
205 
206  /* Program queue addresses and capabilities */
207  ena_set_base ( ena, ENA_ACQ_BASE, ena->acq.rsp );
209  sizeof ( ena->acq.rsp[0] ) );
210  ena_set_base ( ena, ENA_AQ_BASE, ena->aq.req );
212  sizeof ( ena->aq.req[0] ) );
213 
214  DBGC ( ena, "ENA %p AQ [%08lx,%08lx) ACQ [%08lx,%08lx)\n",
215  ena, virt_to_phys ( ena->aq.req ),
216  ( virt_to_phys ( ena->aq.req ) + aq_len ),
217  virt_to_phys ( ena->acq.rsp ),
218  ( virt_to_phys ( ena->acq.rsp ) + acq_len ) );
219  return 0;
220 
221  ena_clear_caps ( ena, ENA_AQ_CAPS );
222  ena_clear_caps ( ena, ENA_ACQ_CAPS );
223  free_phys ( ena->aq.req, aq_len );
224  err_alloc_aq:
225  free_phys ( ena->acq.rsp, acq_len );
226  err_alloc_acq:
227  return rc;
228 }
229 
230 /**
231  * Destroy admin queues
232  *
233  * @v ena ENA device
234  */
235 static void ena_destroy_admin ( struct ena_nic *ena ) {
236  size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
237  size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
238 
239  /* Clear queue capabilities */
240  ena_clear_caps ( ena, ENA_AQ_CAPS );
241  ena_clear_caps ( ena, ENA_ACQ_CAPS );
242  wmb();
243 
244  /* Free queues */
245  free_phys ( ena->aq.req, aq_len );
246  free_phys ( ena->acq.rsp, acq_len );
247  DBGC ( ena, "ENA %p AQ and ACQ destroyed\n", ena );
248 }
249 
250 /**
251  * Get next available admin queue request
252  *
253  * @v ena ENA device
254  * @ret req Admin queue request
255  */
256 static union ena_aq_req * ena_admin_req ( struct ena_nic *ena ) {
257  union ena_aq_req *req;
258  unsigned int index;
259 
260  /* Get next request */
261  index = ( ena->aq.prod % ENA_AQ_COUNT );
262  req = &ena->aq.req[index];
263 
264  /* Initialise request */
265  memset ( ( ( ( void * ) req ) + sizeof ( req->header ) ), 0,
266  ( sizeof ( *req ) - sizeof ( req->header ) ) );
267  req->header.id = ena->aq.prod;
268 
269  /* Increment producer counter */
270  ena->aq.prod++;
271 
272  return req;
273 }
274 
275 /**
276  * Issue admin queue request
277  *
278  * @v ena ENA device
279  * @v req Admin queue request
280  * @v rsp Admin queue response to fill in
281  * @ret rc Return status code
282  */
283 static int ena_admin ( struct ena_nic *ena, union ena_aq_req *req,
284  union ena_acq_rsp **rsp ) {
285  unsigned int index;
286  unsigned int i;
287  int rc;
288 
289  /* Locate response */
290  index = ( ena->acq.cons % ENA_ACQ_COUNT );
291  *rsp = &ena->acq.rsp[index];
292 
293  /* Mark request as ready */
294  req->header.flags ^= ENA_AQ_PHASE;
295  wmb();
296  DBGC2 ( ena, "ENA %p admin request %#x:\n",
297  ena, le16_to_cpu ( req->header.id ) );
298  DBGC2_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
299 
300  /* Ring doorbell */
301  writel ( ena->aq.prod, ( ena->regs + ENA_AQ_DB ) );
302 
303  /* Wait for response */
304  for ( i = 0 ; i < ENA_ADMIN_MAX_WAIT_MS ; i++ ) {
305 
306  /* Check for response */
307  if ( ( (*rsp)->header.flags ^ ena->acq.phase ) & ENA_ACQ_PHASE){
308  mdelay ( 1 );
309  continue;
310  }
311  DBGC2 ( ena, "ENA %p admin response %#x:\n",
312  ena, le16_to_cpu ( (*rsp)->header.id ) );
313  DBGC2_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ));
314 
315  /* Increment consumer counter */
316  ena->acq.cons++;
317  if ( ( ena->acq.cons % ENA_ACQ_COUNT ) == 0 )
318  ena->acq.phase ^= ENA_ACQ_PHASE;
319 
320  /* Check command identifier */
321  if ( (*rsp)->header.id != req->header.id ) {
322  DBGC ( ena, "ENA %p admin response %#x mismatch:\n",
323  ena, le16_to_cpu ( (*rsp)->header.id ) );
324  rc = -EILSEQ;
325  goto err;
326  }
327 
328  /* Check status */
329  if ( (*rsp)->header.status != 0 ) {
330  DBGC ( ena, "ENA %p admin response %#x status %d:\n",
331  ena, le16_to_cpu ( (*rsp)->header.id ),
332  (*rsp)->header.status );
333  rc = -EIO;
334  goto err;
335  }
336 
337  /* Success */
338  return 0;
339  }
340 
341  rc = -ETIMEDOUT;
342  DBGC ( ena, "ENA %p timed out waiting for admin request %#x:\n",
343  ena, le16_to_cpu ( req->header.id ) );
344  err:
345  DBGC_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
346  DBGC_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ) );
347  return rc;
348 }
349 
350 /**
351  * Create submission queue
352  *
353  * @v ena ENA device
354  * @v sq Submission queue
355  * @v cq Corresponding completion queue
356  * @ret rc Return status code
357  */
358 static int ena_create_sq ( struct ena_nic *ena, struct ena_sq *sq,
359  struct ena_cq *cq ) {
360  union ena_aq_req *req;
361  union ena_acq_rsp *rsp;
362  int rc;
363 
364  /* Allocate submission queue entries */
365  sq->sqe.raw = malloc_phys ( sq->len, ENA_ALIGN );
366  if ( ! sq->sqe.raw ) {
367  rc = -ENOMEM;
368  goto err_alloc;
369  }
370  memset ( sq->sqe.raw, 0, sq->len );
371 
372  /* Construct request */
373  req = ena_admin_req ( ena );
374  req->header.opcode = ENA_CREATE_SQ;
375  req->create_sq.direction = sq->direction;
378  req->create_sq.cq_id = cpu_to_le16 ( cq->id );
379  req->create_sq.count = cpu_to_le16 ( sq->count );
380  req->create_sq.address = cpu_to_le64 ( virt_to_bus ( sq->sqe.raw ) );
381 
382  /* Issue request */
383  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
384  goto err_admin;
385 
386  /* Parse response */
387  sq->id = le16_to_cpu ( rsp->create_sq.id );
388  sq->doorbell = le32_to_cpu ( rsp->create_sq.doorbell );
389 
390  /* Reset producer counter and phase */
391  sq->prod = 0;
392  sq->phase = ENA_SQE_PHASE;
393 
394  DBGC ( ena, "ENA %p %s SQ%d at [%08lx,%08lx) db +%04x CQ%d\n",
395  ena, ena_direction ( sq->direction ), sq->id,
396  virt_to_phys ( sq->sqe.raw ),
397  ( virt_to_phys ( sq->sqe.raw ) + sq->len ),
398  sq->doorbell, cq->id );
399  return 0;
400 
401  err_admin:
402  free_phys ( sq->sqe.raw, sq->len );
403  err_alloc:
404  return rc;
405 }
406 
407 /**
408  * Destroy submission queue
409  *
410  * @v ena ENA device
411  * @v sq Submission queue
412  * @ret rc Return status code
413  */
414 static int ena_destroy_sq ( struct ena_nic *ena, struct ena_sq *sq ) {
415  union ena_aq_req *req;
416  union ena_acq_rsp *rsp;
417  int rc;
418 
419  /* Construct request */
420  req = ena_admin_req ( ena );
422  req->destroy_sq.id = cpu_to_le16 ( sq->id );
423  req->destroy_sq.direction = sq->direction;
424 
425  /* Issue request */
426  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
427  return rc;
428 
429  /* Free submission queue entries */
430  free_phys ( sq->sqe.raw, sq->len );
431 
432  DBGC ( ena, "ENA %p %s SQ%d destroyed\n",
433  ena, ena_direction ( sq->direction ), sq->id );
434  return 0;
435 }
436 
437 /**
438  * Create completion queue
439  *
440  * @v ena ENA device
441  * @v cq Completion queue
442  * @ret rc Return status code
443  */
444 static int ena_create_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
445  union ena_aq_req *req;
446  union ena_acq_rsp *rsp;
447  int rc;
448 
449  /* Allocate completion queue entries */
450  cq->cqe.raw = malloc_phys ( cq->len, ENA_ALIGN );
451  if ( ! cq->cqe.raw ) {
452  rc = -ENOMEM;
453  goto err_alloc;
454  }
455  memset ( cq->cqe.raw, 0, cq->len );
456 
457  /* Construct request */
458  req = ena_admin_req ( ena );
459  req->header.opcode = ENA_CREATE_CQ;
460  req->create_cq.size = cq->size;
461  req->create_cq.count = cpu_to_le16 ( cq->requested );
462  req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );
463 
464  /* Issue request */
465  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
466  goto err_admin;
467 
468  /* Parse response */
469  cq->id = le16_to_cpu ( rsp->create_cq.id );
470  cq->actual = le16_to_cpu ( rsp->create_cq.count );
471  cq->doorbell = le32_to_cpu ( rsp->create_cq.doorbell );
472  cq->mask = ( cq->actual - 1 );
473  if ( cq->actual != cq->requested ) {
474  DBGC ( ena, "ENA %p CQ%d requested %d actual %d\n",
475  ena, cq->id, cq->requested, cq->actual );
476  }
477 
478  /* Reset consumer counter and phase */
479  cq->cons = 0;
480  cq->phase = ENA_CQE_PHASE;
481 
482  DBGC ( ena, "ENA %p CQ%d at [%08lx,%08lx) db +%04x\n",
483  ena, cq->id, virt_to_phys ( cq->cqe.raw ),
484  ( virt_to_phys ( cq->cqe.raw ) + cq->len ), cq->doorbell );
485  return 0;
486 
487  err_admin:
488  free_phys ( cq->cqe.raw, cq->len );
489  err_alloc:
490  return rc;
491 }
492 
493 /**
494  * Destroy completion queue
495  *
496  * @v ena ENA device
497  * @v cq Completion queue
498  * @ret rc Return status code
499  */
500 static int ena_destroy_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
501  union ena_aq_req *req;
502  union ena_acq_rsp *rsp;
503  int rc;
504 
505  /* Construct request */
506  req = ena_admin_req ( ena );
508  req->destroy_cq.id = cpu_to_le16 ( cq->id );
509 
510  /* Issue request */
511  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
512  return rc;
513 
514  /* Free completion queue entries */
515  free_phys ( cq->cqe.raw, cq->len );
516 
517  DBGC ( ena, "ENA %p CQ%d destroyed\n", ena, cq->id );
518  return 0;
519 }
520 
521 /**
522  * Create queue pair
523  *
524  * @v ena ENA device
525  * @v qp Queue pair
526  * @ret rc Return status code
527  */
528 static int ena_create_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
529  int rc;
530 
531  /* Create completion queue */
532  if ( ( rc = ena_create_cq ( ena, &qp->cq ) ) != 0 )
533  goto err_create_cq;
534 
535  /* Create submission queue */
536  if ( ( rc = ena_create_sq ( ena, &qp->sq, &qp->cq ) ) != 0 )
537  goto err_create_sq;
538 
539  return 0;
540 
541  ena_destroy_sq ( ena, &qp->sq );
542  err_create_sq:
543  ena_destroy_cq ( ena, &qp->cq );
544  err_create_cq:
545  return rc;
546 }
547 
548 /**
549  * Destroy queue pair
550  *
551  * @v ena ENA device
552  * @v qp Queue pair
553  * @ret rc Return status code
554  */
555 static int ena_destroy_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
556 
557  /* Destroy submission queue */
558  ena_destroy_sq ( ena, &qp->sq );
559 
560  /* Destroy completion queue */
561  ena_destroy_cq ( ena, &qp->cq );
562 
563  return 0;
564 }
565 
566 /**
567  * Get device attributes
568  *
569  * @v netdev Network device
570  * @ret rc Return status code
571  */
573  struct ena_nic *ena = netdev->priv;
574  union ena_aq_req *req;
575  union ena_acq_rsp *rsp;
576  union ena_feature *feature;
577  int rc;
578 
579  /* Construct request */
580  req = ena_admin_req ( ena );
583 
584  /* Issue request */
585  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
586  return rc;
587 
588  /* Parse response */
589  feature = &rsp->get_feature.feature;
590  memcpy ( netdev->hw_addr, feature->device.mac, ETH_ALEN );
591  netdev->max_pkt_len = le32_to_cpu ( feature->device.mtu );
593 
594  DBGC ( ena, "ENA %p MAC %s MTU %zd\n",
595  ena, eth_ntoa ( netdev->hw_addr ), netdev->max_pkt_len );
596  return 0;
597 }
598 
599 /**
600  * Get statistics (for debugging)
601  *
602  * @v ena ENA device
603  * @ret rc Return status code
604  */
605 static int ena_get_stats ( struct ena_nic *ena ) {
606  union ena_aq_req *req;
607  union ena_acq_rsp *rsp;
608  struct ena_get_stats_rsp *stats;
609  int rc;
610 
611  /* Do nothing unless debug messages are enabled */
612  if ( ! DBG_LOG )
613  return 0;
614 
615  /* Construct request */
616  req = ena_admin_req ( ena );
617  req->header.opcode = ENA_GET_STATS;
621 
622  /* Issue request */
623  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
624  return rc;
625 
626  /* Parse response */
627  stats = &rsp->get_stats;
628  DBGC ( ena, "ENA %p TX bytes %#llx packets %#llx\n", ena,
629  ( ( unsigned long long ) le64_to_cpu ( stats->tx_bytes ) ),
630  ( ( unsigned long long ) le64_to_cpu ( stats->tx_packets ) ) );
631  DBGC ( ena, "ENA %p RX bytes %#llx packets %#llx drops %#llx\n", ena,
632  ( ( unsigned long long ) le64_to_cpu ( stats->rx_bytes ) ),
633  ( ( unsigned long long ) le64_to_cpu ( stats->rx_packets ) ),
634  ( ( unsigned long long ) le64_to_cpu ( stats->rx_drops ) ) );
635 
636  return 0;
637 }
638 
639 /******************************************************************************
640  *
641  * Network device interface
642  *
643  ******************************************************************************
644  */
645 
646 /**
647  * Refill receive queue
648  *
649  * @v netdev Network device
650  */
651 static void ena_refill_rx ( struct net_device *netdev ) {
652  struct ena_nic *ena = netdev->priv;
653  struct io_buffer *iobuf;
654  struct ena_rx_sqe *sqe;
655  unsigned int index;
657  size_t len = netdev->max_pkt_len;
658  unsigned int refilled = 0;
659 
660  /* Refill queue */
661  while ( ( ena->rx.sq.prod - ena->rx.cq.cons ) < ENA_RX_COUNT ) {
662 
663  /* Allocate I/O buffer */
664  iobuf = alloc_iob ( len );
665  if ( ! iobuf ) {
666  /* Wait for next refill */
667  break;
668  }
669 
670  /* Get next submission queue entry */
671  index = ( ena->rx.sq.prod % ENA_RX_COUNT );
672  sqe = &ena->rx.sq.sqe.rx[index];
673 
674  /* Construct submission queue entry */
675  address = virt_to_bus ( iobuf->data );
676  sqe->len = cpu_to_le16 ( len );
677  sqe->id = cpu_to_le16 ( ena->rx.sq.prod );
678  sqe->address = cpu_to_le64 ( address );
679  wmb();
681  ena->rx.sq.phase );
682 
683  /* Increment producer counter */
684  ena->rx.sq.prod++;
685  if ( ( ena->rx.sq.prod % ENA_RX_COUNT ) == 0 )
686  ena->rx.sq.phase ^= ENA_SQE_PHASE;
687 
688  /* Record I/O buffer */
689  assert ( ena->rx_iobuf[index] == NULL );
690  ena->rx_iobuf[index] = iobuf;
691 
692  DBGC2 ( ena, "ENA %p RX %d at [%08llx,%08llx)\n", ena, sqe->id,
693  ( ( unsigned long long ) address ),
694  ( ( unsigned long long ) address + len ) );
695  refilled++;
696  }
697 
698  /* Ring doorbell, if applicable */
699  if ( refilled ) {
700  wmb();
701  writel ( ena->rx.sq.prod, ( ena->regs + ena->rx.sq.doorbell ) );
702  }
703 }
704 
705 /**
706  * Discard unused receive I/O buffers
707  *
708  * @v ena ENA device
709  */
710 static void ena_empty_rx ( struct ena_nic *ena ) {
711  unsigned int i;
712 
713  for ( i = 0 ; i < ENA_RX_COUNT ; i++ ) {
714  if ( ena->rx_iobuf[i] )
715  free_iob ( ena->rx_iobuf[i] );
716  ena->rx_iobuf[i] = NULL;
717  }
718 }
719 
720 /**
721  * Open network device
722  *
723  * @v netdev Network device
724  * @ret rc Return status code
725  */
726 static int ena_open ( struct net_device *netdev ) {
727  struct ena_nic *ena = netdev->priv;
728  int rc;
729 
730  /* Create transmit queue pair */
731  if ( ( rc = ena_create_qp ( ena, &ena->tx ) ) != 0 )
732  goto err_create_tx;
733 
734  /* Create receive queue pair */
735  if ( ( rc = ena_create_qp ( ena, &ena->rx ) ) != 0 )
736  goto err_create_rx;
737 
738  /* Refill receive queue */
739  ena_refill_rx ( netdev );
740 
741  return 0;
742 
743  ena_destroy_qp ( ena, &ena->rx );
744  err_create_rx:
745  ena_destroy_qp ( ena, &ena->tx );
746  err_create_tx:
747  return rc;
748 }
749 
750 /**
751  * Close network device
752  *
753  * @v netdev Network device
754  */
755 static void ena_close ( struct net_device *netdev ) {
756  struct ena_nic *ena = netdev->priv;
757 
758  /* Dump statistics (for debugging) */
759  ena_get_stats ( ena );
760 
761  /* Destroy receive queue pair */
762  ena_destroy_qp ( ena, &ena->rx );
763 
764  /* Discard any unused receive buffers */
765  ena_empty_rx ( ena );
766 
767  /* Destroy transmit queue pair */
768  ena_destroy_qp ( ena, &ena->tx );
769 }
770 
771 /**
772  * Transmit packet
773  *
774  * @v netdev Network device
775  * @v iobuf I/O buffer
776  * @ret rc Return status code
777  */
778 static int ena_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
779  struct ena_nic *ena = netdev->priv;
780  struct ena_tx_sqe *sqe;
781  unsigned int index;
783  size_t len;
784 
785  /* Get next submission queue entry */
786  if ( ( ena->tx.sq.prod - ena->tx.cq.cons ) >= ENA_TX_COUNT ) {
787  DBGC ( ena, "ENA %p out of transmit descriptors\n", ena );
788  return -ENOBUFS;
789  }
790  index = ( ena->tx.sq.prod % ENA_TX_COUNT );
791  sqe = &ena->tx.sq.sqe.tx[index];
792 
793  /* Construct submission queue entry */
794  address = virt_to_bus ( iobuf->data );
795  len = iob_len ( iobuf );
796  sqe->len = cpu_to_le16 ( len );
797  sqe->id = ena->tx.sq.prod;
798  sqe->address = cpu_to_le64 ( address );
799  wmb();
801  ena->tx.sq.phase );
802  wmb();
803 
804  /* Increment producer counter */
805  ena->tx.sq.prod++;
806  if ( ( ena->tx.sq.prod % ENA_TX_COUNT ) == 0 )
807  ena->tx.sq.phase ^= ENA_SQE_PHASE;
808 
809  /* Ring doorbell */
810  writel ( ena->tx.sq.prod, ( ena->regs + ena->tx.sq.doorbell ) );
811 
812  DBGC2 ( ena, "ENA %p TX %d at [%08llx,%08llx)\n", ena, sqe->id,
813  ( ( unsigned long long ) address ),
814  ( ( unsigned long long ) address + len ) );
815  return 0;
816 }
817 
818 /**
819  * Poll for completed transmissions
820  *
821  * @v netdev Network device
822  */
823 static void ena_poll_tx ( struct net_device *netdev ) {
824  struct ena_nic *ena = netdev->priv;
825  struct ena_tx_cqe *cqe;
826  unsigned int index;
827 
828  /* Check for completed packets */
829  while ( ena->tx.cq.cons != ena->tx.sq.prod ) {
830 
831  /* Get next completion queue entry */
832  index = ( ena->tx.cq.cons & ena->tx.cq.mask );
833  cqe = &ena->tx.cq.cqe.tx[index];
834 
835  /* Stop if completion queue entry is empty */
836  if ( ( cqe->flags ^ ena->tx.cq.phase ) & ENA_CQE_PHASE )
837  return;
838  DBGC2 ( ena, "ENA %p TX %d complete\n", ena,
839  ( le16_to_cpu ( cqe->id ) >> 2 /* Don't ask */ ) );
840 
841  /* Increment consumer counter */
842  ena->tx.cq.cons++;
843  if ( ! ( ena->tx.cq.cons & ena->tx.cq.mask ) )
844  ena->tx.cq.phase ^= ENA_CQE_PHASE;
845 
846  /* Complete transmit */
848  }
849 }
850 
851 /**
852  * Poll for received packets
853  *
854  * @v netdev Network device
855  */
856 static void ena_poll_rx ( struct net_device *netdev ) {
857  struct ena_nic *ena = netdev->priv;
858  struct ena_rx_cqe *cqe;
859  struct io_buffer *iobuf;
860  unsigned int index;
861  size_t len;
862 
863  /* Check for received packets */
864  while ( ena->rx.cq.cons != ena->rx.sq.prod ) {
865 
866  /* Get next completion queue entry */
867  index = ( ena->rx.cq.cons % ENA_RX_COUNT );
868  cqe = &ena->rx.cq.cqe.rx[index];
869 
870  /* Stop if completion queue entry is empty */
871  if ( ( cqe->flags ^ ena->rx.cq.phase ) & ENA_CQE_PHASE )
872  return;
873 
874  /* Increment consumer counter */
875  ena->rx.cq.cons++;
876  if ( ! ( ena->rx.cq.cons & ena->rx.cq.mask ) )
877  ena->rx.cq.phase ^= ENA_CQE_PHASE;
878 
879  /* Populate I/O buffer */
880  iobuf = ena->rx_iobuf[index];
881  ena->rx_iobuf[index] = NULL;
882  len = le16_to_cpu ( cqe->len );
883  iob_put ( iobuf, len );
884 
885  /* Hand off to network stack */
886  DBGC2 ( ena, "ENA %p RX %d complete (length %zd)\n",
887  ena, le16_to_cpu ( cqe->id ), len );
888  netdev_rx ( netdev, iobuf );
889  }
890 }
891 
892 /**
893  * Poll for completed and received packets
894  *
895  * @v netdev Network device
896  */
897 static void ena_poll ( struct net_device *netdev ) {
898 
899  /* Poll for transmit completions */
900  ena_poll_tx ( netdev );
901 
902  /* Poll for receive completions */
903  ena_poll_rx ( netdev );
904 
905  /* Refill receive ring */
906  ena_refill_rx ( netdev );
907 }
908 
909 /** ENA network device operations */
911  .open = ena_open,
912  .close = ena_close,
913  .transmit = ena_transmit,
914  .poll = ena_poll,
915 };
916 
917 /******************************************************************************
918  *
919  * PCI interface
920  *
921  ******************************************************************************
922  */
923 
924 /**
925  * Probe PCI device
926  *
927  * @v pci PCI device
928  * @ret rc Return status code
929  */
930 static int ena_probe ( struct pci_device *pci ) {
931  struct net_device *netdev;
932  struct ena_nic *ena;
933  int rc;
934 
935  /* Allocate and initialise net device */
936  netdev = alloc_etherdev ( sizeof ( *ena ) );
937  if ( ! netdev ) {
938  rc = -ENOMEM;
939  goto err_alloc;
940  }
942  ena = netdev->priv;
943  pci_set_drvdata ( pci, netdev );
944  netdev->dev = &pci->dev;
945  memset ( ena, 0, sizeof ( *ena ) );
946  ena->acq.phase = ENA_ACQ_PHASE;
947  ena_cq_init ( &ena->tx.cq, ENA_TX_COUNT,
948  sizeof ( ena->tx.cq.cqe.tx[0] ) );
949  ena_sq_init ( &ena->tx.sq, ENA_SQ_TX, ENA_TX_COUNT,
950  sizeof ( ena->tx.sq.sqe.tx[0] ) );
951  ena_cq_init ( &ena->rx.cq, ENA_RX_COUNT,
952  sizeof ( ena->rx.cq.cqe.rx[0] ) );
953  ena_sq_init ( &ena->rx.sq, ENA_SQ_RX, ENA_RX_COUNT,
954  sizeof ( ena->rx.sq.sqe.rx[0] ) );
955 
956  /* Fix up PCI device */
957  adjust_pci_device ( pci );
958 
959  /* Map registers */
960  ena->regs = pci_ioremap ( pci, pci->membase, ENA_BAR_SIZE );
961  if ( ! ena->regs ) {
962  rc = -ENODEV;
963  goto err_ioremap;
964  }
965 
966  /* Reset the NIC */
967  if ( ( rc = ena_reset ( ena ) ) != 0 )
968  goto err_reset;
969 
970  /* Create admin queues */
971  if ( ( rc = ena_create_admin ( ena ) ) != 0 )
972  goto err_create_admin;
973 
974  /* Fetch MAC address */
975  if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
976  goto err_get_device_attributes;
977 
978  /* Register network device */
979  if ( ( rc = register_netdev ( netdev ) ) != 0 )
980  goto err_register_netdev;
981 
982  /* Mark as link up, since we have no way to test link state on
983  * this hardware.
984  */
986 
987  return 0;
988 
990  err_register_netdev:
991  err_get_device_attributes:
992  ena_destroy_admin ( ena );
993  err_create_admin:
994  ena_reset ( ena );
995  err_reset:
996  iounmap ( ena->regs );
997  err_ioremap:
999  netdev_put ( netdev );
1000  err_alloc:
1001  return rc;
1002 }
1003 
1004 /**
1005  * Remove PCI device
1006  *
1007  * @v pci PCI device
1008  */
1009 static void ena_remove ( struct pci_device *pci ) {
1010  struct net_device *netdev = pci_get_drvdata ( pci );
1011  struct ena_nic *ena = netdev->priv;
1012 
1013  /* Unregister network device */
1015 
1016  /* Destroy admin queues */
1017  ena_destroy_admin ( ena );
1018 
1019  /* Reset card */
1020  ena_reset ( ena );
1021 
1022  /* Free network device */
1023  iounmap ( ena->regs );
1024  netdev_nullify ( netdev );
1025  netdev_put ( netdev );
1026 }
1027 
1028 /** ENA PCI device IDs */
1029 static struct pci_device_id ena_nics[] = {
1030  PCI_ROM ( 0x1d0f, 0xec20, "ena-vf", "ENA VF", 0 ),
1031  PCI_ROM ( 0x1d0f, 0xec21, "ena-vf-llq", "ENA VF (LLQ)", 0 ),
1032 };
1033 
1034 /** ENA PCI driver */
1035 struct pci_driver ena_driver __pci_driver = {
1036  .ids = ena_nics,
1037  .id_count = ( sizeof ( ena_nics ) / sizeof ( ena_nics[0] ) ),
1038  .probe = ena_probe,
1039  .remove = ena_remove,
1040 };
struct ena_aq aq
Admin queue.
Definition: ena.h:577
Queue pair.
Definition: ena.h:565
static void ena_refill_rx(struct net_device *netdev)
Refill receive queue.
Definition: ena.c:651
#define __attribute__(x)
Definition: compiler.h:10
#define ENA_ACQ_COUNT
Number of admin completion queue entries.
Definition: ena.h:25
struct ena_tx_sqe * tx
Transmit submission queue entries.
Definition: ena.h:479
static void ena_destroy_admin(struct ena_nic *ena)
Destroy admin queues.
Definition: ena.c:235
struct ena_qp rx
Receive queue.
Definition: ena.h:583
unsigned long membase
Memory base.
Definition: pci.h:200
#define ENA_CTRL
Device control register.
Definition: ena.h:61
uint8_t type
Type.
Definition: ena.h:305
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static void ena_remove(struct pci_device *pci)
Remove PCI device.
Definition: ena.c:1009
wmb()
void * raw
Raw data.
Definition: ena.h:527
#define ENA_BASE_HI
Base address high register offset.
Definition: ena.h:37
#define iob_put(iobuf, len)
Definition: iobuf.h:120
#define ENA_RX_COUNT
Number of receive queue entries.
Definition: ena.h:31
#define ENA_AQ_COUNT
Number of admin queue entries.
Definition: ena.h:22
A PCI driver.
Definition: pci.h:230
static void ena_poll_tx(struct net_device *netdev)
Poll for completed transmissions.
Definition: ena.c:823
uint8_t mask
Actual number of entries minus one.
Definition: ena.h:546
size_t len
Total length of entries.
Definition: ena.h:488
unsigned int doorbell
Doorbell register offset.
Definition: ena.h:486
uint8_t count
Number of entries.
Definition: ena.h:498
#define le32_to_cpu(value)
Definition: byteswap.h:113
int(* open)(struct net_device *netdev)
Open network device.
Definition: netdevice.h:222
#define ENA_GET_FEATURE
Get feature.
Definition: ena.h:269
Error codes.
#define ENA_CREATE_CQ
Create completion queue.
Definition: ena.h:215
#define ENA_ACQ_BASE
Admin completion queue base address register.
Definition: ena.h:49
unsigned int prod
Producer counter.
Definition: ena.h:490
uint16_t id
Completion queue identifier.
Definition: ena.h:257
static void netdev_tx_complete_next(struct net_device *netdev)
Complete network transmission.
Definition: netdevice.h:770
I/O buffers.
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:232
uint64_t address
Base address.
Definition: ena.h:24
struct ena_tx_cqe * tx
Transmit completion queue entries.
Definition: ena.h:523
size_t mtu
Maximum transmission unit length.
Definition: netdevice.h:415
static int ena_get_device_attributes(struct net_device *netdev)
Get device attributes.
Definition: ena.c:572
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
static int ena_get_stats(struct ena_nic *ena)
Get statistics (for debugging)
Definition: ena.c:605
Transmit submission queue entry.
Definition: ena.h:400
struct ena_qp tx
Transmit queue.
Definition: ena.h:581
#define DBGC(...)
Definition: compiler.h:505
uint16_t count
Number of entries.
Definition: ena.h:160
union ena_sq::@39 sqe
Entries.
#define ENA_SQE_CPL
Request completion.
Definition: ena.h:441
Transmit completion queue entry.
Definition: ena.h:444
unsigned long long uint64_t
Definition: stdint.h:13
uint8_t id
Request identifier.
Definition: ena.h:410
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
static int ena_open(struct net_device *netdev)
Open network device.
Definition: ena.c:726
union ena_cq::@40 cqe
Entries.
#define cpu_to_le64(value)
Definition: byteswap.h:108
uint16_t count
Number of entries.
Definition: ena.h:226
uint8_t direction
Direction.
Definition: ena.h:14
uint64_t rx_drops
Receive drop count.
Definition: ena.h:338
uint16_t len
Length.
Definition: ena.h:418
#define ENA_AQ_BASE
Admin queue base address register.
Definition: ena.h:43
#define ENA_GET_STATS
Get statistics.
Definition: ena.h:296
static void ena_close(struct net_device *netdev)
Close network device.
Definition: ena.c:755
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:540
static int ena_create_qp(struct ena_nic *ena, struct ena_qp *qp)
Create queue pair.
Definition: ena.c:528
unsigned int phase
Phase.
Definition: ena.h:536
#define ENA_ACQ_CAPS
Admin completion queue capabilities register.
Definition: ena.h:52
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:154
struct io_buffer * alloc_iob(size_t len)
Allocate I/O buffer.
Definition: iobuf.c:129
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
uint64_t address
Address.
Definition: ena.h:428
struct device dev
Generic device.
Definition: pci.h:193
#define ENA_CTRL_RESET
Reset.
Definition: ena.h:62
#define ENA_TX_COUNT
Number of transmit queue entries.
Definition: ena.h:28
uint64_t address
Base address.
Definition: ena.h:230
unsigned int doorbell
Doorbell register offset.
Definition: ena.h:530
void * regs
Registers.
Definition: ena.h:575
Dynamic memory allocation.
union ena_feature feature
Feature.
Definition: ena.h:14
Completion queue.
Definition: ena.h:519
static int ena_reset_wait(struct ena_nic *ena, uint32_t expected)
Wait for reset operation to be acknowledged.
Definition: ena.c:74
Submission queue.
Definition: ena.h:475
static int ena_destroy_qp(struct ena_nic *ena, struct ena_qp *qp)
Destroy queue pair.
Definition: ena.c:555
static void ena_empty_rx(struct ena_nic *ena)
Discard unused receive I/O buffers.
Definition: ena.c:710
Feature.
Definition: ena.h:131
static void netdev_init(struct net_device *netdev, struct net_device_operations *op)
Initialise a network device.
Definition: netdevice.h:510
#define ENA_DEVICE_ATTRIBUTES
Device attributes.
Definition: ena.h:106
static int ena_destroy_sq(struct ena_nic *ena, struct ena_sq *sq)
Destroy submission queue.
Definition: ena.c:414
size_t len
Total length of entries.
Definition: ena.h:532
#define ENA_CAPS(count, size)
Capability register value.
Definition: ena.h:40
uint16_t id
Request identifier.
Definition: ena.h:446
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:344
#define ENOMEM
Not enough space.
Definition: errno.h:534
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static signed char phys[4]
Definition: epic100.c:88
uint64_t tx_packets
Transmit packet count.
Definition: ena.h:332
#define EILSEQ
Illegal byte sequence.
Definition: errno.h:413
uint16_t id
Request identifier.
Definition: ena.h:424
uint8_t id
Feature identifier.
Definition: ena.h:282
#define ENA_BASE_LO
Base address low register offset.
Definition: ena.h:34
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define ETH_HLEN
Definition: if_ether.h:9
#define ENA_STAT_RESET
Reset in progress.
Definition: ena.h:69
static void ena_set_caps(struct ena_nic *ena, unsigned int offset, unsigned int count, size_t size)
Set queue capabilities.
Definition: ena.c:159
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
static void netdev_put(struct net_device *netdev)
Drop reference to network device.
Definition: netdevice.h:567
Ethernet protocol.
union ena_aq_req * req
Requests.
Definition: ena.h:384
Transmit.
Definition: ena.h:139
uint64_t address
Base address.
Definition: ena.h:162
void * priv
Driver private data.
Definition: netdevice.h:431
static int ena_transmit(struct net_device *netdev, struct io_buffer *iobuf)
Transmit packet.
Definition: ena.c:778
#define DBGC_HDA(...)
Definition: compiler.h:506
uint16_t cq_id
Completion queue identifier.
Definition: ena.h:158
unsigned int cons
Consumer counter.
Definition: ena.h:394
static void netdev_link_up(struct net_device *netdev)
Mark network device as having link up.
Definition: netdevice.h:780
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
uint8_t flags
Flags.
Definition: ena.h:450
unsigned int cons
Consumer counter.
Definition: ena.h:534
uint16_t policy
Policy.
Definition: ena.h:156
#define ENA_CQE_PHASE
Completion queue ownership phase flag.
Definition: ena.h:472
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
static struct net_device * netdev
Definition: gdbudp.c:52
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:224
uint8_t flags
Flags.
Definition: ena.h:80
static int ena_probe(struct pci_device *pci)
Probe PCI device.
Definition: ena.c:930
#define ENA_STATS_TYPE_BASIC
Basic statistics.
Definition: ena.h:317
#define ENA_SQE_FIRST
This is the first descriptor.
Definition: ena.h:435
void unregister_netdev(struct net_device *netdev)
Unregister network device.
Definition: netdevice.c:899
unsigned int phase
Phase.
Definition: ena.h:492
struct ena_get_stats_req get_stats
Get statistics.
Definition: ena.h:356
static struct net_device_operations ena_operations
ENA network device operations.
Definition: ena.c:910
struct ena_create_cq_req create_cq
Create completion queue.
Definition: ena.h:350
struct ena_destroy_cq_req destroy_cq
Destroy completion queue.
Definition: ena.h:352
#define DBGC2_HDA(...)
Definition: compiler.h:523
uint8_t direction
Direction.
Definition: ena.h:152
uint64_t rsp
Definition: librm.h:267
static void ena_poll(struct net_device *netdev)
Poll for completed and received packets.
Definition: ena.c:897
struct ena_rx_cqe * rx
Receive completion queue entries.
Definition: ena.h:525
struct ena_cq cq
Completion queue.
Definition: ena.h:569
uint8_t flags
Flags.
Definition: ena.h:462
static int ena_create_admin(struct ena_nic *ena)
Create admin queues.
Definition: ena.c:185
uint8_t flags
Flags.
Definition: ena.h:406
#define ENA_ACQ_PHASE
Admin completion queue ownership phase flag.
Definition: ena.h:103
static int ena_create_cq(struct ena_nic *ena, struct ena_cq *cq)
Create completion queue.
Definition: ena.c:444
PCI bus.
A PCI device.
Definition: pci.h:191
int register_netdev(struct net_device *netdev)
Register network device.
Definition: netdevice.c:722
#define ENA_SQE_LAST
This is the last descriptor.
Definition: ena.h:438
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
const char * eth_ntoa(const void *ll_addr)
Transcribe Ethernet address.
Definition: ethernet.c:175
#define ENA_AQ_DB
Admin queue doorbell register.
Definition: ena.h:55
#define ENA_AQ_CAPS
Admin queue capabilities register.
Definition: ena.h:46
static int ena_admin(struct ena_nic *ena, union ena_aq_req *req, union ena_acq_rsp **rsp)
Issue admin queue request.
Definition: ena.c:283
A network device.
Definition: netdevice.h:352
#define ENODEV
No such device.
Definition: errno.h:509
#define ENA_ALIGN
Queue alignment.
Definition: ena.h:19
static void netdev_nullify(struct net_device *netdev)
Stop using a network device.
Definition: netdevice.h:523
uint16_t id
Completion queue identifier.
Definition: ena.h:538
#define ENA_CREATE_SQ
Create submission queue.
Definition: ena.h:145
#define ENA_AQ_PHASE
Admin queue ownership phase flag.
Definition: ena.h:84
#define ENA_BAR_SIZE
BAR size.
Definition: ena.h:16
uint8_t actual
Actual number of entries.
Definition: ena.h:544
static void ena_poll_rx(struct net_device *netdev)
Poll for received packets.
Definition: ena.c:856
#define ENA_STATS_SCOPE_ETH
Ethernet statistics.
Definition: ena.h:320
#define ETH_ALEN
Definition: if_ether.h:8
A PCI device ID list entry.
Definition: pci.h:155
#define le16_to_cpu(value)
Definition: byteswap.h:112
unsigned int uint32_t
Definition: stdint.h:12
uint8_t requested
Requested number of entries.
Definition: ena.h:542
#define ENA_DESTROY_CQ
Destroy completion queue.
Definition: ena.h:250
struct ena_acq acq
Admin completion queue.
Definition: ena.h:579
uint16_t base
Base address.
Definition: edd.h:14
static struct xen_remove_from_physmap * remove
Definition: xenmem.h:39
uint16_t id
Submission queue identifier.
Definition: ena.h:201
struct io_buffer * rx_iobuf[ENA_RX_COUNT]
Receive I/O buffers.
Definition: ena.h:585
Use host memory.
Definition: ena.h:172
Memory is contiguous.
Definition: ena.h:174
Network device operations.
Definition: netdevice.h:213
void netdev_rx(struct net_device *netdev, struct io_buffer *iobuf)
Add packet to receive queue.
Definition: netdevice.c:504
struct device * dev
Underlying hardware device.
Definition: netdevice.h:364
Network device management.
unsigned long physaddr_t
Definition: stdint.h:20
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:354
A named feature.
Definition: features.h:78
struct ena_get_feature_req get_feature
Get feature.
Definition: ena.h:354
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
Get statistics response.
Definition: ena.h:326
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
struct ena_sq sq
Submission queue.
Definition: ena.h:567
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
uint64_t rx_bytes
Receive byte count.
Definition: ena.h:334
uint32_t len
Length.
Definition: ena.h:14
uint16_t len
Length.
Definition: ena.h:464
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
uint16_t id
Request identifier.
Definition: ena.h:466
#define DBGC2(...)
Definition: compiler.h:522
Receive submission queue entry.
Definition: ena.h:416
int(* probe)(struct pci_device *pci)
Probe device.
Definition: pci.h:243
#define ENA_DEVICE_MINE
My device.
Definition: ena.h:323
uint64_t address
Address.
Definition: ena.h:412
void * data
Start of data.
Definition: iobuf.h:48
Receive completion queue entry.
Definition: ena.h:458
uint8_t direction
Direction.
Definition: ena.h:203
Admin completion queue response.
Definition: ena.h:362
#define EIO
Input/output error.
Definition: errno.h:433
#define ENA_RESET_MAX_WAIT_MS
Maximum time to wait for reset.
Definition: ena.h:65
uint16_t count
Number of entries.
Definition: ena.h:22
struct net_device * alloc_etherdev(size_t priv_size)
Allocate Ethernet device.
Definition: ethernet.c:264
uint8_t opcode
Opcode.
Definition: ena.h:78
#define cpu_to_le16(value)
Definition: byteswap.h:106
union ena_acq_rsp * rsp
Responses.
Definition: ena.h:392
uint64_t tx_bytes
Transmit byte count.
Definition: ena.h:330
uint8_t flags
Flags.
Definition: ena.h:422
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:16
void iounmap(volatile const void *io_addr)
Unmap I/O address.
#define ENA_SQE_PHASE
Submission queue ownership phase flag.
Definition: ena.h:432
uint16_t len
Length.
Definition: ena.h:402
uint16_t id
Submission queue identifier.
Definition: ena.h:494
uint8_t scope
Scope.
Definition: ena.h:307
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
static union ena_aq_req * ena_admin_req(struct ena_nic *ena)
Get next available admin queue request.
Definition: ena.c:256
static void ena_set_base(struct ena_nic *ena, unsigned int offset, void *base)
Set queue base address.
Definition: ena.c:135
struct ena_destroy_sq_req destroy_sq
Destroy submission queue.
Definition: ena.h:348
static struct pci_device_id ena_nics[]
ENA PCI device IDs.
Definition: ena.c:1029
struct ena_rx_sqe * rx
Receive submission queue entries.
Definition: ena.h:481
#define ENA_ADMIN_MAX_WAIT_MS
Maximum time to wait for admin requests.
Definition: ena.h:58
An ENA network card.
Definition: ena.h:573
Admin queue request.
Definition: ena.h:342
struct pci_driver ena_driver __pci_driver
ENA PCI driver.
Definition: ena.c:1035
static const char * ena_direction(unsigned int direction)
Get direction name (for debugging)
Definition: ena.c:51
static int ena_destroy_cq(struct ena_nic *ena, struct ena_cq *cq)
Destroy completion queue.
Definition: ena.c:500
uint8_t id
Request identifier.
Definition: ena.h:74
struct ena_aq_header header
Header.
Definition: ena.h:344
size_t max_pkt_len
Maximum packet length.
Definition: netdevice.h:409
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
Amazon ENA network driver.
#define ENA_DESTROY_SQ
Destroy submission queue.
Definition: ena.h:194
unsigned int prod
Producer counter.
Definition: ena.h:386
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
#define le64_to_cpu(value)
Definition: byteswap.h:114
uint64_t rx_packets
Receive packet count.
Definition: ena.h:336
static void ena_clear_caps(struct ena_nic *ena, unsigned int offset)
Clear queue capabilities.
Definition: ena.c:173
#define DBG_LOG
Definition: compiler.h:317
#define ENA_STAT
Device status register.
Definition: ena.h:68
uint8_t hw_addr[MAX_HW_ADDR_LEN]
Hardware address.
Definition: netdevice.h:381
#define NULL
NULL pointer (VOID *)
Definition: Base.h:362
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:669
Receive.
Definition: ena.h:141
String functions.
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:289
uint8_t direction
Direction.
Definition: ena.h:496
void * raw
Raw data.
Definition: ena.h:483
struct ena_create_sq_req create_sq
Create submission queue.
Definition: ena.h:346
static int ena_reset(struct ena_nic *ena)
Reset hardware.
Definition: ena.c:101
static int ena_create_sq(struct ena_nic *ena, struct ena_sq *sq, struct ena_cq *cq)
Create submission queue.
Definition: ena.c:358
unsigned int phase
Phase.
Definition: ena.h:396
uint16_t device
Device ID.
Definition: ena.h:313
void * memset(void *dest, int character, size_t len) __nonnull
A persistent I/O buffer.
Definition: iobuf.h:33