iPXE
ena.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2018 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdint.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <unistd.h>
30 #include <errno.h>
31 #include <byteswap.h>
32 #include <ipxe/netdevice.h>
33 #include <ipxe/ethernet.h>
34 #include <ipxe/if_ether.h>
35 #include <ipxe/iobuf.h>
36 #include <ipxe/malloc.h>
37 #include <ipxe/pci.h>
38 #include <ipxe/pcibridge.h>
39 #include <ipxe/version.h>
40 #include "ena.h"
41 
42 /** @file
43  *
44  * Amazon ENA network driver
45  *
46  */
47 
48 /**
49  * Get direction name (for debugging)
50  *
51  * @v direction Direction
52  * @ret name Direction name
53  */
54 static const char * ena_direction ( unsigned int direction ) {
55 
56  switch ( direction ) {
57  case ENA_SQ_TX: return "TX";
58  case ENA_SQ_RX: return "RX";
59  default: return "<UNKNOWN>";
60  }
61 }
62 
63 /******************************************************************************
64  *
65  * Device reset
66  *
67  ******************************************************************************
68  */
69 
70 /**
71  * Wait for reset operation to be acknowledged
72  *
73  * @v ena ENA device
74  * @v expected Expected reset state
75  * @ret rc Return status code
76  */
77 static int ena_reset_wait ( struct ena_nic *ena, uint32_t expected ) {
78  uint32_t stat;
79  unsigned int i;
80 
81  /* Wait for reset to complete */
82  for ( i = 0 ; i < ENA_RESET_MAX_WAIT_MS ; i++ ) {
83 
84  /* Check if device is ready */
85  stat = readl ( ena->regs + ENA_STAT );
86  if ( ( stat & ENA_STAT_RESET ) == expected )
87  return 0;
88 
89  /* Delay */
90  mdelay ( 1 );
91  }
92 
93  DBGC ( ena, "ENA %p timed out waiting for reset status %#08x "
94  "(got %#08x)\n", ena, expected, stat );
95  return -ETIMEDOUT;
96 }
97 
98 /**
99  * Reset hardware
100  *
101  * @v ena ENA device
102  * @ret rc Return status code
103  */
104 static int ena_reset ( struct ena_nic *ena ) {
105  int rc;
106 
107  /* Trigger reset */
108  writel ( ENA_CTRL_RESET, ( ena->regs + ENA_CTRL ) );
109 
110  /* Wait for reset to take effect */
111  if ( ( rc = ena_reset_wait ( ena, ENA_STAT_RESET ) ) != 0 )
112  return rc;
113 
114  /* Clear reset */
115  writel ( 0, ( ena->regs + ENA_CTRL ) );
116 
117  /* Wait for reset to clear */
118  if ( ( rc = ena_reset_wait ( ena, 0 ) ) != 0 )
119  return rc;
120 
121  return 0;
122 }
123 
124 /******************************************************************************
125  *
126  * Admin queue
127  *
128  ******************************************************************************
129  */
130 
131 /**
132  * Set queue base address
133  *
134  * @v ena ENA device
135  * @v offset Register offset
136  * @v address Base address
137  */
138 static inline void ena_set_base ( struct ena_nic *ena, unsigned int offset,
139  void *base ) {
141 
142  /* Program base address registers */
143  writel ( ( phys & 0xffffffffUL ),
144  ( ena->regs + offset + ENA_BASE_LO ) );
145  if ( sizeof ( phys ) > sizeof ( uint32_t ) ) {
146  writel ( ( ( ( uint64_t ) phys ) >> 32 ),
147  ( ena->regs + offset + ENA_BASE_HI ) );
148  } else {
149  writel ( 0, ( ena->regs + offset + ENA_BASE_HI ) );
150  }
151 }
152 
153 /**
154  * Set queue capabilities
155  *
156  * @v ena ENA device
157  * @v offset Register offset
158  * @v count Number of entries
159  * @v size Size of each entry
160  */
161 static inline __attribute__ (( always_inline )) void
162 ena_set_caps ( struct ena_nic *ena, unsigned int offset, unsigned int count,
163  size_t size ) {
164 
165  /* Program capabilities register */
166  writel ( ENA_CAPS ( count, size ), ( ena->regs + offset ) );
167 }
168 
169 /**
170  * Clear queue capabilities
171  *
172  * @v ena ENA device
173  * @v offset Register offset
174  */
175 static inline __attribute__ (( always_inline )) void
176 ena_clear_caps ( struct ena_nic *ena, unsigned int offset ) {
177 
178  /* Clear capabilities register */
179  writel ( 0, ( ena->regs + offset ) );
180 }
181 
182 /**
183  * Create admin queues
184  *
185  * @v ena ENA device
186  * @ret rc Return status code
187  */
188 static int ena_create_admin ( struct ena_nic *ena ) {
189  size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
190  size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
191  int rc;
192 
193  /* Allocate admin completion queue */
194  ena->acq.rsp = malloc_phys ( acq_len, acq_len );
195  if ( ! ena->acq.rsp ) {
196  rc = -ENOMEM;
197  goto err_alloc_acq;
198  }
199  memset ( ena->acq.rsp, 0, acq_len );
200 
201  /* Allocate admin queue */
202  ena->aq.req = malloc_phys ( aq_len, aq_len );
203  if ( ! ena->aq.req ) {
204  rc = -ENOMEM;
205  goto err_alloc_aq;
206  }
207  memset ( ena->aq.req, 0, aq_len );
208 
209  /* Program queue addresses and capabilities */
210  ena_set_base ( ena, ENA_ACQ_BASE, ena->acq.rsp );
212  sizeof ( ena->acq.rsp[0] ) );
213  ena_set_base ( ena, ENA_AQ_BASE, ena->aq.req );
215  sizeof ( ena->aq.req[0] ) );
216 
217  DBGC ( ena, "ENA %p AQ [%08lx,%08lx) ACQ [%08lx,%08lx)\n",
218  ena, virt_to_phys ( ena->aq.req ),
219  ( virt_to_phys ( ena->aq.req ) + aq_len ),
220  virt_to_phys ( ena->acq.rsp ),
221  ( virt_to_phys ( ena->acq.rsp ) + acq_len ) );
222  return 0;
223 
224  ena_clear_caps ( ena, ENA_AQ_CAPS );
225  ena_clear_caps ( ena, ENA_ACQ_CAPS );
226  free_phys ( ena->aq.req, aq_len );
227  err_alloc_aq:
228  free_phys ( ena->acq.rsp, acq_len );
229  err_alloc_acq:
230  return rc;
231 }
232 
233 /**
234  * Destroy admin queues
235  *
236  * @v ena ENA device
237  */
238 static void ena_destroy_admin ( struct ena_nic *ena ) {
239  size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
240  size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
241 
242  /* Clear queue capabilities */
243  ena_clear_caps ( ena, ENA_AQ_CAPS );
244  ena_clear_caps ( ena, ENA_ACQ_CAPS );
245  wmb();
246 
247  /* Free queues */
248  free_phys ( ena->aq.req, aq_len );
249  free_phys ( ena->acq.rsp, acq_len );
250  DBGC ( ena, "ENA %p AQ and ACQ destroyed\n", ena );
251 }
252 
253 /**
254  * Get next available admin queue request
255  *
256  * @v ena ENA device
257  * @ret req Admin queue request
258  */
259 static union ena_aq_req * ena_admin_req ( struct ena_nic *ena ) {
260  union ena_aq_req *req;
261  unsigned int index;
262 
263  /* Get next request */
264  index = ( ena->aq.prod % ENA_AQ_COUNT );
265  req = &ena->aq.req[index];
266 
267  /* Initialise request */
268  memset ( ( ( ( void * ) req ) + sizeof ( req->header ) ), 0,
269  ( sizeof ( *req ) - sizeof ( req->header ) ) );
270  req->header.id = ena->aq.prod;
271 
272  /* Increment producer counter */
273  ena->aq.prod++;
274 
275  return req;
276 }
277 
278 /**
279  * Issue admin queue request
280  *
281  * @v ena ENA device
282  * @v req Admin queue request
283  * @v rsp Admin queue response to fill in
284  * @ret rc Return status code
285  */
286 static int ena_admin ( struct ena_nic *ena, union ena_aq_req *req,
287  union ena_acq_rsp **rsp ) {
288  unsigned int index;
289  unsigned int i;
290  int rc;
291 
292  /* Locate response */
293  index = ( ena->acq.cons % ENA_ACQ_COUNT );
294  *rsp = &ena->acq.rsp[index];
295 
296  /* Mark request as ready */
297  req->header.flags ^= ENA_AQ_PHASE;
298  wmb();
299  DBGC2 ( ena, "ENA %p admin request %#x:\n",
300  ena, le16_to_cpu ( req->header.id ) );
301  DBGC2_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
302 
303  /* Ring doorbell */
304  writel ( ena->aq.prod, ( ena->regs + ENA_AQ_DB ) );
305 
306  /* Wait for response */
307  for ( i = 0 ; i < ENA_ADMIN_MAX_WAIT_MS ; i++ ) {
308 
309  /* Check for response */
310  if ( ( (*rsp)->header.flags ^ ena->acq.phase ) & ENA_ACQ_PHASE){
311  mdelay ( 1 );
312  continue;
313  }
314  DBGC2 ( ena, "ENA %p admin response %#x:\n",
315  ena, le16_to_cpu ( (*rsp)->header.id ) );
316  DBGC2_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ));
317 
318  /* Increment consumer counter */
319  ena->acq.cons++;
320  if ( ( ena->acq.cons % ENA_ACQ_COUNT ) == 0 )
321  ena->acq.phase ^= ENA_ACQ_PHASE;
322 
323  /* Check command identifier */
324  if ( (*rsp)->header.id != req->header.id ) {
325  DBGC ( ena, "ENA %p admin response %#x mismatch:\n",
326  ena, le16_to_cpu ( (*rsp)->header.id ) );
327  rc = -EILSEQ;
328  goto err;
329  }
330 
331  /* Check status */
332  if ( (*rsp)->header.status != 0 ) {
333  DBGC ( ena, "ENA %p admin response %#x status %d:\n",
334  ena, le16_to_cpu ( (*rsp)->header.id ),
335  (*rsp)->header.status );
336  rc = -EIO;
337  goto err;
338  }
339 
340  /* Success */
341  return 0;
342  }
343 
344  rc = -ETIMEDOUT;
345  DBGC ( ena, "ENA %p timed out waiting for admin request %#x:\n",
346  ena, le16_to_cpu ( req->header.id ) );
347  err:
348  DBGC_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
349  DBGC_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ) );
350  return rc;
351 }
352 
353 /**
354  * Create submission queue
355  *
356  * @v ena ENA device
357  * @v sq Submission queue
358  * @v cq Corresponding completion queue
359  * @ret rc Return status code
360  */
361 static int ena_create_sq ( struct ena_nic *ena, struct ena_sq *sq,
362  struct ena_cq *cq ) {
363  union ena_aq_req *req;
364  union ena_acq_rsp *rsp;
365  unsigned int i;
366  int rc;
367 
368  /* Allocate submission queue entries */
369  sq->sqe.raw = malloc_phys ( sq->len, ENA_ALIGN );
370  if ( ! sq->sqe.raw ) {
371  rc = -ENOMEM;
372  goto err_alloc;
373  }
374  memset ( sq->sqe.raw, 0, sq->len );
375 
376  /* Construct request */
377  req = ena_admin_req ( ena );
378  req->header.opcode = ENA_CREATE_SQ;
379  req->create_sq.direction = sq->direction;
382  req->create_sq.cq_id = cpu_to_le16 ( cq->id );
383  req->create_sq.count = cpu_to_le16 ( sq->count );
384  req->create_sq.address = cpu_to_le64 ( virt_to_bus ( sq->sqe.raw ) );
385 
386  /* Issue request */
387  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
388  goto err_admin;
389 
390  /* Parse response */
391  sq->id = le16_to_cpu ( rsp->create_sq.id );
392  sq->doorbell = le32_to_cpu ( rsp->create_sq.doorbell );
393 
394  /* Reset producer counter and phase */
395  sq->prod = 0;
396  sq->phase = ENA_SQE_PHASE;
397 
398  /* Calculate fill level */
399  sq->fill = sq->max;
400  if ( sq->fill > cq->actual )
401  sq->fill = cq->actual;
402 
403  /* Initialise buffer ID ring */
404  for ( i = 0 ; i < sq->count ; i++ )
405  sq->ids[i] = i;
406 
407  DBGC ( ena, "ENA %p %s SQ%d at [%08lx,%08lx) fill %d db +%04x CQ%d\n",
408  ena, ena_direction ( sq->direction ), sq->id,
409  virt_to_phys ( sq->sqe.raw ),
410  ( virt_to_phys ( sq->sqe.raw ) + sq->len ),
411  sq->fill, sq->doorbell, cq->id );
412  return 0;
413 
414  err_admin:
415  free_phys ( sq->sqe.raw, sq->len );
416  err_alloc:
417  return rc;
418 }
419 
420 /**
421  * Destroy submission queue
422  *
423  * @v ena ENA device
424  * @v sq Submission queue
425  * @ret rc Return status code
426  */
427 static int ena_destroy_sq ( struct ena_nic *ena, struct ena_sq *sq ) {
428  union ena_aq_req *req;
429  union ena_acq_rsp *rsp;
430  int rc;
431 
432  /* Construct request */
433  req = ena_admin_req ( ena );
435  req->destroy_sq.id = cpu_to_le16 ( sq->id );
436  req->destroy_sq.direction = sq->direction;
437 
438  /* Issue request */
439  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
440  return rc;
441 
442  /* Free submission queue entries */
443  free_phys ( sq->sqe.raw, sq->len );
444 
445  DBGC ( ena, "ENA %p %s SQ%d destroyed\n",
446  ena, ena_direction ( sq->direction ), sq->id );
447  return 0;
448 }
449 
450 /**
451  * Create completion queue
452  *
453  * @v ena ENA device
454  * @v cq Completion queue
455  * @ret rc Return status code
456  */
457 static int ena_create_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
458  union ena_aq_req *req;
459  union ena_acq_rsp *rsp;
460  int rc;
461 
462  /* Allocate completion queue entries */
463  cq->cqe.raw = malloc_phys ( cq->len, ENA_ALIGN );
464  if ( ! cq->cqe.raw ) {
465  rc = -ENOMEM;
466  goto err_alloc;
467  }
468  memset ( cq->cqe.raw, 0, cq->len );
469 
470  /* Construct request */
471  req = ena_admin_req ( ena );
472  req->header.opcode = ENA_CREATE_CQ;
473  req->create_cq.size = cq->size;
474  req->create_cq.count = cpu_to_le16 ( cq->requested );
476  req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );
477 
478  /* Issue request */
479  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
480  goto err_admin;
481 
482  /* Parse response */
483  cq->id = le16_to_cpu ( rsp->create_cq.id );
484  cq->actual = le16_to_cpu ( rsp->create_cq.count );
485  cq->doorbell = le32_to_cpu ( rsp->create_cq.doorbell );
486  cq->mask = ( cq->actual - 1 );
487  if ( cq->actual != cq->requested ) {
488  DBGC ( ena, "ENA %p CQ%d requested %d actual %d\n",
489  ena, cq->id, cq->requested, cq->actual );
490  }
491 
492  /* Reset consumer counter and phase */
493  cq->cons = 0;
494  cq->phase = ENA_CQE_PHASE;
495 
496  DBGC ( ena, "ENA %p CQ%d at [%08lx,%08lx) db +%04x\n",
497  ena, cq->id, virt_to_phys ( cq->cqe.raw ),
498  ( virt_to_phys ( cq->cqe.raw ) + cq->len ), cq->doorbell );
499  return 0;
500 
501  err_admin:
502  free_phys ( cq->cqe.raw, cq->len );
503  err_alloc:
504  return rc;
505 }
506 
507 /**
508  * Destroy completion queue
509  *
510  * @v ena ENA device
511  * @v cq Completion queue
512  * @ret rc Return status code
513  */
514 static int ena_destroy_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
515  union ena_aq_req *req;
516  union ena_acq_rsp *rsp;
517  int rc;
518 
519  /* Construct request */
520  req = ena_admin_req ( ena );
522  req->destroy_cq.id = cpu_to_le16 ( cq->id );
523 
524  /* Issue request */
525  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
526  return rc;
527 
528  /* Free completion queue entries */
529  free_phys ( cq->cqe.raw, cq->len );
530 
531  DBGC ( ena, "ENA %p CQ%d destroyed\n", ena, cq->id );
532  return 0;
533 }
534 
535 /**
536  * Create queue pair
537  *
538  * @v ena ENA device
539  * @v qp Queue pair
540  * @ret rc Return status code
541  */
542 static int ena_create_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
543  int rc;
544 
545  /* Create completion queue */
546  if ( ( rc = ena_create_cq ( ena, &qp->cq ) ) != 0 )
547  goto err_create_cq;
548 
549  /* Create submission queue */
550  if ( ( rc = ena_create_sq ( ena, &qp->sq, &qp->cq ) ) != 0 )
551  goto err_create_sq;
552 
553  return 0;
554 
555  ena_destroy_sq ( ena, &qp->sq );
556  err_create_sq:
557  ena_destroy_cq ( ena, &qp->cq );
558  err_create_cq:
559  return rc;
560 }
561 
562 /**
563  * Destroy queue pair
564  *
565  * @v ena ENA device
566  * @v qp Queue pair
567  * @ret rc Return status code
568  */
569 static int ena_destroy_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
570 
571  /* Destroy submission queue */
572  ena_destroy_sq ( ena, &qp->sq );
573 
574  /* Destroy completion queue */
575  ena_destroy_cq ( ena, &qp->cq );
576 
577  return 0;
578 }
579 
580 /**
581  * Get device attributes
582  *
583  * @v netdev Network device
584  * @ret rc Return status code
585  */
587  struct ena_nic *ena = netdev->priv;
588  union ena_aq_req *req;
589  union ena_acq_rsp *rsp;
590  union ena_feature *feature;
591  int rc;
592 
593  /* Construct request */
594  req = ena_admin_req ( ena );
597 
598  /* Issue request */
599  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
600  return rc;
601 
602  /* Parse response */
603  feature = &rsp->get_feature.feature;
604  memcpy ( netdev->hw_addr, feature->device.mac, ETH_ALEN );
605  netdev->max_pkt_len = le32_to_cpu ( feature->device.mtu );
607 
608  DBGC ( ena, "ENA %p MAC %s MTU %zd\n",
609  ena, eth_ntoa ( netdev->hw_addr ), netdev->max_pkt_len );
610  return 0;
611 }
612 
613 /**
614  * Set host attributes
615  *
616  * @v ena ENA device
617  * @ret rc Return status code
618  */
619 static int ena_set_host_attributes ( struct ena_nic *ena ) {
620  union ena_aq_req *req;
621  union ena_acq_rsp *rsp;
622  union ena_feature *feature;
623  int rc;
624 
625  /* Construct request */
626  req = ena_admin_req ( ena );
629  feature = &req->set_feature.feature;
630  feature->host.info = cpu_to_le64 ( virt_to_bus ( ena->info ) );
631 
632  /* Issue request */
633  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
634  return rc;
635 
636  return 0;
637 }
638 
639 /**
640  * Get statistics (for debugging)
641  *
642  * @v ena ENA device
643  * @ret rc Return status code
644  */
645 static int ena_get_stats ( struct ena_nic *ena ) {
646  union ena_aq_req *req;
647  union ena_acq_rsp *rsp;
648  struct ena_get_stats_rsp *stats;
649  int rc;
650 
651  /* Do nothing unless debug messages are enabled */
652  if ( ! DBG_LOG )
653  return 0;
654 
655  /* Construct request */
656  req = ena_admin_req ( ena );
657  req->header.opcode = ENA_GET_STATS;
661 
662  /* Issue request */
663  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
664  return rc;
665 
666  /* Parse response */
667  stats = &rsp->get_stats;
668  DBGC ( ena, "ENA %p TX bytes %#llx packets %#llx\n", ena,
669  ( ( unsigned long long ) le64_to_cpu ( stats->tx_bytes ) ),
670  ( ( unsigned long long ) le64_to_cpu ( stats->tx_packets ) ) );
671  DBGC ( ena, "ENA %p RX bytes %#llx packets %#llx drops %#llx\n", ena,
672  ( ( unsigned long long ) le64_to_cpu ( stats->rx_bytes ) ),
673  ( ( unsigned long long ) le64_to_cpu ( stats->rx_packets ) ),
674  ( ( unsigned long long ) le64_to_cpu ( stats->rx_drops ) ) );
675 
676  return 0;
677 }
678 
679 /******************************************************************************
680  *
681  * Network device interface
682  *
683  ******************************************************************************
684  */
685 
686 /**
687  * Refill receive queue
688  *
689  * @v netdev Network device
690  */
691 static void ena_refill_rx ( struct net_device *netdev ) {
692  struct ena_nic *ena = netdev->priv;
693  struct io_buffer *iobuf;
694  struct ena_rx_sqe *sqe;
696  size_t len = netdev->max_pkt_len;
697  unsigned int refilled = 0;
698  unsigned int index;
699  unsigned int id;
700 
701  /* Refill queue */
702  while ( ( ena->rx.sq.prod - ena->rx.cq.cons ) < ena->rx.sq.fill ) {
703 
704  /* Allocate I/O buffer */
705  iobuf = alloc_iob ( len );
706  if ( ! iobuf ) {
707  /* Wait for next refill */
708  break;
709  }
710 
711  /* Get next submission queue entry and buffer ID */
712  index = ( ena->rx.sq.prod % ENA_RX_COUNT );
713  sqe = &ena->rx.sq.sqe.rx[index];
714  id = ena->rx_ids[index];
715 
716  /* Construct submission queue entry */
717  address = virt_to_bus ( iobuf->data );
718  sqe->len = cpu_to_le16 ( len );
719  sqe->id = cpu_to_le16 ( id );
720  sqe->address = cpu_to_le64 ( address );
721  wmb();
723  ena->rx.sq.phase );
724 
725  /* Increment producer counter */
726  ena->rx.sq.prod++;
727  if ( ( ena->rx.sq.prod % ENA_RX_COUNT ) == 0 )
728  ena->rx.sq.phase ^= ENA_SQE_PHASE;
729 
730  /* Record I/O buffer */
731  assert ( ena->rx_iobuf[id] == NULL );
732  ena->rx_iobuf[id] = iobuf;
733 
734  DBGC2 ( ena, "ENA %p RX %d at [%08llx,%08llx)\n", ena, id,
735  ( ( unsigned long long ) address ),
736  ( ( unsigned long long ) address + len ) );
737  refilled++;
738  }
739 
740  /* Ring doorbell, if applicable */
741  if ( refilled ) {
742  wmb();
743  writel ( ena->rx.sq.prod, ( ena->regs + ena->rx.sq.doorbell ) );
744  }
745 }
746 
747 /**
748  * Discard unused receive I/O buffers
749  *
750  * @v ena ENA device
751  */
752 static void ena_empty_rx ( struct ena_nic *ena ) {
753  unsigned int i;
754 
755  for ( i = 0 ; i < ENA_RX_COUNT ; i++ ) {
756  if ( ena->rx_iobuf[i] )
757  free_iob ( ena->rx_iobuf[i] );
758  ena->rx_iobuf[i] = NULL;
759  }
760 }
761 
762 /**
763  * Open network device
764  *
765  * @v netdev Network device
766  * @ret rc Return status code
767  */
768 static int ena_open ( struct net_device *netdev ) {
769  struct ena_nic *ena = netdev->priv;
770  int rc;
771 
772  /* Create transmit queue pair */
773  if ( ( rc = ena_create_qp ( ena, &ena->tx ) ) != 0 )
774  goto err_create_tx;
775 
776  /* Create receive queue pair */
777  if ( ( rc = ena_create_qp ( ena, &ena->rx ) ) != 0 )
778  goto err_create_rx;
779 
780  /* Refill receive queue */
781  ena_refill_rx ( netdev );
782 
783  return 0;
784 
785  ena_destroy_qp ( ena, &ena->rx );
786  err_create_rx:
787  ena_destroy_qp ( ena, &ena->tx );
788  err_create_tx:
789  return rc;
790 }
791 
792 /**
793  * Close network device
794  *
795  * @v netdev Network device
796  */
797 static void ena_close ( struct net_device *netdev ) {
798  struct ena_nic *ena = netdev->priv;
799 
800  /* Dump statistics (for debugging) */
801  ena_get_stats ( ena );
802 
803  /* Destroy receive queue pair */
804  ena_destroy_qp ( ena, &ena->rx );
805 
806  /* Discard any unused receive buffers */
807  ena_empty_rx ( ena );
808 
809  /* Destroy transmit queue pair */
810  ena_destroy_qp ( ena, &ena->tx );
811 }
812 
813 /**
814  * Transmit packet
815  *
816  * @v netdev Network device
817  * @v iobuf I/O buffer
818  * @ret rc Return status code
819  */
820 static int ena_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
821  struct ena_nic *ena = netdev->priv;
822  struct ena_tx_sqe *sqe;
824  unsigned int index;
825  unsigned int id;
826  size_t len;
827 
828  /* Get next submission queue entry */
829  if ( ( ena->tx.sq.prod - ena->tx.cq.cons ) >= ena->tx.sq.fill ) {
830  DBGC ( ena, "ENA %p out of transmit descriptors\n", ena );
831  return -ENOBUFS;
832  }
833  index = ( ena->tx.sq.prod % ENA_TX_COUNT );
834  sqe = &ena->tx.sq.sqe.tx[index];
835  id = ena->tx_ids[index];
836 
837  /* Construct submission queue entry */
838  address = virt_to_bus ( iobuf->data );
839  len = iob_len ( iobuf );
840  sqe->len = cpu_to_le16 ( len );
841  sqe->id = cpu_to_le16 ( id );
842  sqe->address = cpu_to_le64 ( address );
843  wmb();
845  ena->tx.sq.phase );
846  wmb();
847 
848  /* Increment producer counter */
849  ena->tx.sq.prod++;
850  if ( ( ena->tx.sq.prod % ENA_TX_COUNT ) == 0 )
851  ena->tx.sq.phase ^= ENA_SQE_PHASE;
852 
853  /* Record I/O buffer */
854  assert ( ena->tx_iobuf[id] == NULL );
855  ena->tx_iobuf[id] = iobuf;
856 
857  /* Ring doorbell */
858  writel ( ena->tx.sq.prod, ( ena->regs + ena->tx.sq.doorbell ) );
859 
860  DBGC2 ( ena, "ENA %p TX %d at [%08llx,%08llx)\n", ena, id,
861  ( ( unsigned long long ) address ),
862  ( ( unsigned long long ) address + len ) );
863  return 0;
864 }
865 
866 /**
867  * Poll for completed transmissions
868  *
869  * @v netdev Network device
870  */
871 static void ena_poll_tx ( struct net_device *netdev ) {
872  struct ena_nic *ena = netdev->priv;
873  struct ena_tx_cqe *cqe;
874  struct io_buffer *iobuf;
875  unsigned int index;
876  unsigned int id;
877 
878  /* Check for completed packets */
879  while ( ena->tx.cq.cons != ena->tx.sq.prod ) {
880 
881  /* Get next completion queue entry */
882  index = ( ena->tx.cq.cons & ena->tx.cq.mask );
883  cqe = &ena->tx.cq.cqe.tx[index];
884 
885  /* Stop if completion queue entry is empty */
886  if ( ( cqe->flags ^ ena->tx.cq.phase ) & ENA_CQE_PHASE )
887  return;
888 
889  /* Increment consumer counter */
890  ena->tx.cq.cons++;
891  if ( ! ( ena->tx.cq.cons & ena->tx.cq.mask ) )
892  ena->tx.cq.phase ^= ENA_CQE_PHASE;
893 
894  /* Identify and free buffer ID */
895  id = ENA_TX_CQE_ID ( le16_to_cpu ( cqe->id ) );
896  ena->tx_ids[index] = id;
897 
898  /* Identify I/O buffer */
899  iobuf = ena->tx_iobuf[id];
900  assert ( iobuf != NULL );
901  ena->tx_iobuf[id] = NULL;
902 
903  /* Complete transmit */
904  DBGC2 ( ena, "ENA %p TX %d complete\n", ena, id );
905  netdev_tx_complete ( netdev, iobuf );
906  }
907 }
908 
909 /**
910  * Poll for received packets
911  *
912  * @v netdev Network device
913  */
914 static void ena_poll_rx ( struct net_device *netdev ) {
915  struct ena_nic *ena = netdev->priv;
916  struct ena_rx_cqe *cqe;
917  struct io_buffer *iobuf;
918  unsigned int index;
919  unsigned int id;
920  size_t len;
921 
922  /* Check for received packets */
923  while ( ena->rx.cq.cons != ena->rx.sq.prod ) {
924 
925  /* Get next completion queue entry */
926  index = ( ena->rx.cq.cons & ena->rx.cq.mask );
927  cqe = &ena->rx.cq.cqe.rx[index];
928 
929  /* Stop if completion queue entry is empty */
930  if ( ( cqe->flags ^ ena->rx.cq.phase ) & ENA_CQE_PHASE )
931  return;
932 
933  /* Increment consumer counter */
934  ena->rx.cq.cons++;
935  if ( ! ( ena->rx.cq.cons & ena->rx.cq.mask ) )
936  ena->rx.cq.phase ^= ENA_CQE_PHASE;
937 
938  /* Identify and free buffer ID */
939  id = le16_to_cpu ( cqe->id );
940  ena->rx_ids[index] = id;
941 
942  /* Populate I/O buffer */
943  iobuf = ena->rx_iobuf[id];
944  assert ( iobuf != NULL );
945  ena->rx_iobuf[id] = NULL;
946  len = le16_to_cpu ( cqe->len );
947  iob_put ( iobuf, len );
948 
949  /* Hand off to network stack */
950  DBGC2 ( ena, "ENA %p RX %d complete (length %zd)\n",
951  ena, id, len );
952  netdev_rx ( netdev, iobuf );
953  }
954 }
955 
956 /**
957  * Poll for completed and received packets
958  *
959  * @v netdev Network device
960  */
961 static void ena_poll ( struct net_device *netdev ) {
962 
963  /* Poll for transmit completions */
964  ena_poll_tx ( netdev );
965 
966  /* Poll for receive completions */
967  ena_poll_rx ( netdev );
968 
969  /* Refill receive ring */
970  ena_refill_rx ( netdev );
971 }
972 
973 /** ENA network device operations */
975  .open = ena_open,
976  .close = ena_close,
977  .transmit = ena_transmit,
978  .poll = ena_poll,
979 };
980 
981 /******************************************************************************
982  *
983  * PCI interface
984  *
985  ******************************************************************************
986  */
987 
988 /**
989  * Assign memory BAR
990  *
991  * @v ena ENA device
992  * @v pci PCI device
993  * @ret rc Return status code
994  *
995  * Some BIOSes in AWS EC2 are observed to fail to assign a base
996  * address to the ENA device. The device is the only device behind
997  * its bridge, and the BIOS does assign a memory window to the bridge.
998  * We therefore place the device at the start of the memory window.
999  */
1000 static int ena_membase ( struct ena_nic *ena, struct pci_device *pci ) {
1001  struct pci_bridge *bridge;
1002 
1003  /* Locate PCI bridge */
1004  bridge = pcibridge_find ( pci );
1005  if ( ! bridge ) {
1006  DBGC ( ena, "ENA %p found no PCI bridge\n", ena );
1007  return -ENOTCONN;
1008  }
1009 
1010  /* Sanity check */
1011  if ( PCI_SLOT ( pci->busdevfn ) || PCI_FUNC ( pci->busdevfn ) ) {
1012  DBGC ( ena, "ENA %p at " PCI_FMT " may not be only device "
1013  "on bus\n", ena, PCI_ARGS ( pci ) );
1014  return -ENOTSUP;
1015  }
1016 
1017  /* Place device at start of memory window */
1019  pci->membase = bridge->membase;
1020  DBGC ( ena, "ENA %p at " PCI_FMT " claiming bridge " PCI_FMT " mem "
1021  "%08x\n", ena, PCI_ARGS ( pci ), PCI_ARGS ( bridge->pci ),
1022  bridge->membase );
1023 
1024  return 0;
1025 }
1026 
1027 /**
1028  * Probe PCI device
1029  *
1030  * @v pci PCI device
1031  * @ret rc Return status code
1032  */
1033 static int ena_probe ( struct pci_device *pci ) {
1034  struct net_device *netdev;
1035  struct ena_nic *ena;
1036  struct ena_host_info *info;
1037  int rc;
1038 
1039  /* Allocate and initialise net device */
1040  netdev = alloc_etherdev ( sizeof ( *ena ) );
1041  if ( ! netdev ) {
1042  rc = -ENOMEM;
1043  goto err_alloc;
1044  }
1046  ena = netdev->priv;
1047  pci_set_drvdata ( pci, netdev );
1048  netdev->dev = &pci->dev;
1049  memset ( ena, 0, sizeof ( *ena ) );
1050  ena->acq.phase = ENA_ACQ_PHASE;
1051  ena_cq_init ( &ena->tx.cq, ENA_TX_COUNT,
1052  sizeof ( ena->tx.cq.cqe.tx[0] ) );
1053  ena_sq_init ( &ena->tx.sq, ENA_SQ_TX, ENA_TX_COUNT, ENA_TX_COUNT,
1054  sizeof ( ena->tx.sq.sqe.tx[0] ), ena->tx_ids );
1055  ena_cq_init ( &ena->rx.cq, ENA_RX_COUNT,
1056  sizeof ( ena->rx.cq.cqe.rx[0] ) );
1057  ena_sq_init ( &ena->rx.sq, ENA_SQ_RX, ENA_RX_COUNT, ENA_RX_FILL,
1058  sizeof ( ena->rx.sq.sqe.rx[0] ), ena->rx_ids );
1059 
1060  /* Fix up PCI device */
1061  adjust_pci_device ( pci );
1062 
1063  /* Fix up PCI BAR if left unassigned by BIOS */
1064  if ( ( ! pci->membase ) && ( ( rc = ena_membase ( ena, pci ) ) != 0 ) )
1065  goto err_membase;
1066 
1067  /* Map registers */
1068  ena->regs = pci_ioremap ( pci, pci->membase, ENA_BAR_SIZE );
1069  if ( ! ena->regs ) {
1070  rc = -ENODEV;
1071  goto err_ioremap;
1072  }
1073 
1074  /* Allocate and initialise host info */
1076  if ( ! info ) {
1077  rc = -ENOMEM;
1078  goto err_info;
1079  }
1080  ena->info = info;
1081  memset ( info, 0, PAGE_SIZE );
1083  snprintf ( info->dist_str, sizeof ( info->dist_str ), "%s",
1085  snprintf ( info->kernel_str, sizeof ( info->kernel_str ), "%s",
1086  product_version );
1089  info->busdevfn = cpu_to_le16 ( pci->busdevfn );
1090  DBGC2 ( ena, "ENA %p host info:\n", ena );
1091  DBGC2_HDA ( ena, virt_to_phys ( info ), info, sizeof ( *info ) );
1092 
1093  /* Reset the NIC */
1094  if ( ( rc = ena_reset ( ena ) ) != 0 )
1095  goto err_reset;
1096 
1097  /* Create admin queues */
1098  if ( ( rc = ena_create_admin ( ena ) ) != 0 )
1099  goto err_create_admin;
1100 
1101  /* Set host attributes */
1102  if ( ( rc = ena_set_host_attributes ( ena ) ) != 0 )
1103  goto err_set_host_attributes;
1104 
1105  /* Fetch MAC address */
1106  if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
1107  goto err_get_device_attributes;
1108 
1109  /* Register network device */
1110  if ( ( rc = register_netdev ( netdev ) ) != 0 )
1111  goto err_register_netdev;
1112 
1113  /* Mark as link up, since we have no way to test link state on
1114  * this hardware.
1115  */
1116  netdev_link_up ( netdev );
1117 
1118  return 0;
1119 
1121  err_register_netdev:
1122  err_get_device_attributes:
1123  err_set_host_attributes:
1124  ena_destroy_admin ( ena );
1125  err_create_admin:
1126  ena_reset ( ena );
1127  err_reset:
1128  free_phys ( ena->info, PAGE_SIZE );
1129  err_info:
1130  iounmap ( ena->regs );
1131  err_ioremap:
1132  err_membase:
1133  netdev_nullify ( netdev );
1134  netdev_put ( netdev );
1135  err_alloc:
1136  return rc;
1137 }
1138 
1139 /**
1140  * Remove PCI device
1141  *
1142  * @v pci PCI device
1143  */
1144 static void ena_remove ( struct pci_device *pci ) {
1145  struct net_device *netdev = pci_get_drvdata ( pci );
1146  struct ena_nic *ena = netdev->priv;
1147 
1148  /* Unregister network device */
1150 
1151  /* Destroy admin queues */
1152  ena_destroy_admin ( ena );
1153 
1154  /* Reset card */
1155  ena_reset ( ena );
1156 
1157  /* Free host info */
1158  free_phys ( ena->info, PAGE_SIZE );
1159 
1160  /* Free network device */
1161  iounmap ( ena->regs );
1162  netdev_nullify ( netdev );
1163  netdev_put ( netdev );
1164 }
1165 
1166 /** ENA PCI device IDs */
1167 static struct pci_device_id ena_nics[] = {
1168  PCI_ROM ( 0x1d0f, 0xec20, "ena-vf", "ENA VF", 0 ),
1169  PCI_ROM ( 0x1d0f, 0xec21, "ena-vf-llq", "ENA VF (LLQ)", 0 ),
1170 };
1171 
1172 /** ENA PCI driver */
1173 struct pci_driver ena_driver __pci_driver = {
1174  .ids = ena_nics,
1175  .id_count = ( sizeof ( ena_nics ) / sizeof ( ena_nics[0] ) ),
1176  .probe = ena_probe,
1177  .remove = ena_remove,
1178 };
const char product_short_name[]
Product short name string.
Definition: version.c:76
#define PCI_FUNC(busdevfn)
Definition: pci.h:279
struct ena_aq aq
Admin queue.
Definition: ena.h:702
Queue pair.
Definition: ena.h:688
static void ena_refill_rx(struct net_device *netdev)
Refill receive queue.
Definition: ena.c:691
#define __attribute__(x)
Definition: compiler.h:10
#define ENA_ACQ_COUNT
Number of admin completion queue entries.
Definition: ena.h:25
struct ena_tx_sqe * tx
Transmit submission queue entries.
Definition: ena.h:592
static void ena_destroy_admin(struct ena_nic *ena)
Destroy admin queues.
Definition: ena.c:238
struct ena_qp rx
Receive queue.
Definition: ena.h:708
unsigned long membase
Memory base.
Definition: pci.h:215
#define ENA_CTRL
Device control register.
Definition: ena.h:64
uint8_t type
Type.
Definition: ena.h:413
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static void ena_remove(struct pci_device *pci)
Remove PCI device.
Definition: ena.c:1144
static void netdev_tx_complete(struct net_device *netdev, struct io_buffer *iobuf)
Complete network transmission.
Definition: netdevice.h:758
wmb()
void * raw
Raw data.
Definition: ena.h:650
#define ENA_BASE_HI
Base address high register offset.
Definition: ena.h:40
Host information.
Definition: ena.h:147
#define iob_put(iobuf, len)
Definition: iobuf.h:120
u32 info
Definition: ar9003_mac.h:67
#define ENA_RX_COUNT
Number of receive queue entries.
Definition: ena.h:31
#define ENA_AQ_COUNT
Number of admin queue entries.
Definition: ena.h:22
A PCI driver.
Definition: pci.h:245
uint8_t rx_ids[ENA_RX_COUNT]
Receive buffer IDs.
Definition: ena.h:714
static void ena_poll_tx(struct net_device *netdev)
Poll for completed transmissions.
Definition: ena.c:871
uint8_t mask
Actual number of entries minus one.
Definition: ena.h:669
size_t len
Total length of entries.
Definition: ena.h:603
#define ENA_HOST_INFO_TYPE_LINUX
Linux operating system type.
Definition: ena.h:181
unsigned int doorbell
Doorbell register offset.
Definition: ena.h:601
uint8_t count
Number of entries.
Definition: ena.h:613
#define le32_to_cpu(value)
Definition: byteswap.h:113
int(* open)(struct net_device *netdev)
Open network device.
Definition: netdevice.h:222
#define ENA_GET_FEATURE
Get feature.
Definition: ena.h:356
#define ENA_HOST_ATTRIBUTES
Host attributes.
Definition: ena.h:134
Error codes.
#define ENA_CREATE_CQ
Create completion queue.
Definition: ena.h:294
#define ENA_ACQ_BASE
Admin completion queue base address register.
Definition: ena.h:52
uint8_t * ids
Buffer IDs.
Definition: ena.h:599
unsigned int prod
Producer counter.
Definition: ena.h:605
uint16_t id
Completion queue identifier.
Definition: ena.h:344
I/O buffers.
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:247
uint64_t address
Base address.
Definition: ena.h:24
struct ena_tx_cqe * tx
Transmit completion queue entries.
Definition: ena.h:646
size_t mtu
Maximum transmission unit length.
Definition: netdevice.h:415
static int ena_get_device_attributes(struct net_device *netdev)
Get device attributes.
Definition: ena.c:586
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
static int ena_get_stats(struct ena_nic *ena)
Get statistics (for debugging)
Definition: ena.c:645
Transmit submission queue entry.
Definition: ena.h:510
struct ena_qp tx
Transmit queue.
Definition: ena.h:706
#define DBGC(...)
Definition: compiler.h:505
struct ena_set_feature_req set_feature
Set feature.
Definition: ena.h:464
uint16_t count
Number of entries.
Definition: ena.h:239
union ena_sq::@39 sqe
Entries.
#define ENA_SQE_CPL
Request completion.
Definition: ena.h:551
Transmit completion queue entry.
Definition: ena.h:554
unsigned long long uint64_t
Definition: stdint.h:13
uint8_t id
Request identifier.
Definition: ena.h:520
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
static int ena_open(struct net_device *netdev)
Open network device.
Definition: ena.c:768
#define PCI_BASE_ADDRESS_0
Definition: pci.h:62
union ena_cq::@40 cqe
Entries.
#define cpu_to_le64(value)
Definition: byteswap.h:108
uint16_t count
Number of entries.
Definition: ena.h:305
uint8_t direction
Direction.
Definition: ena.h:14
uint64_t rx_drops
Receive drop count.
Definition: ena.h:446
uint16_t len
Length.
Definition: ena.h:528
#define ENA_AQ_BASE
Admin queue base address register.
Definition: ena.h:46
#define ENA_GET_STATS
Get statistics.
Definition: ena.h:404
#define PAGE_SIZE
Page size.
Definition: io.h:27
static void ena_close(struct net_device *netdev)
Close network device.
Definition: ena.c:797
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:663
static int ena_create_qp(struct ena_nic *ena, struct ena_qp *qp)
Create queue pair.
Definition: ena.c:542
unsigned int phase
Phase.
Definition: ena.h:659
#define ENA_ACQ_CAPS
Admin completion queue capabilities register.
Definition: ena.h:55
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:154
struct io_buffer * alloc_iob(size_t len)
Allocate I/O buffer.
Definition: iobuf.c:129
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
uint64_t address
Address.
Definition: ena.h:538
struct device dev
Generic device.
Definition: pci.h:208
#define ENA_CTRL_RESET
Reset.
Definition: ena.h:65
#define ENA_TX_COUNT
Number of transmit queue entries.
Definition: ena.h:28
uint64_t address
Base address.
Definition: ena.h:309
#define ENOTSUP
Operation not supported.
Definition: errno.h:589
unsigned int doorbell
Doorbell register offset.
Definition: ena.h:653
void * regs
Registers.
Definition: ena.h:698
Dynamic memory allocation.
uint8_t tx_ids[ENA_TX_COUNT]
Transmit buffer IDs.
Definition: ena.h:710
union ena_feature feature
Feature.
Definition: ena.h:14
Completion queue.
Definition: ena.h:642
static int ena_reset_wait(struct ena_nic *ena, uint32_t expected)
Wait for reset operation to be acknowledged.
Definition: ena.c:77
Submission queue.
Definition: ena.h:588
static int ena_set_host_attributes(struct ena_nic *ena)
Set host attributes.
Definition: ena.c:619
static int ena_destroy_qp(struct ena_nic *ena, struct ena_qp *qp)
Destroy queue pair.
Definition: ena.c:569
static void ena_empty_rx(struct ena_nic *ena)
Discard unused receive I/O buffers.
Definition: ena.c:752
Feature.
Definition: ena.h:208
static void netdev_init(struct net_device *netdev, struct net_device_operations *op)
Initialise a network device.
Definition: netdevice.h:510
#define ENA_DEVICE_ATTRIBUTES
Device attributes.
Definition: ena.h:109
static int ena_destroy_sq(struct ena_nic *ena, struct ena_sq *sq)
Destroy submission queue.
Definition: ena.c:427
size_t len
Total length of entries.
Definition: ena.h:655
#define ENA_CAPS(count, size)
Capability register value.
Definition: ena.h:43
uint16_t id
Request identifier.
Definition: ena.h:556
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:357
#define ENOMEM
Not enough space.
Definition: errno.h:534
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static signed char phys[4]
Definition: epic100.c:88
uint64_t tx_packets
Transmit packet count.
Definition: ena.h:440
#define EILSEQ
Illegal byte sequence.
Definition: errno.h:413
uint16_t id
Request identifier.
Definition: ena.h:534
uint8_t id
Feature identifier.
Definition: ena.h:369
#define ENA_BASE_LO
Base address low register offset.
Definition: ena.h:37
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define ETH_HLEN
Definition: if_ether.h:9
#define ENA_STAT_RESET
Reset in progress.
Definition: ena.h:72
static void ena_set_caps(struct ena_nic *ena, unsigned int offset, unsigned int count, size_t size)
Set queue capabilities.
Definition: ena.c:162
struct pci_bridge * pcibridge_find(struct pci_device *pci)
Find bridge attached to a PCI device.
Definition: pcibridge.c:48
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
static void netdev_put(struct net_device *netdev)
Drop reference to network device.
Definition: netdevice.h:567
Ethernet protocol.
#define ENA_TX_CQE_ID(id)
Transmit completion request identifier.
Definition: ena.h:568
union ena_aq_req * req
Requests.
Definition: ena.h:494
Transmit.
Definition: ena.h:218
uint64_t address
Base address.
Definition: ena.h:241
void * priv
Driver private data.
Definition: netdevice.h:431
static int ena_transmit(struct net_device *netdev, struct io_buffer *iobuf)
Transmit packet.
Definition: ena.c:820
#define DBGC_HDA(...)
Definition: compiler.h:506
uint16_t cq_id
Completion queue identifier.
Definition: ena.h:237
unsigned int cons
Consumer counter.
Definition: ena.h:504
static void netdev_link_up(struct net_device *netdev)
Mark network device as having link up.
Definition: netdevice.h:780
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
uint8_t flags
Flags.
Definition: ena.h:560
struct pci_device * pci
PCI device.
Definition: pcibridge.h:19
unsigned int cons
Consumer counter.
Definition: ena.h:657
uint16_t policy
Policy.
Definition: ena.h:235
#define ENA_CQE_PHASE
Completion queue ownership phase flag.
Definition: ena.h:585
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
static struct net_device * netdev
Definition: gdbudp.c:52
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:303
uint8_t flags
Flags.
Definition: ena.h:83
static int ena_probe(struct pci_device *pci)
Probe PCI device.
Definition: ena.c:1033
#define ENA_STATS_TYPE_BASIC
Basic statistics.
Definition: ena.h:425
#define ENA_SQE_FIRST
This is the first descriptor.
Definition: ena.h:545
void unregister_netdev(struct net_device *netdev)
Unregister network device.
Definition: netdevice.c:899
uint8_t id
Feature identifier.
Definition: ena.h:396
unsigned int phase
Phase.
Definition: ena.h:607
struct ena_get_stats_req get_stats
Get statistics.
Definition: ena.h:466
static struct net_device_operations ena_operations
ENA network device operations.
Definition: ena.c:974
#define cpu_to_le32(value)
Definition: byteswap.h:107
uint8_t id
Request identifier.
Definition: ena.h:12
struct ena_create_cq_req create_cq
Create completion queue.
Definition: ena.h:458
struct ena_destroy_cq_req destroy_cq
Destroy completion queue.
Definition: ena.h:460
#define ENOTCONN
The socket is not connected.
Definition: errno.h:569
#define DBGC2_HDA(...)
Definition: compiler.h:523
uint8_t direction
Direction.
Definition: ena.h:231
#define ENA_MSIX_NONE
Empty MSI-X vector.
Definition: ena.h:318
uint64_t rsp
Definition: librm.h:267
static void ena_poll(struct net_device *netdev)
Poll for completed and received packets.
Definition: ena.c:961
struct ena_rx_cqe * rx
Receive completion queue entries.
Definition: ena.h:648
struct ena_cq cq
Completion queue.
Definition: ena.h:692
uint8_t flags
Flags.
Definition: ena.h:575
A PCI-to-PCI bridge.
Definition: pcibridge.h:17
#define ENA_HOST_INFO_VERSION_WTF
Driver version.
Definition: ena.h:202
uint32_t vector
MSI-X vector.
Definition: ena.h:307
static int ena_create_admin(struct ena_nic *ena)
Create admin queues.
Definition: ena.c:188
uint8_t flags
Flags.
Definition: ena.h:516
#define ENA_ACQ_PHASE
Admin completion queue ownership phase flag.
Definition: ena.h:106
#define PCI_FMT
PCI device debug message format.
Definition: pci.h:305
#define PCI_SLOT(busdevfn)
Definition: pci.h:278
static int ena_create_cq(struct ena_nic *ena, struct ena_cq *cq)
Create completion queue.
Definition: ena.c:457
PCI bus.
A PCI device.
Definition: pci.h:206
int register_netdev(struct net_device *netdev)
Register network device.
Definition: netdevice.c:722
#define ENA_SQE_LAST
This is the last descriptor.
Definition: ena.h:548
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
const char * eth_ntoa(const void *ll_addr)
Transcribe Ethernet address.
Definition: ethernet.c:175
#define ENA_AQ_DB
Admin queue doorbell register.
Definition: ena.h:58
#define ENA_AQ_CAPS
Admin queue capabilities register.
Definition: ena.h:49
static int ena_admin(struct ena_nic *ena, union ena_aq_req *req, union ena_acq_rsp **rsp)
Issue admin queue request.
Definition: ena.c:286
A network device.
Definition: netdevice.h:352
#define ENODEV
No such device.
Definition: errno.h:509
#define ENA_ALIGN
Queue alignment.
Definition: ena.h:19
static void netdev_nullify(struct net_device *netdev)
Stop using a network device.
Definition: netdevice.h:523
uint16_t id
Completion queue identifier.
Definition: ena.h:661
const char product_version[]
Product version string.
Definition: version.c:70
#define ENA_CREATE_SQ
Create submission queue.
Definition: ena.h:224
#define ENA_AQ_PHASE
Admin queue ownership phase flag.
Definition: ena.h:87
#define ENA_BAR_SIZE
BAR size.
Definition: ena.h:16
uint8_t actual
Actual number of entries.
Definition: ena.h:667
static void ena_poll_rx(struct net_device *netdev)
Poll for received packets.
Definition: ena.c:914
#define ENA_STATS_SCOPE_ETH
Ethernet statistics.
Definition: ena.h:428
#define ETH_ALEN
Definition: if_ether.h:8
A PCI device ID list entry.
Definition: pci.h:170
Version number.
#define le16_to_cpu(value)
Definition: byteswap.h:112
unsigned int uint32_t
Definition: stdint.h:12
uint8_t requested
Requested number of entries.
Definition: ena.h:665
#define ENA_DESTROY_CQ
Destroy completion queue.
Definition: ena.h:337
struct ena_acq acq
Admin completion queue.
Definition: ena.h:704
uint16_t base
Base address.
Definition: edd.h:14
static struct xen_remove_from_physmap * remove
Definition: xenmem.h:39
uint16_t id
Submission queue identifier.
Definition: ena.h:280
struct io_buffer * rx_iobuf[ENA_RX_COUNT]
Receive I/O buffers, indexed by buffer ID.
Definition: ena.h:716
Use host memory.
Definition: ena.h:251
Memory is contiguous.
Definition: ena.h:253
Network device operations.
Definition: netdevice.h:213
void netdev_rx(struct net_device *netdev, struct io_buffer *iobuf)
Add packet to receive queue.
Definition: netdevice.c:504
struct device * dev
Underlying hardware device.
Definition: netdevice.h:364
Network device management.
unsigned long physaddr_t
Definition: stdint.h:20
uint8_t max
Maximum fill level.
Definition: ena.h:615
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:367
A named feature.
Definition: features.h:78
struct ena_get_feature_req get_feature
Get feature.
Definition: ena.h:462
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
Get statistics response.
Definition: ena.h:434
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
uint32_t busdevfn
Segment, bus, device, and function (bus:dev.fn) number.
Definition: pci.h:231
struct ena_host_info * info
Host info.
Definition: ena.h:700
#define ENA_SET_FEATURE
Set feature.
Definition: ena.h:383
struct ena_sq sq
Submission queue.
Definition: ena.h:690
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
uint64_t rx_bytes
Receive byte count.
Definition: ena.h:442
uint32_t len
Length.
Definition: ena.h:14
uint16_t len
Length.
Definition: ena.h:577
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
uint16_t id
Request identifier.
Definition: ena.h:579
#define DBGC2(...)
Definition: compiler.h:522
Receive submission queue entry.
Definition: ena.h:526
int(* probe)(struct pci_device *pci)
Probe device.
Definition: pci.h:258
#define ENA_DEVICE_MINE
My device.
Definition: ena.h:431
uint64_t address
Address.
Definition: ena.h:522
void * data
Start of data.
Definition: iobuf.h:48
Receive completion queue entry.
Definition: ena.h:571
uint8_t direction
Direction.
Definition: ena.h:282
Admin completion queue response.
Definition: ena.h:472
#define EIO
Input/output error.
Definition: errno.h:433
PCI-to-PCI bridge.
#define ENA_RESET_MAX_WAIT_MS
Maximum time to wait for reset.
Definition: ena.h:68
uint16_t count
Number of entries.
Definition: ena.h:22
struct net_device * alloc_etherdev(size_t priv_size)
Allocate Ethernet device.
Definition: ethernet.c:264
#define PCI_ARGS(pci)
PCI device debug message arguments.
Definition: pci.h:308
uint8_t opcode
Opcode.
Definition: ena.h:81
int pci_write_config_dword(struct pci_device *pci, unsigned int where, uint32_t value)
Write 32-bit dword to PCI configuration space.
#define cpu_to_le16(value)
Definition: byteswap.h:106
union ena_acq_rsp * rsp
Responses.
Definition: ena.h:502
uint64_t tx_bytes
Transmit byte count.
Definition: ena.h:438
uint8_t flags
Flags.
Definition: ena.h:532
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:16
void iounmap(volatile const void *io_addr)
Unmap I/O address.
#define ENA_SQE_PHASE
Submission queue ownership phase flag.
Definition: ena.h:542
uint16_t len
Length.
Definition: ena.h:512
union ena_feature feature
Feature.
Definition: ena.h:400
uint16_t id
Submission queue identifier.
Definition: ena.h:609
uint8_t scope
Scope.
Definition: ena.h:415
int snprintf(char *buf, size_t size, const char *fmt,...)
Write a formatted string to a buffer.
Definition: vsprintf.c:382
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
static union ena_aq_req * ena_admin_req(struct ena_nic *ena)
Get next available admin queue request.
Definition: ena.c:259
static void ena_set_base(struct ena_nic *ena, unsigned int offset, void *base)
Set queue base address.
Definition: ena.c:138
struct ena_destroy_sq_req destroy_sq
Destroy submission queue.
Definition: ena.h:456
static struct pci_device_id ena_nics[]
ENA PCI device IDs.
Definition: ena.c:1167
struct ena_rx_sqe * rx
Receive submission queue entries.
Definition: ena.h:594
uint8_t fill
Fill level (limited to completion queue size)
Definition: ena.h:617
#define ENA_ADMIN_MAX_WAIT_MS
Maximum time to wait for admin requests.
Definition: ena.h:61
An ENA network card.
Definition: ena.h:696
Admin queue request.
Definition: ena.h:450
struct pci_driver ena_driver __pci_driver
ENA PCI driver.
Definition: ena.c:1173
static const char * ena_direction(unsigned int direction)
Get direction name (for debugging)
Definition: ena.c:54
const char product_name[]
Product name string.
Definition: version.c:73
static int ena_destroy_cq(struct ena_nic *ena, struct ena_cq *cq)
Destroy completion queue.
Definition: ena.c:514
uint8_t id
Request identifier.
Definition: ena.h:77
struct ena_aq_header header
Header.
Definition: ena.h:452
size_t max_pkt_len
Maximum packet length.
Definition: netdevice.h:409
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
Amazon ENA network driver.
#define ENA_DESTROY_SQ
Destroy submission queue.
Definition: ena.h:273
unsigned int prod
Producer counter.
Definition: ena.h:496
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
#define le64_to_cpu(value)
Definition: byteswap.h:114
struct io_buffer * tx_iobuf[ENA_TX_COUNT]
Transmit I/O buffers, indexed by buffer ID.
Definition: ena.h:712
uint64_t rx_packets
Receive packet count.
Definition: ena.h:444
static void ena_clear_caps(struct ena_nic *ena, unsigned int offset)
Clear queue capabilities.
Definition: ena.c:176
#define DBG_LOG
Definition: compiler.h:317
#define ENA_STAT
Device status register.
Definition: ena.h:71
static int ena_membase(struct ena_nic *ena, struct pci_device *pci)
Assign memory BAR.
Definition: ena.c:1000
uint8_t hw_addr[MAX_HW_ADDR_LEN]
Hardware address.
Definition: netdevice.h:381
#define NULL
NULL pointer (VOID *)
Definition: Base.h:362
#define ENA_HOST_INFO_SPEC_2_0
ENA specification version.
Definition: ena.h:205
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:669
Receive.
Definition: ena.h:220
String functions.
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:301
uint8_t direction
Direction.
Definition: ena.h:611
#define ENA_RX_FILL
Receive queue maximum fill level.
Definition: ena.h:34
void * raw
Raw data.
Definition: ena.h:596
struct ena_create_sq_req create_sq
Create submission queue.
Definition: ena.h:454
static int ena_reset(struct ena_nic *ena)
Reset hardware.
Definition: ena.c:104
static int ena_create_sq(struct ena_nic *ena, struct ena_sq *sq, struct ena_cq *cq)
Create submission queue.
Definition: ena.c:361
unsigned int phase
Phase.
Definition: ena.h:506
uint16_t device
Device ID.
Definition: ena.h:421
uint32_t membase
Memory base.
Definition: pcibridge.h:34
void * memset(void *dest, int character, size_t len) __nonnull
A persistent I/O buffer.
Definition: iobuf.h:33