iPXE
ena.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2018 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdint.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <unistd.h>
30 #include <errno.h>
31 #include <byteswap.h>
32 #include <ipxe/netdevice.h>
33 #include <ipxe/ethernet.h>
34 #include <ipxe/if_ether.h>
35 #include <ipxe/iobuf.h>
36 #include <ipxe/malloc.h>
37 #include <ipxe/pci.h>
38 #include <ipxe/pcibridge.h>
39 #include <ipxe/version.h>
40 #include "ena.h"
41 
42 /** @file
43  *
44  * Amazon ENA network driver
45  *
46  */
47 
48 /**
49  * Get direction name (for debugging)
50  *
51  * @v direction Direction
52  * @ret name Direction name
53  */
54 static const char * ena_direction ( unsigned int direction ) {
55 
56  switch ( direction ) {
57  case ENA_SQ_TX: return "TX";
58  case ENA_SQ_RX: return "RX";
59  default: return "<UNKNOWN>";
60  }
61 }
62 
63 /******************************************************************************
64  *
65  * Device reset
66  *
67  ******************************************************************************
68  */
69 
70 /**
71  * Wait for reset operation to be acknowledged
72  *
73  * @v ena ENA device
74  * @v expected Expected reset state
75  * @ret rc Return status code
76  */
77 static int ena_reset_wait ( struct ena_nic *ena, uint32_t expected ) {
78  uint32_t stat;
79  unsigned int i;
80 
81  /* Wait for reset to complete */
82  for ( i = 0 ; i < ENA_RESET_MAX_WAIT_MS ; i++ ) {
83 
84  /* Check if device is ready */
85  stat = readl ( ena->regs + ENA_STAT );
86  if ( ( stat & ENA_STAT_RESET ) == expected )
87  return 0;
88 
89  /* Delay */
90  mdelay ( 1 );
91  }
92 
93  DBGC ( ena, "ENA %p timed out waiting for reset status %#08x "
94  "(got %#08x)\n", ena, expected, stat );
95  return -ETIMEDOUT;
96 }
97 
98 /**
99  * Reset hardware
100  *
101  * @v ena ENA device
102  * @ret rc Return status code
103  */
104 static int ena_reset ( struct ena_nic *ena ) {
105  int rc;
106 
107  /* Trigger reset */
108  writel ( ENA_CTRL_RESET, ( ena->regs + ENA_CTRL ) );
109 
110  /* Wait for reset to take effect */
111  if ( ( rc = ena_reset_wait ( ena, ENA_STAT_RESET ) ) != 0 )
112  return rc;
113 
114  /* Clear reset */
115  writel ( 0, ( ena->regs + ENA_CTRL ) );
116 
117  /* Wait for reset to clear */
118  if ( ( rc = ena_reset_wait ( ena, 0 ) ) != 0 )
119  return rc;
120 
121  return 0;
122 }
123 
124 /******************************************************************************
125  *
126  * Admin queue
127  *
128  ******************************************************************************
129  */
130 
131 /**
132  * Set queue base address
133  *
134  * @v ena ENA device
135  * @v offset Register offset
136  * @v address Base address
137  */
138 static inline void ena_set_base ( struct ena_nic *ena, unsigned int offset,
139  void *base ) {
141 
142  /* Program base address registers */
143  writel ( ( phys & 0xffffffffUL ),
144  ( ena->regs + offset + ENA_BASE_LO ) );
145  if ( sizeof ( phys ) > sizeof ( uint32_t ) ) {
146  writel ( ( ( ( uint64_t ) phys ) >> 32 ),
147  ( ena->regs + offset + ENA_BASE_HI ) );
148  } else {
149  writel ( 0, ( ena->regs + offset + ENA_BASE_HI ) );
150  }
151 }
152 
153 /**
154  * Set queue capabilities
155  *
156  * @v ena ENA device
157  * @v offset Register offset
158  * @v count Number of entries
159  * @v size Size of each entry
160  */
161 static inline __attribute__ (( always_inline )) void
162 ena_set_caps ( struct ena_nic *ena, unsigned int offset, unsigned int count,
163  size_t size ) {
164 
165  /* Program capabilities register */
166  writel ( ENA_CAPS ( count, size ), ( ena->regs + offset ) );
167 }
168 
169 /**
170  * Clear queue capabilities
171  *
172  * @v ena ENA device
173  * @v offset Register offset
174  */
175 static inline __attribute__ (( always_inline )) void
176 ena_clear_caps ( struct ena_nic *ena, unsigned int offset ) {
177 
178  /* Clear capabilities register */
179  writel ( 0, ( ena->regs + offset ) );
180 }
181 
182 /**
183  * Create admin queues
184  *
185  * @v ena ENA device
186  * @ret rc Return status code
187  */
188 static int ena_create_admin ( struct ena_nic *ena ) {
189  size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
190  size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
191  int rc;
192 
193  /* Allocate admin completion queue */
194  ena->acq.rsp = malloc_phys ( acq_len, acq_len );
195  if ( ! ena->acq.rsp ) {
196  rc = -ENOMEM;
197  goto err_alloc_acq;
198  }
199  memset ( ena->acq.rsp, 0, acq_len );
200 
201  /* Allocate admin queue */
202  ena->aq.req = malloc_phys ( aq_len, aq_len );
203  if ( ! ena->aq.req ) {
204  rc = -ENOMEM;
205  goto err_alloc_aq;
206  }
207  memset ( ena->aq.req, 0, aq_len );
208 
209  /* Program queue addresses and capabilities */
210  ena_set_base ( ena, ENA_ACQ_BASE, ena->acq.rsp );
212  sizeof ( ena->acq.rsp[0] ) );
213  ena_set_base ( ena, ENA_AQ_BASE, ena->aq.req );
215  sizeof ( ena->aq.req[0] ) );
216 
217  DBGC ( ena, "ENA %p AQ [%08lx,%08lx) ACQ [%08lx,%08lx)\n",
218  ena, virt_to_phys ( ena->aq.req ),
219  ( virt_to_phys ( ena->aq.req ) + aq_len ),
220  virt_to_phys ( ena->acq.rsp ),
221  ( virt_to_phys ( ena->acq.rsp ) + acq_len ) );
222  return 0;
223 
224  ena_clear_caps ( ena, ENA_AQ_CAPS );
225  ena_clear_caps ( ena, ENA_ACQ_CAPS );
226  free_phys ( ena->aq.req, aq_len );
227  err_alloc_aq:
228  free_phys ( ena->acq.rsp, acq_len );
229  err_alloc_acq:
230  return rc;
231 }
232 
233 /**
234  * Destroy admin queues
235  *
236  * @v ena ENA device
237  */
238 static void ena_destroy_admin ( struct ena_nic *ena ) {
239  size_t aq_len = ( ENA_AQ_COUNT * sizeof ( ena->aq.req[0] ) );
240  size_t acq_len = ( ENA_ACQ_COUNT * sizeof ( ena->acq.rsp[0] ) );
241 
242  /* Clear queue capabilities */
243  ena_clear_caps ( ena, ENA_AQ_CAPS );
244  ena_clear_caps ( ena, ENA_ACQ_CAPS );
245  wmb();
246 
247  /* Free queues */
248  free_phys ( ena->aq.req, aq_len );
249  free_phys ( ena->acq.rsp, acq_len );
250  DBGC ( ena, "ENA %p AQ and ACQ destroyed\n", ena );
251 }
252 
253 /**
254  * Get next available admin queue request
255  *
256  * @v ena ENA device
257  * @ret req Admin queue request
258  */
259 static union ena_aq_req * ena_admin_req ( struct ena_nic *ena ) {
260  union ena_aq_req *req;
261  unsigned int index;
262 
263  /* Get next request */
264  index = ( ena->aq.prod % ENA_AQ_COUNT );
265  req = &ena->aq.req[index];
266 
267  /* Initialise request */
268  memset ( ( ( ( void * ) req ) + sizeof ( req->header ) ), 0,
269  ( sizeof ( *req ) - sizeof ( req->header ) ) );
270  req->header.id = ena->aq.prod;
271 
272  /* Increment producer counter */
273  ena->aq.prod++;
274 
275  return req;
276 }
277 
278 /**
279  * Issue admin queue request
280  *
281  * @v ena ENA device
282  * @v req Admin queue request
283  * @v rsp Admin queue response to fill in
284  * @ret rc Return status code
285  */
286 static int ena_admin ( struct ena_nic *ena, union ena_aq_req *req,
287  union ena_acq_rsp **rsp ) {
288  unsigned int index;
289  unsigned int i;
290  int rc;
291 
292  /* Locate response */
293  index = ( ena->acq.cons % ENA_ACQ_COUNT );
294  *rsp = &ena->acq.rsp[index];
295 
296  /* Mark request as ready */
297  req->header.flags ^= ENA_AQ_PHASE;
298  wmb();
299  DBGC2 ( ena, "ENA %p admin request %#x:\n",
300  ena, le16_to_cpu ( req->header.id ) );
301  DBGC2_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
302 
303  /* Ring doorbell */
304  writel ( ena->aq.prod, ( ena->regs + ENA_AQ_DB ) );
305 
306  /* Wait for response */
307  for ( i = 0 ; i < ENA_ADMIN_MAX_WAIT_MS ; i++ ) {
308 
309  /* Check for response */
310  if ( ( (*rsp)->header.flags ^ ena->acq.phase ) & ENA_ACQ_PHASE){
311  mdelay ( 1 );
312  continue;
313  }
314  DBGC2 ( ena, "ENA %p admin response %#x:\n",
315  ena, le16_to_cpu ( (*rsp)->header.id ) );
316  DBGC2_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ));
317 
318  /* Increment consumer counter */
319  ena->acq.cons++;
320  if ( ( ena->acq.cons % ENA_ACQ_COUNT ) == 0 )
321  ena->acq.phase ^= ENA_ACQ_PHASE;
322 
323  /* Check command identifier */
324  if ( (*rsp)->header.id != req->header.id ) {
325  DBGC ( ena, "ENA %p admin response %#x mismatch:\n",
326  ena, le16_to_cpu ( (*rsp)->header.id ) );
327  rc = -EILSEQ;
328  goto err;
329  }
330 
331  /* Check status */
332  if ( (*rsp)->header.status != 0 ) {
333  DBGC ( ena, "ENA %p admin response %#x status %d:\n",
334  ena, le16_to_cpu ( (*rsp)->header.id ),
335  (*rsp)->header.status );
336  rc = -EIO;
337  goto err;
338  }
339 
340  /* Success */
341  return 0;
342  }
343 
344  rc = -ETIMEDOUT;
345  DBGC ( ena, "ENA %p timed out waiting for admin request %#x:\n",
346  ena, le16_to_cpu ( req->header.id ) );
347  err:
348  DBGC_HDA ( ena, virt_to_phys ( req ), req, sizeof ( *req ) );
349  DBGC_HDA ( ena, virt_to_phys ( *rsp ), *rsp, sizeof ( **rsp ) );
350  return rc;
351 }
352 
353 /**
354  * Set async event notification queue config
355  *
356  * @v ena ENA device
357  * @v enabled Bitmask of the groups to enable
358  * @ret rc Return status code
359  */
360 static int ena_set_aenq_config ( struct ena_nic *ena, uint32_t enabled ) {
361  union ena_aq_req *req;
362  union ena_acq_rsp *rsp;
363  union ena_feature *feature;
364  int rc;
365 
366  /* Construct request */
367  req = ena_admin_req ( ena );
370  feature = &req->set_feature.feature;
371  feature->aenq.enabled = cpu_to_le32 ( enabled );
372 
373  /* Issue request */
374  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
375  return rc;
376 
377  return 0;
378 }
379 
380 /**
381  * Create async event notification queue
382  *
383  * @v ena ENA device
384  * @ret rc Return status code
385  */
386 static int ena_create_async ( struct ena_nic *ena ) {
387  size_t aenq_len = ( ENA_AENQ_COUNT * sizeof ( ena->aenq.evt[0] ) );
388  int rc;
389 
390  /* Allocate async event notification queue */
391  ena->aenq.evt = malloc_phys ( aenq_len, aenq_len );
392  if ( ! ena->aenq.evt ) {
393  rc = -ENOMEM;
394  goto err_alloc_aenq;
395  }
396  memset ( ena->aenq.evt, 0, aenq_len );
397 
398  /* Program queue address and capabilities */
399  ena_set_base ( ena, ENA_AENQ_BASE, ena->aenq.evt );
401  sizeof ( ena->aenq.evt[0] ) );
402 
403  DBGC ( ena, "ENA %p AENQ [%08lx,%08lx)\n",
404  ena, virt_to_phys ( ena->aenq.evt ),
405  ( virt_to_phys ( ena->aenq.evt ) + aenq_len ) );
406 
407  /* Disable all events */
408  if ( ( rc = ena_set_aenq_config ( ena, 0 ) ) != 0 )
409  goto err_set_aenq_config;
410 
411  return 0;
412 
413  err_set_aenq_config:
414  ena_clear_caps ( ena, ENA_AENQ_CAPS );
415  free_phys ( ena->aenq.evt, aenq_len );
416  err_alloc_aenq:
417  return rc;
418 }
419 
420 /**
421  * Destroy async event notification queue
422  *
423  * @v ena ENA device
424  */
425 static void ena_destroy_async ( struct ena_nic *ena ) {
426  size_t aenq_len = ( ENA_AENQ_COUNT * sizeof ( ena->aenq.evt[0] ) );
427 
428  /* Clear queue capabilities */
429  ena_clear_caps ( ena, ENA_AENQ_CAPS );
430  wmb();
431 
432  /* Free queue */
433  free_phys ( ena->aenq.evt, aenq_len );
434  DBGC ( ena, "ENA %p AENQ destroyed\n", ena );
435 }
436 
437 /**
438  * Create submission queue
439  *
440  * @v ena ENA device
441  * @v sq Submission queue
442  * @v cq Corresponding completion queue
443  * @ret rc Return status code
444  */
445 static int ena_create_sq ( struct ena_nic *ena, struct ena_sq *sq,
446  struct ena_cq *cq ) {
447  union ena_aq_req *req;
448  union ena_acq_rsp *rsp;
449  unsigned int i;
450  int rc;
451 
452  /* Allocate submission queue entries */
453  sq->sqe.raw = malloc_phys ( sq->len, ENA_ALIGN );
454  if ( ! sq->sqe.raw ) {
455  rc = -ENOMEM;
456  goto err_alloc;
457  }
458  memset ( sq->sqe.raw, 0, sq->len );
459 
460  /* Construct request */
461  req = ena_admin_req ( ena );
462  req->header.opcode = ENA_CREATE_SQ;
463  req->create_sq.direction = sq->direction;
466  req->create_sq.cq_id = cpu_to_le16 ( cq->id );
467  req->create_sq.count = cpu_to_le16 ( sq->count );
468  req->create_sq.address = cpu_to_le64 ( virt_to_bus ( sq->sqe.raw ) );
469 
470  /* Issue request */
471  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
472  goto err_admin;
473 
474  /* Parse response */
475  sq->id = le16_to_cpu ( rsp->create_sq.id );
476  sq->doorbell = le32_to_cpu ( rsp->create_sq.doorbell );
477 
478  /* Reset producer counter and phase */
479  sq->prod = 0;
480  sq->phase = ENA_SQE_PHASE;
481 
482  /* Calculate fill level */
483  sq->fill = sq->max;
484  if ( sq->fill > cq->actual )
485  sq->fill = cq->actual;
486 
487  /* Initialise buffer ID ring */
488  for ( i = 0 ; i < sq->count ; i++ )
489  sq->ids[i] = i;
490 
491  DBGC ( ena, "ENA %p %s SQ%d at [%08lx,%08lx) fill %d db +%04x CQ%d\n",
492  ena, ena_direction ( sq->direction ), sq->id,
493  virt_to_phys ( sq->sqe.raw ),
494  ( virt_to_phys ( sq->sqe.raw ) + sq->len ),
495  sq->fill, sq->doorbell, cq->id );
496  return 0;
497 
498  err_admin:
499  free_phys ( sq->sqe.raw, sq->len );
500  err_alloc:
501  return rc;
502 }
503 
504 /**
505  * Destroy submission queue
506  *
507  * @v ena ENA device
508  * @v sq Submission queue
509  * @ret rc Return status code
510  */
511 static int ena_destroy_sq ( struct ena_nic *ena, struct ena_sq *sq ) {
512  union ena_aq_req *req;
513  union ena_acq_rsp *rsp;
514  int rc;
515 
516  /* Construct request */
517  req = ena_admin_req ( ena );
519  req->destroy_sq.id = cpu_to_le16 ( sq->id );
520  req->destroy_sq.direction = sq->direction;
521 
522  /* Issue request */
523  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
524  return rc;
525 
526  /* Free submission queue entries */
527  free_phys ( sq->sqe.raw, sq->len );
528 
529  DBGC ( ena, "ENA %p %s SQ%d destroyed\n",
530  ena, ena_direction ( sq->direction ), sq->id );
531  return 0;
532 }
533 
534 /**
535  * Create completion queue
536  *
537  * @v ena ENA device
538  * @v cq Completion queue
539  * @ret rc Return status code
540  */
541 static int ena_create_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
542  union ena_aq_req *req;
543  union ena_acq_rsp *rsp;
544  int rc;
545 
546  /* Allocate completion queue entries */
547  cq->cqe.raw = malloc_phys ( cq->len, ENA_ALIGN );
548  if ( ! cq->cqe.raw ) {
549  rc = -ENOMEM;
550  goto err_alloc;
551  }
552  memset ( cq->cqe.raw, 0, cq->len );
553 
554  /* Construct request */
555  req = ena_admin_req ( ena );
556  req->header.opcode = ENA_CREATE_CQ;
557  req->create_cq.size = cq->size;
558  req->create_cq.count = cpu_to_le16 ( cq->requested );
560  req->create_cq.address = cpu_to_le64 ( virt_to_bus ( cq->cqe.raw ) );
561 
562  /* Issue request */
563  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
564  goto err_admin;
565 
566  /* Parse response */
567  cq->id = le16_to_cpu ( rsp->create_cq.id );
568  cq->actual = le16_to_cpu ( rsp->create_cq.count );
569  cq->doorbell = le32_to_cpu ( rsp->create_cq.doorbell );
570  cq->mask = ( cq->actual - 1 );
571  if ( cq->actual != cq->requested ) {
572  DBGC ( ena, "ENA %p CQ%d requested %d actual %d\n",
573  ena, cq->id, cq->requested, cq->actual );
574  }
575 
576  /* Reset consumer counter and phase */
577  cq->cons = 0;
578  cq->phase = ENA_CQE_PHASE;
579 
580  DBGC ( ena, "ENA %p CQ%d at [%08lx,%08lx) db +%04x\n",
581  ena, cq->id, virt_to_phys ( cq->cqe.raw ),
582  ( virt_to_phys ( cq->cqe.raw ) + cq->len ), cq->doorbell );
583  return 0;
584 
585  err_admin:
586  free_phys ( cq->cqe.raw, cq->len );
587  err_alloc:
588  return rc;
589 }
590 
591 /**
592  * Destroy completion queue
593  *
594  * @v ena ENA device
595  * @v cq Completion queue
596  * @ret rc Return status code
597  */
598 static int ena_destroy_cq ( struct ena_nic *ena, struct ena_cq *cq ) {
599  union ena_aq_req *req;
600  union ena_acq_rsp *rsp;
601  int rc;
602 
603  /* Construct request */
604  req = ena_admin_req ( ena );
606  req->destroy_cq.id = cpu_to_le16 ( cq->id );
607 
608  /* Issue request */
609  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
610  return rc;
611 
612  /* Free completion queue entries */
613  free_phys ( cq->cqe.raw, cq->len );
614 
615  DBGC ( ena, "ENA %p CQ%d destroyed\n", ena, cq->id );
616  return 0;
617 }
618 
619 /**
620  * Create queue pair
621  *
622  * @v ena ENA device
623  * @v qp Queue pair
624  * @ret rc Return status code
625  */
626 static int ena_create_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
627  int rc;
628 
629  /* Create completion queue */
630  if ( ( rc = ena_create_cq ( ena, &qp->cq ) ) != 0 )
631  goto err_create_cq;
632 
633  /* Create submission queue */
634  if ( ( rc = ena_create_sq ( ena, &qp->sq, &qp->cq ) ) != 0 )
635  goto err_create_sq;
636 
637  return 0;
638 
639  ena_destroy_sq ( ena, &qp->sq );
640  err_create_sq:
641  ena_destroy_cq ( ena, &qp->cq );
642  err_create_cq:
643  return rc;
644 }
645 
646 /**
647  * Destroy queue pair
648  *
649  * @v ena ENA device
650  * @v qp Queue pair
651  * @ret rc Return status code
652  */
653 static int ena_destroy_qp ( struct ena_nic *ena, struct ena_qp *qp ) {
654 
655  /* Destroy submission queue */
656  ena_destroy_sq ( ena, &qp->sq );
657 
658  /* Destroy completion queue */
659  ena_destroy_cq ( ena, &qp->cq );
660 
661  return 0;
662 }
663 
664 /**
665  * Get device attributes
666  *
667  * @v netdev Network device
668  * @ret rc Return status code
669  */
671  struct ena_nic *ena = netdev->priv;
672  union ena_aq_req *req;
673  union ena_acq_rsp *rsp;
674  union ena_feature *feature;
675  int rc;
676 
677  /* Construct request */
678  req = ena_admin_req ( ena );
681 
682  /* Issue request */
683  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
684  return rc;
685 
686  /* Parse response */
687  feature = &rsp->get_feature.feature;
688  memcpy ( netdev->hw_addr, feature->device.mac, ETH_ALEN );
689  netdev->max_pkt_len = le32_to_cpu ( feature->device.mtu );
691 
692  DBGC ( ena, "ENA %p MAC %s MTU %zd\n",
693  ena, eth_ntoa ( netdev->hw_addr ), netdev->max_pkt_len );
694  return 0;
695 }
696 
697 /**
698  * Set host attributes
699  *
700  * @v ena ENA device
701  * @ret rc Return status code
702  */
703 static int ena_set_host_attributes ( struct ena_nic *ena ) {
704  union ena_aq_req *req;
705  union ena_acq_rsp *rsp;
706  union ena_feature *feature;
707  int rc;
708 
709  /* Construct request */
710  req = ena_admin_req ( ena );
713  feature = &req->set_feature.feature;
714  feature->host.info = cpu_to_le64 ( virt_to_bus ( ena->info ) );
715 
716  /* Issue request */
717  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
718  return rc;
719 
720  return 0;
721 }
722 
723 /**
724  * Get statistics (for debugging)
725  *
726  * @v ena ENA device
727  * @ret rc Return status code
728  */
729 static int ena_get_stats ( struct ena_nic *ena ) {
730  union ena_aq_req *req;
731  union ena_acq_rsp *rsp;
732  struct ena_get_stats_rsp *stats;
733  int rc;
734 
735  /* Do nothing unless debug messages are enabled */
736  if ( ! DBG_LOG )
737  return 0;
738 
739  /* Construct request */
740  req = ena_admin_req ( ena );
741  req->header.opcode = ENA_GET_STATS;
745 
746  /* Issue request */
747  if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
748  return rc;
749 
750  /* Parse response */
751  stats = &rsp->get_stats;
752  DBGC ( ena, "ENA %p TX bytes %#llx packets %#llx\n", ena,
753  ( ( unsigned long long ) le64_to_cpu ( stats->tx_bytes ) ),
754  ( ( unsigned long long ) le64_to_cpu ( stats->tx_packets ) ) );
755  DBGC ( ena, "ENA %p RX bytes %#llx packets %#llx drops %#llx\n", ena,
756  ( ( unsigned long long ) le64_to_cpu ( stats->rx_bytes ) ),
757  ( ( unsigned long long ) le64_to_cpu ( stats->rx_packets ) ),
758  ( ( unsigned long long ) le64_to_cpu ( stats->rx_drops ) ) );
759 
760  return 0;
761 }
762 
763 /******************************************************************************
764  *
765  * Network device interface
766  *
767  ******************************************************************************
768  */
769 
770 /**
771  * Refill receive queue
772  *
773  * @v netdev Network device
774  */
775 static void ena_refill_rx ( struct net_device *netdev ) {
776  struct ena_nic *ena = netdev->priv;
777  struct io_buffer *iobuf;
778  struct ena_rx_sqe *sqe;
780  size_t len = netdev->max_pkt_len;
781  unsigned int refilled = 0;
782  unsigned int index;
783  unsigned int id;
784 
785  /* Refill queue */
786  while ( ( ena->rx.sq.prod - ena->rx.cq.cons ) < ena->rx.sq.fill ) {
787 
788  /* Allocate I/O buffer */
789  iobuf = alloc_iob ( len );
790  if ( ! iobuf ) {
791  /* Wait for next refill */
792  break;
793  }
794 
795  /* Get next submission queue entry and buffer ID */
796  index = ( ena->rx.sq.prod % ENA_RX_COUNT );
797  sqe = &ena->rx.sq.sqe.rx[index];
798  id = ena->rx_ids[index];
799 
800  /* Construct submission queue entry */
801  address = virt_to_bus ( iobuf->data );
802  sqe->len = cpu_to_le16 ( len );
803  sqe->id = cpu_to_le16 ( id );
804  sqe->address = cpu_to_le64 ( address );
805  wmb();
807  ena->rx.sq.phase );
808 
809  /* Increment producer counter */
810  ena->rx.sq.prod++;
811  if ( ( ena->rx.sq.prod % ENA_RX_COUNT ) == 0 )
812  ena->rx.sq.phase ^= ENA_SQE_PHASE;
813 
814  /* Record I/O buffer */
815  assert ( ena->rx_iobuf[id] == NULL );
816  ena->rx_iobuf[id] = iobuf;
817 
818  DBGC2 ( ena, "ENA %p RX %d at [%08llx,%08llx)\n", ena, id,
819  ( ( unsigned long long ) address ),
820  ( ( unsigned long long ) address + len ) );
821  refilled++;
822  }
823 
824  /* Ring doorbell, if applicable */
825  if ( refilled ) {
826  wmb();
827  writel ( ena->rx.sq.prod, ( ena->regs + ena->rx.sq.doorbell ) );
828  }
829 }
830 
831 /**
832  * Discard unused receive I/O buffers
833  *
834  * @v ena ENA device
835  */
836 static void ena_empty_rx ( struct ena_nic *ena ) {
837  unsigned int i;
838 
839  for ( i = 0 ; i < ENA_RX_COUNT ; i++ ) {
840  if ( ena->rx_iobuf[i] )
841  free_iob ( ena->rx_iobuf[i] );
842  ena->rx_iobuf[i] = NULL;
843  }
844 }
845 
846 /**
847  * Open network device
848  *
849  * @v netdev Network device
850  * @ret rc Return status code
851  */
852 static int ena_open ( struct net_device *netdev ) {
853  struct ena_nic *ena = netdev->priv;
854  int rc;
855 
856  /* Create transmit queue pair */
857  if ( ( rc = ena_create_qp ( ena, &ena->tx ) ) != 0 )
858  goto err_create_tx;
859 
860  /* Create receive queue pair */
861  if ( ( rc = ena_create_qp ( ena, &ena->rx ) ) != 0 )
862  goto err_create_rx;
863 
864  /* Refill receive queue */
865  ena_refill_rx ( netdev );
866 
867  return 0;
868 
869  ena_destroy_qp ( ena, &ena->rx );
870  err_create_rx:
871  ena_destroy_qp ( ena, &ena->tx );
872  err_create_tx:
873  return rc;
874 }
875 
876 /**
877  * Close network device
878  *
879  * @v netdev Network device
880  */
881 static void ena_close ( struct net_device *netdev ) {
882  struct ena_nic *ena = netdev->priv;
883 
884  /* Dump statistics (for debugging) */
885  ena_get_stats ( ena );
886 
887  /* Destroy receive queue pair */
888  ena_destroy_qp ( ena, &ena->rx );
889 
890  /* Discard any unused receive buffers */
891  ena_empty_rx ( ena );
892 
893  /* Destroy transmit queue pair */
894  ena_destroy_qp ( ena, &ena->tx );
895 }
896 
897 /**
898  * Transmit packet
899  *
900  * @v netdev Network device
901  * @v iobuf I/O buffer
902  * @ret rc Return status code
903  */
904 static int ena_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
905  struct ena_nic *ena = netdev->priv;
906  struct ena_tx_sqe *sqe;
908  unsigned int index;
909  unsigned int id;
910  size_t len;
911 
912  /* Get next submission queue entry */
913  if ( ( ena->tx.sq.prod - ena->tx.cq.cons ) >= ena->tx.sq.fill ) {
914  DBGC ( ena, "ENA %p out of transmit descriptors\n", ena );
915  return -ENOBUFS;
916  }
917  index = ( ena->tx.sq.prod % ENA_TX_COUNT );
918  sqe = &ena->tx.sq.sqe.tx[index];
919  id = ena->tx_ids[index];
920 
921  /* Construct submission queue entry */
922  address = virt_to_bus ( iobuf->data );
923  len = iob_len ( iobuf );
924  sqe->len = cpu_to_le16 ( len );
925  sqe->id = cpu_to_le16 ( id );
926  sqe->address = cpu_to_le64 ( address );
927  wmb();
929  ena->tx.sq.phase );
930  wmb();
931 
932  /* Increment producer counter */
933  ena->tx.sq.prod++;
934  if ( ( ena->tx.sq.prod % ENA_TX_COUNT ) == 0 )
935  ena->tx.sq.phase ^= ENA_SQE_PHASE;
936 
937  /* Record I/O buffer */
938  assert ( ena->tx_iobuf[id] == NULL );
939  ena->tx_iobuf[id] = iobuf;
940 
941  /* Ring doorbell */
942  writel ( ena->tx.sq.prod, ( ena->regs + ena->tx.sq.doorbell ) );
943 
944  DBGC2 ( ena, "ENA %p TX %d at [%08llx,%08llx)\n", ena, id,
945  ( ( unsigned long long ) address ),
946  ( ( unsigned long long ) address + len ) );
947  return 0;
948 }
949 
950 /**
951  * Poll for completed transmissions
952  *
953  * @v netdev Network device
954  */
955 static void ena_poll_tx ( struct net_device *netdev ) {
956  struct ena_nic *ena = netdev->priv;
957  struct ena_tx_cqe *cqe;
958  struct io_buffer *iobuf;
959  unsigned int index;
960  unsigned int id;
961 
962  /* Check for completed packets */
963  while ( ena->tx.cq.cons != ena->tx.sq.prod ) {
964 
965  /* Get next completion queue entry */
966  index = ( ena->tx.cq.cons & ena->tx.cq.mask );
967  cqe = &ena->tx.cq.cqe.tx[index];
968 
969  /* Stop if completion queue entry is empty */
970  if ( ( cqe->flags ^ ena->tx.cq.phase ) & ENA_CQE_PHASE )
971  return;
972 
973  /* Increment consumer counter */
974  ena->tx.cq.cons++;
975  if ( ! ( ena->tx.cq.cons & ena->tx.cq.mask ) )
976  ena->tx.cq.phase ^= ENA_CQE_PHASE;
977 
978  /* Identify and free buffer ID */
979  id = ENA_TX_CQE_ID ( le16_to_cpu ( cqe->id ) );
980  ena->tx_ids[index] = id;
981 
982  /* Identify I/O buffer */
983  iobuf = ena->tx_iobuf[id];
984  assert ( iobuf != NULL );
985  ena->tx_iobuf[id] = NULL;
986 
987  /* Complete transmit */
988  DBGC2 ( ena, "ENA %p TX %d complete\n", ena, id );
989  netdev_tx_complete ( netdev, iobuf );
990  }
991 }
992 
993 /**
994  * Poll for received packets
995  *
996  * @v netdev Network device
997  */
998 static void ena_poll_rx ( struct net_device *netdev ) {
999  struct ena_nic *ena = netdev->priv;
1000  struct ena_rx_cqe *cqe;
1001  struct io_buffer *iobuf;
1002  unsigned int index;
1003  unsigned int id;
1004  size_t len;
1005 
1006  /* Check for received packets */
1007  while ( ena->rx.cq.cons != ena->rx.sq.prod ) {
1008 
1009  /* Get next completion queue entry */
1010  index = ( ena->rx.cq.cons & ena->rx.cq.mask );
1011  cqe = &ena->rx.cq.cqe.rx[index];
1012 
1013  /* Stop if completion queue entry is empty */
1014  if ( ( cqe->flags ^ ena->rx.cq.phase ) & ENA_CQE_PHASE )
1015  return;
1016 
1017  /* Increment consumer counter */
1018  ena->rx.cq.cons++;
1019  if ( ! ( ena->rx.cq.cons & ena->rx.cq.mask ) )
1020  ena->rx.cq.phase ^= ENA_CQE_PHASE;
1021 
1022  /* Identify and free buffer ID */
1023  id = le16_to_cpu ( cqe->id );
1024  ena->rx_ids[index] = id;
1025 
1026  /* Populate I/O buffer */
1027  iobuf = ena->rx_iobuf[id];
1028  assert ( iobuf != NULL );
1029  ena->rx_iobuf[id] = NULL;
1030  len = le16_to_cpu ( cqe->len );
1031  iob_put ( iobuf, len );
1032 
1033  /* Hand off to network stack */
1034  DBGC2 ( ena, "ENA %p RX %d complete (length %zd)\n",
1035  ena, id, len );
1036  netdev_rx ( netdev, iobuf );
1037  }
1038 }
1039 
1040 /**
1041  * Poll for completed and received packets
1042  *
1043  * @v netdev Network device
1044  */
1045 static void ena_poll ( struct net_device *netdev ) {
1046 
1047  /* Poll for transmit completions */
1048  ena_poll_tx ( netdev );
1049 
1050  /* Poll for receive completions */
1051  ena_poll_rx ( netdev );
1052 
1053  /* Refill receive ring */
1054  ena_refill_rx ( netdev );
1055 }
1056 
1057 /** ENA network device operations */
1059  .open = ena_open,
1060  .close = ena_close,
1061  .transmit = ena_transmit,
1062  .poll = ena_poll,
1063 };
1064 
1065 /******************************************************************************
1066  *
1067  * PCI interface
1068  *
1069  ******************************************************************************
1070  */
1071 
1072 /**
1073  * Assign memory BAR
1074  *
1075  * @v ena ENA device
1076  * @v pci PCI device
1077  * @ret rc Return status code
1078  *
1079  * Some BIOSes in AWS EC2 are observed to fail to assign a base
1080  * address to the ENA device. The device is the only device behind
1081  * its bridge, and the BIOS does assign a memory window to the bridge.
1082  * We therefore place the device at the start of the memory window.
1083  */
1084 static int ena_membase ( struct ena_nic *ena, struct pci_device *pci ) {
1085  struct pci_bridge *bridge;
1086 
1087  /* Locate PCI bridge */
1088  bridge = pcibridge_find ( pci );
1089  if ( ! bridge ) {
1090  DBGC ( ena, "ENA %p found no PCI bridge\n", ena );
1091  return -ENOTCONN;
1092  }
1093 
1094  /* Sanity check */
1095  if ( PCI_SLOT ( pci->busdevfn ) || PCI_FUNC ( pci->busdevfn ) ) {
1096  DBGC ( ena, "ENA %p at " PCI_FMT " may not be only device "
1097  "on bus\n", ena, PCI_ARGS ( pci ) );
1098  return -ENOTSUP;
1099  }
1100 
1101  /* Place device at start of memory window */
1103  pci->membase = bridge->membase;
1104  DBGC ( ena, "ENA %p at " PCI_FMT " claiming bridge " PCI_FMT " mem "
1105  "%08x\n", ena, PCI_ARGS ( pci ), PCI_ARGS ( bridge->pci ),
1106  bridge->membase );
1107 
1108  return 0;
1109 }
1110 
1111 /**
1112  * Probe PCI device
1113  *
1114  * @v pci PCI device
1115  * @ret rc Return status code
1116  */
1117 static int ena_probe ( struct pci_device *pci ) {
1118  struct net_device *netdev;
1119  struct ena_nic *ena;
1120  struct ena_host_info *info;
1121  int rc;
1122 
1123  /* Allocate and initialise net device */
1124  netdev = alloc_etherdev ( sizeof ( *ena ) );
1125  if ( ! netdev ) {
1126  rc = -ENOMEM;
1127  goto err_alloc;
1128  }
1130  ena = netdev->priv;
1131  pci_set_drvdata ( pci, netdev );
1132  netdev->dev = &pci->dev;
1133  memset ( ena, 0, sizeof ( *ena ) );
1134  ena->acq.phase = ENA_ACQ_PHASE;
1135  ena_cq_init ( &ena->tx.cq, ENA_TX_COUNT,
1136  sizeof ( ena->tx.cq.cqe.tx[0] ) );
1137  ena_sq_init ( &ena->tx.sq, ENA_SQ_TX, ENA_TX_COUNT, ENA_TX_COUNT,
1138  sizeof ( ena->tx.sq.sqe.tx[0] ), ena->tx_ids );
1139  ena_cq_init ( &ena->rx.cq, ENA_RX_COUNT,
1140  sizeof ( ena->rx.cq.cqe.rx[0] ) );
1141  ena_sq_init ( &ena->rx.sq, ENA_SQ_RX, ENA_RX_COUNT, ENA_RX_FILL,
1142  sizeof ( ena->rx.sq.sqe.rx[0] ), ena->rx_ids );
1143 
1144  /* Fix up PCI device */
1145  adjust_pci_device ( pci );
1146 
1147  /* Fix up PCI BAR if left unassigned by BIOS */
1148  if ( ( ! pci->membase ) && ( ( rc = ena_membase ( ena, pci ) ) != 0 ) )
1149  goto err_membase;
1150 
1151  /* Map registers */
1152  ena->regs = pci_ioremap ( pci, pci->membase, ENA_BAR_SIZE );
1153  if ( ! ena->regs ) {
1154  rc = -ENODEV;
1155  goto err_ioremap;
1156  }
1157 
1158  /* Allocate and initialise host info */
1160  if ( ! info ) {
1161  rc = -ENOMEM;
1162  goto err_info;
1163  }
1164  ena->info = info;
1165  memset ( info, 0, PAGE_SIZE );
1167  snprintf ( info->dist_str, sizeof ( info->dist_str ), "%s",
1169  snprintf ( info->kernel_str, sizeof ( info->kernel_str ), "%s",
1170  product_version );
1173  info->busdevfn = cpu_to_le16 ( pci->busdevfn );
1174  DBGC2 ( ena, "ENA %p host info:\n", ena );
1175  DBGC2_HDA ( ena, virt_to_phys ( info ), info, sizeof ( *info ) );
1176 
1177  /* Reset the NIC */
1178  if ( ( rc = ena_reset ( ena ) ) != 0 )
1179  goto err_reset;
1180 
1181  /* Create admin queues */
1182  if ( ( rc = ena_create_admin ( ena ) ) != 0 )
1183  goto err_create_admin;
1184 
1185  /* Create async event notification queue */
1186  if ( ( rc = ena_create_async ( ena ) ) != 0 )
1187  goto err_create_async;
1188 
1189  /* Set host attributes */
1190  if ( ( rc = ena_set_host_attributes ( ena ) ) != 0 )
1191  goto err_set_host_attributes;
1192 
1193  /* Fetch MAC address */
1194  if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
1195  goto err_get_device_attributes;
1196 
1197  /* Register network device */
1198  if ( ( rc = register_netdev ( netdev ) ) != 0 )
1199  goto err_register_netdev;
1200 
1201  /* Mark as link up, since we have no way to test link state on
1202  * this hardware.
1203  */
1204  netdev_link_up ( netdev );
1205 
1206  return 0;
1207 
1209  err_register_netdev:
1210  err_get_device_attributes:
1211  err_set_host_attributes:
1212  ena_destroy_async ( ena );
1213  err_create_async:
1214  ena_destroy_admin ( ena );
1215  err_create_admin:
1216  ena_reset ( ena );
1217  err_reset:
1218  free_phys ( ena->info, PAGE_SIZE );
1219  err_info:
1220  iounmap ( ena->regs );
1221  err_ioremap:
1222  err_membase:
1223  netdev_nullify ( netdev );
1224  netdev_put ( netdev );
1225  err_alloc:
1226  return rc;
1227 }
1228 
1229 /**
1230  * Remove PCI device
1231  *
1232  * @v pci PCI device
1233  */
1234 static void ena_remove ( struct pci_device *pci ) {
1235  struct net_device *netdev = pci_get_drvdata ( pci );
1236  struct ena_nic *ena = netdev->priv;
1237 
1238  /* Unregister network device */
1240 
1241  /* Destroy async event notification queue */
1242  ena_destroy_async ( ena );
1243 
1244  /* Destroy admin queues */
1245  ena_destroy_admin ( ena );
1246 
1247  /* Reset card */
1248  ena_reset ( ena );
1249 
1250  /* Free host info */
1251  free_phys ( ena->info, PAGE_SIZE );
1252 
1253  /* Free network device */
1254  iounmap ( ena->regs );
1255  netdev_nullify ( netdev );
1256  netdev_put ( netdev );
1257 }
1258 
1259 /** ENA PCI device IDs */
1260 static struct pci_device_id ena_nics[] = {
1261  PCI_ROM ( 0x1d0f, 0xec20, "ena-vf", "ENA VF", 0 ),
1262  PCI_ROM ( 0x1d0f, 0xec21, "ena-vf-llq", "ENA VF (LLQ)", 0 ),
1263 };
1264 
1265 /** ENA PCI driver */
1266 struct pci_driver ena_driver __pci_driver = {
1267  .ids = ena_nics,
1268  .id_count = ( sizeof ( ena_nics ) / sizeof ( ena_nics[0] ) ),
1269  .probe = ena_probe,
1270  .remove = ena_remove,
1271 };
const char product_short_name[]
Product short name string.
Definition: version.c:76
#define PCI_FUNC(busdevfn)
Definition: pci.h:281
struct ena_aq aq
Admin queue.
Definition: ena.h:746
Queue pair.
Definition: ena.h:732
static void ena_refill_rx(struct net_device *netdev)
Refill receive queue.
Definition: ena.c:775
#define __attribute__(x)
Definition: compiler.h:10
#define ENA_ACQ_COUNT
Number of admin completion queue entries.
Definition: ena.h:25
struct ena_tx_sqe * tx
Transmit submission queue entries.
Definition: ena.h:636
static void ena_destroy_admin(struct ena_nic *ena)
Destroy admin queues.
Definition: ena.c:238
struct ena_qp rx
Receive queue.
Definition: ena.h:754
unsigned long membase
Memory base.
Definition: pci.h:215
#define ENA_CTRL
Device control register.
Definition: ena.h:73
uint8_t type
Type.
Definition: ena.h:435
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
static void ena_remove(struct pci_device *pci)
Remove PCI device.
Definition: ena.c:1234
static void netdev_tx_complete(struct net_device *netdev, struct io_buffer *iobuf)
Complete network transmission.
Definition: netdevice.h:752
wmb()
void * raw
Raw data.
Definition: ena.h:694
#define ENA_BASE_HI
Base address high register offset.
Definition: ena.h:43
Host information.
Definition: ena.h:167
#define iob_put(iobuf, len)
Definition: iobuf.h:120
u32 info
Definition: ar9003_mac.h:67
#define ENA_RX_COUNT
Number of receive queue entries.
Definition: ena.h:34
#define ENA_AQ_COUNT
Number of admin queue entries.
Definition: ena.h:22
A PCI driver.
Definition: pci.h:247
uint8_t rx_ids[ENA_RX_COUNT]
Receive buffer IDs.
Definition: ena.h:760
#define ENA_AENQ_CONFIG
Async event notification queue config.
Definition: ena.h:143
static void ena_poll_tx(struct net_device *netdev)
Poll for completed transmissions.
Definition: ena.c:955
uint8_t mask
Actual number of entries minus one.
Definition: ena.h:713
size_t len
Total length of entries.
Definition: ena.h:647
#define ENA_HOST_INFO_TYPE_LINUX
Linux operating system type.
Definition: ena.h:201
unsigned int doorbell
Doorbell register offset.
Definition: ena.h:645
uint8_t count
Number of entries.
Definition: ena.h:657
#define le32_to_cpu(value)
Definition: byteswap.h:113
int(* open)(struct net_device *netdev)
Open network device.
Definition: netdevice.h:222
#define ENA_GET_FEATURE
Get feature.
Definition: ena.h:378
#define ENA_HOST_ATTRIBUTES
Host attributes.
Definition: ena.h:154
Error codes.
#define ENA_CREATE_CQ
Create completion queue.
Definition: ena.h:316
#define ENA_ACQ_BASE
Admin completion queue base address register.
Definition: ena.h:55
uint8_t * ids
Buffer IDs.
Definition: ena.h:643
unsigned int prod
Producer counter.
Definition: ena.h:649
uint16_t id
Completion queue identifier.
Definition: ena.h:366
I/O buffers.
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:249
uint64_t address
Base address.
Definition: ena.h:24
struct ena_tx_cqe * tx
Transmit completion queue entries.
Definition: ena.h:690
size_t mtu
Maximum transmission unit length.
Definition: netdevice.h:415
static int ena_get_device_attributes(struct net_device *netdev)
Get device attributes.
Definition: ena.c:670
uint32_t readl(volatile uint32_t *io_addr)
Read 32-bit dword from memory-mapped device.
static int ena_get_stats(struct ena_nic *ena)
Get statistics (for debugging)
Definition: ena.c:729
Transmit submission queue entry.
Definition: ena.h:554
struct ena_qp tx
Transmit queue.
Definition: ena.h:752
#define DBGC(...)
Definition: compiler.h:505
struct ena_set_feature_req set_feature
Set feature.
Definition: ena.h:486
uint16_t count
Number of entries.
Definition: ena.h:261
union ena_sq::@39 sqe
Entries.
struct ena_aenq aenq
Async event notification queue.
Definition: ena.h:750
#define ENA_SQE_CPL
Request completion.
Definition: ena.h:595
Transmit completion queue entry.
Definition: ena.h:598
unsigned long long uint64_t
Definition: stdint.h:13
uint8_t id
Request identifier.
Definition: ena.h:564
static void *__malloc malloc_phys(size_t size, size_t phys_align)
Allocate memory with specified physical alignment.
Definition: malloc.h:62
static int ena_open(struct net_device *netdev)
Open network device.
Definition: ena.c:852
#define PCI_BASE_ADDRESS_0
Definition: pci.h:62
union ena_cq::@40 cqe
Entries.
#define cpu_to_le64(value)
Definition: byteswap.h:108
uint16_t count
Number of entries.
Definition: ena.h:327
uint8_t direction
Direction.
Definition: ena.h:14
uint64_t rx_drops
Receive drop count.
Definition: ena.h:468
uint16_t len
Length.
Definition: ena.h:572
#define ENA_AQ_BASE
Admin queue base address register.
Definition: ena.h:49
#define ENA_GET_STATS
Get statistics.
Definition: ena.h:426
#define PAGE_SIZE
Page size.
Definition: io.h:27
static void ena_close(struct net_device *netdev)
Close network device.
Definition: ena.c:881
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:707
static int ena_create_qp(struct ena_nic *ena, struct ena_qp *qp)
Create queue pair.
Definition: ena.c:626
unsigned int phase
Phase.
Definition: ena.h:703
#define ENA_ACQ_CAPS
Admin completion queue capabilities register.
Definition: ena.h:58
static const void * base
Base address.
Definition: crypto.h:335
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:154
struct io_buffer * alloc_iob(size_t len)
Allocate I/O buffer.
Definition: iobuf.c:129
static __always_inline unsigned long virt_to_phys(volatile const void *addr)
Convert virtual address to a physical address.
Definition: uaccess.h:287
uint64_t address
Address.
Definition: ena.h:582
struct device dev
Generic device.
Definition: pci.h:208
#define ENA_CTRL_RESET
Reset.
Definition: ena.h:74
uint32_t enabled
Bitmask of enabled AENQ groups (host -> device)
Definition: ena.h:14
#define ENA_TX_COUNT
Number of transmit queue entries.
Definition: ena.h:31
#define ENA_AENQ_BASE
Async event notification queue base address register.
Definition: ena.h:70
uint64_t address
Base address.
Definition: ena.h:331
#define ENOTSUP
Operation not supported.
Definition: errno.h:589
unsigned int doorbell
Doorbell register offset.
Definition: ena.h:697
void * regs
Registers.
Definition: ena.h:742
Dynamic memory allocation.
uint8_t tx_ids[ENA_TX_COUNT]
Transmit buffer IDs.
Definition: ena.h:756
union ena_feature feature
Feature.
Definition: ena.h:14
Completion queue.
Definition: ena.h:686
static int ena_reset_wait(struct ena_nic *ena, uint32_t expected)
Wait for reset operation to be acknowledged.
Definition: ena.c:77
Submission queue.
Definition: ena.h:632
static int ena_set_host_attributes(struct ena_nic *ena)
Set host attributes.
Definition: ena.c:703
static int ena_destroy_qp(struct ena_nic *ena, struct ena_qp *qp)
Destroy queue pair.
Definition: ena.c:653
static void ena_empty_rx(struct ena_nic *ena)
Discard unused receive I/O buffers.
Definition: ena.c:836
Feature.
Definition: ena.h:228
static void netdev_init(struct net_device *netdev, struct net_device_operations *op)
Initialise a network device.
Definition: netdevice.h:515
#define ENA_DEVICE_ATTRIBUTES
Device attributes.
Definition: ena.h:118
static int ena_destroy_sq(struct ena_nic *ena, struct ena_sq *sq)
Destroy submission queue.
Definition: ena.c:511
size_t len
Total length of entries.
Definition: ena.h:699
#define ENA_CAPS(count, size)
Capability register value.
Definition: ena.h:46
uint16_t id
Request identifier.
Definition: ena.h:600
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:359
#define ENOMEM
Not enough space.
Definition: errno.h:534
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static signed char phys[4]
Definition: epic100.c:88
uint64_t tx_packets
Transmit packet count.
Definition: ena.h:462
#define EILSEQ
Illegal byte sequence.
Definition: errno.h:413
uint16_t id
Request identifier.
Definition: ena.h:578
uint8_t id
Feature identifier.
Definition: ena.h:391
#define ENA_BASE_LO
Base address low register offset.
Definition: ena.h:40
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define ETH_HLEN
Definition: if_ether.h:9
#define ENA_STAT_RESET
Reset in progress.
Definition: ena.h:81
static void ena_set_caps(struct ena_nic *ena, unsigned int offset, unsigned int count, size_t size)
Set queue capabilities.
Definition: ena.c:162
struct pci_bridge * pcibridge_find(struct pci_device *pci)
Find bridge attached to a PCI device.
Definition: pcibridge.c:48
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
static void netdev_put(struct net_device *netdev)
Drop reference to network device.
Definition: netdevice.h:572
Ethernet protocol.
#define ENA_TX_CQE_ID(id)
Transmit completion request identifier.
Definition: ena.h:612
union ena_aq_req * req
Requests.
Definition: ena.h:516
#define ENA_AENQ_CAPS
Async event notification queue capabilities register.
Definition: ena.h:67
Transmit.
Definition: ena.h:240
uint64_t address
Base address.
Definition: ena.h:263
void * priv
Driver private data.
Definition: netdevice.h:431
static int ena_transmit(struct net_device *netdev, struct io_buffer *iobuf)
Transmit packet.
Definition: ena.c:904
#define DBGC_HDA(...)
Definition: compiler.h:506
uint16_t cq_id
Completion queue identifier.
Definition: ena.h:259
unsigned int cons
Consumer counter.
Definition: ena.h:526
static int ena_set_aenq_config(struct ena_nic *ena, uint32_t enabled)
Set async event notification queue config.
Definition: ena.c:360
static void netdev_link_up(struct net_device *netdev)
Mark network device as having link up.
Definition: netdevice.h:774
void writel(uint32_t data, volatile uint32_t *io_addr)
Write 32-bit dword to memory-mapped device.
uint8_t flags
Flags.
Definition: ena.h:604
struct pci_device * pci
PCI device.
Definition: pcibridge.h:19
unsigned int cons
Consumer counter.
Definition: ena.h:701
uint16_t policy
Policy.
Definition: ena.h:257
#define ENA_CQE_PHASE
Completion queue ownership phase flag.
Definition: ena.h:629
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
static struct net_device * netdev
Definition: gdbudp.c:52
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:325
uint8_t flags
Flags.
Definition: ena.h:92
static int ena_probe(struct pci_device *pci)
Probe PCI device.
Definition: ena.c:1117
#define ENA_STATS_TYPE_BASIC
Basic statistics.
Definition: ena.h:447
#define ENA_AENQ_COUNT
Number of async event notification queue entries.
Definition: ena.h:28
struct ena_aenq_event * evt
Events.
Definition: ena.h:550
#define ENA_SQE_FIRST
This is the first descriptor.
Definition: ena.h:589
void unregister_netdev(struct net_device *netdev)
Unregister network device.
Definition: netdevice.c:941
uint8_t id
Feature identifier.
Definition: ena.h:418
unsigned int phase
Phase.
Definition: ena.h:651
struct ena_get_stats_req get_stats
Get statistics.
Definition: ena.h:488
static struct net_device_operations ena_operations
ENA network device operations.
Definition: ena.c:1058
#define cpu_to_le32(value)
Definition: byteswap.h:107
struct ena_create_cq_req create_cq
Create completion queue.
Definition: ena.h:480
struct ena_destroy_cq_req destroy_cq
Destroy completion queue.
Definition: ena.h:482
#define ENOTCONN
The socket is not connected.
Definition: errno.h:569
#define DBGC2_HDA(...)
Definition: compiler.h:523
uint8_t direction
Direction.
Definition: ena.h:253
#define ENA_MSIX_NONE
Empty MSI-X vector.
Definition: ena.h:340
uint64_t rsp
Definition: librm.h:267
static void ena_poll(struct net_device *netdev)
Poll for completed and received packets.
Definition: ena.c:1045
struct ena_rx_cqe * rx
Receive completion queue entries.
Definition: ena.h:692
struct ena_cq cq
Completion queue.
Definition: ena.h:736
uint8_t flags
Flags.
Definition: ena.h:619
uint8_t id
Request identifier.
Definition: ena.h:12
A PCI-to-PCI bridge.
Definition: pcibridge.h:17
#define ENA_HOST_INFO_VERSION_WTF
Driver version.
Definition: ena.h:222
uint32_t vector
MSI-X vector.
Definition: ena.h:329
static int ena_create_admin(struct ena_nic *ena)
Create admin queues.
Definition: ena.c:188
uint8_t flags
Flags.
Definition: ena.h:560
#define ENA_ACQ_PHASE
Admin completion queue ownership phase flag.
Definition: ena.h:115
#define PCI_FMT
PCI device debug message format.
Definition: pci.h:307
#define PCI_SLOT(busdevfn)
Definition: pci.h:280
static int ena_create_cq(struct ena_nic *ena, struct ena_cq *cq)
Create completion queue.
Definition: ena.c:541
PCI bus.
A PCI device.
Definition: pci.h:206
int register_netdev(struct net_device *netdev)
Register network device.
Definition: netdevice.c:759
#define ENA_SQE_LAST
This is the last descriptor.
Definition: ena.h:592
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
const char * eth_ntoa(const void *ll_addr)
Transcribe Ethernet address.
Definition: ethernet.c:175
#define ENA_AQ_DB
Admin queue doorbell register.
Definition: ena.h:61
#define ENA_AQ_CAPS
Admin queue capabilities register.
Definition: ena.h:52
static int ena_admin(struct ena_nic *ena, union ena_aq_req *req, union ena_acq_rsp **rsp)
Issue admin queue request.
Definition: ena.c:286
A network device.
Definition: netdevice.h:352
#define ENODEV
No such device.
Definition: errno.h:509
#define ENA_ALIGN
Queue alignment.
Definition: ena.h:19
static void netdev_nullify(struct net_device *netdev)
Stop using a network device.
Definition: netdevice.h:528
uint16_t id
Completion queue identifier.
Definition: ena.h:705
const char product_version[]
Product version string.
Definition: version.c:70
#define ENA_CREATE_SQ
Create submission queue.
Definition: ena.h:246
#define ENA_AQ_PHASE
Admin queue ownership phase flag.
Definition: ena.h:96
#define ENA_BAR_SIZE
BAR size.
Definition: ena.h:16
static void ena_destroy_async(struct ena_nic *ena)
Destroy async event notification queue.
Definition: ena.c:425
uint8_t actual
Actual number of entries.
Definition: ena.h:711
static void ena_poll_rx(struct net_device *netdev)
Poll for received packets.
Definition: ena.c:998
#define ENA_STATS_SCOPE_ETH
Ethernet statistics.
Definition: ena.h:450
#define ETH_ALEN
Definition: if_ether.h:8
A PCI device ID list entry.
Definition: pci.h:170
Version number.
#define le16_to_cpu(value)
Definition: byteswap.h:112
unsigned int uint32_t
Definition: stdint.h:12
uint8_t requested
Requested number of entries.
Definition: ena.h:709
#define ENA_DESTROY_CQ
Destroy completion queue.
Definition: ena.h:359
struct ena_acq acq
Admin completion queue.
Definition: ena.h:748
static struct xen_remove_from_physmap * remove
Definition: xenmem.h:39
uint16_t id
Submission queue identifier.
Definition: ena.h:302
struct io_buffer * rx_iobuf[ENA_RX_COUNT]
Receive I/O buffers, indexed by buffer ID.
Definition: ena.h:762
Use host memory.
Definition: ena.h:273
Memory is contiguous.
Definition: ena.h:275
Network device operations.
Definition: netdevice.h:213
void netdev_rx(struct net_device *netdev, struct io_buffer *iobuf)
Add packet to receive queue.
Definition: netdevice.c:548
struct device * dev
Underlying hardware device.
Definition: netdevice.h:364
Network device management.
unsigned long physaddr_t
Definition: stdint.h:20
uint8_t max
Maximum fill level.
Definition: ena.h:659
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:369
A named feature.
Definition: features.h:78
struct ena_get_feature_req get_feature
Get feature.
Definition: ena.h:484
struct arbelprm_qp_db_record qp
Definition: arbel.h:13
Get statistics response.
Definition: ena.h:456
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
uint32_t busdevfn
Segment, bus, device, and function (bus:dev.fn) number.
Definition: pci.h:233
struct ena_host_info * info
Host info.
Definition: ena.h:744
#define ENA_SET_FEATURE
Set feature.
Definition: ena.h:405
struct ena_sq sq
Submission queue.
Definition: ena.h:734
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
uint64_t rx_bytes
Receive byte count.
Definition: ena.h:464
uint32_t len
Length.
Definition: ena.h:14
uint16_t len
Length.
Definition: ena.h:621
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
uint16_t id
Request identifier.
Definition: ena.h:623
#define DBGC2(...)
Definition: compiler.h:522
Receive submission queue entry.
Definition: ena.h:570
int(* probe)(struct pci_device *pci)
Probe device.
Definition: pci.h:260
#define ENA_DEVICE_MINE
My device.
Definition: ena.h:453
uint64_t address
Address.
Definition: ena.h:566
void * data
Start of data.
Definition: iobuf.h:48
Receive completion queue entry.
Definition: ena.h:615
uint8_t direction
Direction.
Definition: ena.h:304
Admin completion queue response.
Definition: ena.h:494
#define EIO
Input/output error.
Definition: errno.h:433
PCI-to-PCI bridge.
#define ENA_RESET_MAX_WAIT_MS
Maximum time to wait for reset.
Definition: ena.h:77
uint16_t count
Number of entries.
Definition: ena.h:22
struct net_device * alloc_etherdev(size_t priv_size)
Allocate Ethernet device.
Definition: ethernet.c:264
#define PCI_ARGS(pci)
PCI device debug message arguments.
Definition: pci.h:310
uint8_t opcode
Opcode.
Definition: ena.h:90
int pci_write_config_dword(struct pci_device *pci, unsigned int where, uint32_t value)
Write 32-bit dword to PCI configuration space.
#define cpu_to_le16(value)
Definition: byteswap.h:106
union ena_acq_rsp * rsp
Responses.
Definition: ena.h:524
uint64_t tx_bytes
Transmit byte count.
Definition: ena.h:460
uint8_t flags
Flags.
Definition: ena.h:576
uint8_t size
Entry size (in 32-bit words)
Definition: ena.h:16
void iounmap(volatile const void *io_addr)
Unmap I/O address.
#define ENA_SQE_PHASE
Submission queue ownership phase flag.
Definition: ena.h:586
uint16_t len
Length.
Definition: ena.h:556
static int ena_create_async(struct ena_nic *ena)
Create async event notification queue.
Definition: ena.c:386
union ena_feature feature
Feature.
Definition: ena.h:422
uint16_t id
Submission queue identifier.
Definition: ena.h:653
uint8_t scope
Scope.
Definition: ena.h:437
int snprintf(char *buf, size_t size, const char *fmt,...)
Write a formatted string to a buffer.
Definition: vsprintf.c:382
static void free_phys(void *ptr, size_t size)
Free memory allocated with malloc_phys()
Definition: malloc.h:77
static union ena_aq_req * ena_admin_req(struct ena_nic *ena)
Get next available admin queue request.
Definition: ena.c:259
static void ena_set_base(struct ena_nic *ena, unsigned int offset, void *base)
Set queue base address.
Definition: ena.c:138
struct ena_destroy_sq_req destroy_sq
Destroy submission queue.
Definition: ena.h:478
static struct pci_device_id ena_nics[]
ENA PCI device IDs.
Definition: ena.c:1260
struct ena_rx_sqe * rx
Receive submission queue entries.
Definition: ena.h:638
uint8_t fill
Fill level (limited to completion queue size)
Definition: ena.h:661
#define ENA_ADMIN_MAX_WAIT_MS
Maximum time to wait for admin requests.
Definition: ena.h:64
An ENA network card.
Definition: ena.h:740
Admin queue request.
Definition: ena.h:472
struct pci_driver ena_driver __pci_driver
ENA PCI driver.
Definition: ena.c:1266
static const char * ena_direction(unsigned int direction)
Get direction name (for debugging)
Definition: ena.c:54
const char product_name[]
Product name string.
Definition: version.c:73
static int ena_destroy_cq(struct ena_nic *ena, struct ena_cq *cq)
Destroy completion queue.
Definition: ena.c:598
uint8_t id
Request identifier.
Definition: ena.h:86
struct ena_aq_header header
Header.
Definition: ena.h:474
size_t max_pkt_len
Maximum packet length.
Definition: netdevice.h:409
uint64_t index
Index of the first segment within the content.
Definition: pccrc.h:21
Amazon ENA network driver.
#define ENA_DESTROY_SQ
Destroy submission queue.
Definition: ena.h:295
unsigned int prod
Producer counter.
Definition: ena.h:518
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
#define le64_to_cpu(value)
Definition: byteswap.h:114
struct io_buffer * tx_iobuf[ENA_TX_COUNT]
Transmit I/O buffers, indexed by buffer ID.
Definition: ena.h:758
uint64_t rx_packets
Receive packet count.
Definition: ena.h:466
static void ena_clear_caps(struct ena_nic *ena, unsigned int offset)
Clear queue capabilities.
Definition: ena.c:176
#define DBG_LOG
Definition: compiler.h:317
#define ENA_STAT
Device status register.
Definition: ena.h:80
static int ena_membase(struct ena_nic *ena, struct pci_device *pci)
Assign memory BAR.
Definition: ena.c:1084
uint8_t hw_addr[MAX_HW_ADDR_LEN]
Hardware address.
Definition: netdevice.h:381
#define NULL
NULL pointer (VOID *)
Definition: Base.h:321
#define ENA_HOST_INFO_SPEC_2_0
ENA specification version.
Definition: ena.h:225
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:669
Receive.
Definition: ena.h:242
String functions.
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:303
uint8_t direction
Direction.
Definition: ena.h:655
#define ENA_RX_FILL
Receive queue maximum fill level.
Definition: ena.h:37
void * raw
Raw data.
Definition: ena.h:640
struct ena_create_sq_req create_sq
Create submission queue.
Definition: ena.h:476
static int ena_reset(struct ena_nic *ena)
Reset hardware.
Definition: ena.c:104
static int ena_create_sq(struct ena_nic *ena, struct ena_sq *sq, struct ena_cq *cq)
Create submission queue.
Definition: ena.c:445
unsigned int phase
Phase.
Definition: ena.h:528
uint16_t device
Device ID.
Definition: ena.h:443
uint32_t membase
Memory base.
Definition: pcibridge.h:34
void * memset(void *dest, int character, size_t len) __nonnull
A persistent I/O buffer.
Definition: iobuf.h:33