iPXE
thunderx.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdint.h>
27 #include <string.h>
28 #include <strings.h>
29 #include <stdio.h>
30 #include <unistd.h>
31 #include <errno.h>
32 #include <assert.h>
33 #include <byteswap.h>
34 #include <ipxe/netdevice.h>
35 #include <ipxe/ethernet.h>
36 #include <ipxe/if_ether.h>
37 #include <ipxe/iobuf.h>
38 #include <ipxe/malloc.h>
39 #include <ipxe/pci.h>
40 #include <ipxe/pciea.h>
41 #include <ipxe/umalloc.h>
42 #include "thunderx.h"
43 #include "thunderxcfg.h"
44 
45 /** @file
46  *
47  * Cavium ThunderX Ethernet driver
48  *
49  */
50 
51 /** List of BGX Ethernet interfaces */
52 static LIST_HEAD ( txnic_bgxs );
53 
54 /** List of physical functions */
55 static LIST_HEAD ( txnic_pfs );
56 
57 /** Debug colour for physical function and BGX messages */
58 #define TXNICCOL(x) ( &txnic_pfs + (x)->node )
59 
60 /** Board configuration protocol */
63 
64 /******************************************************************************
65  *
66  * Diagnostics
67  *
68  ******************************************************************************
69  */
70 
71 /**
72  * Show virtual NIC diagnostics (for debugging)
73  *
74  * @v vnic Virtual NIC
75  */
76 static __attribute__ (( unused )) void txnic_diag ( struct txnic *vnic ) {
77 
78  DBGC ( vnic, "TXNIC %s SQ %05zx(%05llx)/%05zx(%05llx) %08llx\n",
79  vnic->name,
80  ( ( vnic->sq.prod % TXNIC_SQES ) * TXNIC_SQ_STRIDE ),
81  readq ( vnic->regs + TXNIC_QS_SQ_TAIL(0) ),
82  ( ( vnic->sq.cons % TXNIC_SQES ) * TXNIC_SQ_STRIDE ),
83  readq ( vnic->regs + TXNIC_QS_SQ_HEAD(0) ),
84  readq ( vnic->regs + TXNIC_QS_SQ_STATUS(0) ) );
85  DBGC ( vnic, "TXNIC %s RQ %05zx(%05llx)/%05zx(%05llx) %016llx\n",
86  vnic->name,
87  ( ( vnic->rq.prod % TXNIC_RQES ) * TXNIC_RQ_STRIDE ),
88  readq ( vnic->regs + TXNIC_QS_RBDR_TAIL(0) ),
89  ( ( vnic->rq.cons % TXNIC_RQES ) * TXNIC_RQ_STRIDE ),
90  readq ( vnic->regs + TXNIC_QS_RBDR_HEAD(0) ),
91  readq ( vnic->regs + TXNIC_QS_RBDR_STATUS0(0) ) );
92  DBGC ( vnic, "TXNIC %s CQ xxxxx(%05llx)/%05x(%05llx) %08llx:%08llx\n",
93  vnic->name, readq ( vnic->regs + TXNIC_QS_CQ_TAIL(0) ),
94  ( ( vnic->cq.cons % TXNIC_CQES ) * TXNIC_CQ_STRIDE ),
95  readq ( vnic->regs + TXNIC_QS_CQ_HEAD(0) ),
96  readq ( vnic->regs + TXNIC_QS_CQ_STATUS(0) ),
97  readq ( vnic->regs + TXNIC_QS_CQ_STATUS2(0) ) );
98 }
99 
100 /******************************************************************************
101  *
102  * Send queue
103  *
104  ******************************************************************************
105  */
106 
107 /**
108  * Create send queue
109  *
110  * @v vnic Virtual NIC
111  * @ret rc Return status code
112  */
113 static int txnic_create_sq ( struct txnic *vnic ) {
114 
115  /* Reset send queue */
116  vnic->sq.prod = 0;
117  vnic->sq.cons = 0;
118  writeq ( TXNIC_QS_SQ_CFG_RESET, ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
119 
120  /* Configure and enable send queue */
121  writeq ( user_to_phys ( vnic->sq.sqe, 0 ),
122  ( vnic->regs + TXNIC_QS_SQ_BASE(0) ) );
124  ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
125 
126  DBGC ( vnic, "TXNIC %s SQ at [%08lx,%08lx)\n",
127  vnic->name, user_to_phys ( vnic->sq.sqe, 0 ),
128  user_to_phys ( vnic->sq.sqe, TXNIC_SQ_SIZE ) );
129  return 0;
130 }
131 
132 /**
133  * Disable send queue
134  *
135  * @v vnic Virtual NIC
136  * @ret rc Return status code
137  */
138 static int txnic_disable_sq ( struct txnic *vnic ) {
140  unsigned int i;
141 
142  /* Disable send queue */
143  writeq ( 0, ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
144 
145  /* Wait for send queue to be stopped */
146  for ( i = 0 ; i < TXNIC_SQ_STOP_MAX_WAIT_MS ; i++ ) {
147 
148  /* Check if send queue is stopped */
149  status = readq ( vnic->regs + TXNIC_QS_SQ_STATUS(0) );
151  return 0;
152 
153  /* Delay */
154  mdelay ( 1 );
155  }
156 
157  DBGC ( vnic, "TXNIC %s SQ disable timed out\n", vnic->name );
158  return -ETIMEDOUT;
159 }
160 
161 /**
162  * Destroy send queue
163  *
164  * @v vnic Virtual NIC
165  */
166 static void txnic_destroy_sq ( struct txnic *vnic ) {
167  int rc;
168 
169  /* Disable send queue */
170  if ( ( rc = txnic_disable_sq ( vnic ) ) != 0 ) {
171  /* Nothing else we can do */
172  return;
173  }
174 
175  /* Reset send queue */
176  writeq ( TXNIC_QS_SQ_CFG_RESET, ( vnic->regs + TXNIC_QS_SQ_CFG(0) ) );
177 }
178 
179 /**
180  * Send packet
181  *
182  * @v vnic Virtual NIC
183  * @v iobuf I/O buffer
184  * @ret rc Return status code
185  */
186 static int txnic_send ( struct txnic *vnic, struct io_buffer *iobuf ) {
187  struct txnic_sqe sqe;
188  unsigned int sq_idx;
189  size_t offset;
190  size_t len;
191 
192  /* Get next send queue entry */
193  if ( ( vnic->sq.prod - vnic->sq.cons ) >= TXNIC_SQ_FILL ) {
194  DBGC ( vnic, "TXNIC %s out of send queue entries\n",
195  vnic->name );
196  return -ENOBUFS;
197  }
198  sq_idx = ( vnic->sq.prod++ % TXNIC_SQES );
199  offset = ( sq_idx * TXNIC_SQ_STRIDE );
200 
201  /* Populate send descriptor */
202  len = iob_len ( iobuf );
203  memset ( &sqe, 0, sizeof ( sqe ) );
204  sqe.hdr.total = cpu_to_le32 ( ( len >= ETH_ZLEN ) ? len : ETH_ZLEN );
205  sqe.hdr.subdcnt = ( TXNIC_SQE_SUBDESCS - 1 );
207  sqe.gather.size = cpu_to_le16 ( len );
209  sqe.gather.addr = cpu_to_le64 ( virt_to_bus ( iobuf->data ) );
210  DBGC2 ( vnic, "TXNIC %s SQE %#03x is [%08lx,%08lx)\n",
211  vnic->name, sq_idx, virt_to_bus ( iobuf->data ),
212  ( virt_to_bus ( iobuf->data ) + len ) );
213 
214  /* Copy send descriptor to ring */
215  copy_to_user ( vnic->sq.sqe, offset, &sqe, sizeof ( sqe ) );
216 
217  /* Ring doorbell */
218  wmb();
219  writeq ( TXNIC_SQE_SUBDESCS, ( vnic->regs + TXNIC_QS_SQ_DOOR(0) ) );
220 
221  return 0;
222 }
223 
224 /**
225  * Complete send queue entry
226  *
227  * @v vnic Virtual NIC
228  * @v cqe Send completion queue entry
229  */
230 static void txnic_complete_sqe ( struct txnic *vnic,
231  struct txnic_cqe_send *cqe ) {
232  struct net_device *netdev = vnic->netdev;
233  unsigned int sq_idx;
234  unsigned int status;
235 
236  /* Parse completion */
237  sq_idx = ( le16_to_cpu ( cqe->sqe_ptr ) / TXNIC_SQE_SUBDESCS );
238  status = cqe->send_status;
239 
240  /* Sanity check */
241  assert ( sq_idx == ( vnic->sq.cons % TXNIC_SQES ) );
242 
243  /* Free send queue entry */
244  vnic->sq.cons++;
245 
246  /* Complete transmission */
247  if ( status ) {
248  DBGC ( vnic, "TXNIC %s SQE %#03x complete (status %#02x)\n",
249  vnic->name, sq_idx, status );
251  } else {
252  DBGC2 ( vnic, "TXNIC %s SQE %#03x complete\n",
253  vnic->name, sq_idx );
255  }
256 }
257 
258 /******************************************************************************
259  *
260  * Receive queue
261  *
262  ******************************************************************************
263  */
264 
265 /**
266  * Create receive queue
267  *
268  * @v vnic Virtual NIC
269  * @ret rc Return status code
270  */
271 static int txnic_create_rq ( struct txnic *vnic ) {
272 
273  /* Reset receive buffer descriptor ring */
274  vnic->rq.prod = 0;
275  vnic->rq.cons = 0;
277  ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
278 
279  /* Configure and enable receive buffer descriptor ring */
280  writeq ( user_to_phys ( vnic->rq.rqe, 0 ),
281  ( vnic->regs + TXNIC_QS_RBDR_BASE(0) ) );
284  TXNIC_LINE_SIZE ) ),
285  ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
286 
287  /* Enable receive queue */
288  writeq ( TXNIC_QS_RQ_CFG_ENA, ( vnic->regs + TXNIC_QS_RQ_CFG(0) ) );
289 
290  DBGC ( vnic, "TXNIC %s RQ at [%08lx,%08lx)\n",
291  vnic->name, user_to_phys ( vnic->rq.rqe, 0 ),
292  user_to_phys ( vnic->rq.rqe, TXNIC_RQ_SIZE ) );
293  return 0;
294 }
295 
296 /**
297  * Disable receive queue
298  *
299  * @v vnic Virtual NIC
300  * @ret rc Return status code
301  */
302 static int txnic_disable_rq ( struct txnic *vnic ) {
303  uint64_t cfg;
304  unsigned int i;
305 
306  /* Disable receive queue */
307  writeq ( 0, ( vnic->regs + TXNIC_QS_RQ_CFG(0) ) );
308 
309  /* Wait for receive queue to be disabled */
310  for ( i = 0 ; i < TXNIC_RQ_DISABLE_MAX_WAIT_MS ; i++ ) {
311 
312  /* Check if receive queue is disabled */
313  cfg = readq ( vnic->regs + TXNIC_QS_RQ_CFG(0) );
314  if ( ! ( cfg & TXNIC_QS_RQ_CFG_ENA ) )
315  return 0;
316 
317  /* Delay */
318  mdelay ( 1 );
319  }
320 
321  DBGC ( vnic, "TXNIC %s RQ disable timed out\n", vnic->name );
322  return -ETIMEDOUT;
323 }
324 
325 /**
326  * Destroy receive queue
327  *
328  * @v vnic Virtual NIC
329  */
330 static void txnic_destroy_rq ( struct txnic *vnic ) {
331  unsigned int i;
332  int rc;
333 
334  /* Disable receive queue */
335  if ( ( rc = txnic_disable_rq ( vnic ) ) != 0 ) {
336  /* Leak memory; there's nothing else we can do */
337  return;
338  }
339 
340  /* Disable receive buffer descriptor ring */
341  writeq ( 0, ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
342 
343  /* Reset receive buffer descriptor ring */
345  ( vnic->regs + TXNIC_QS_RBDR_CFG(0) ) );
346 
347  /* Free any unused I/O buffers */
348  for ( i = 0 ; i < TXNIC_RQ_FILL ; i++ ) {
349  if ( vnic->rq.iobuf[i] )
350  free_iob ( vnic->rq.iobuf[i] );
351  vnic->rq.iobuf[i] = NULL;
352  }
353 }
354 
355 /**
356  * Refill receive queue
357  *
358  * @v vnic Virtual NIC
359  */
360 static void txnic_refill_rq ( struct txnic *vnic ) {
361  struct io_buffer *iobuf;
362  struct txnic_rqe rqe;
363  unsigned int rq_idx;
364  unsigned int rq_iobuf_idx;
365  unsigned int refilled = 0;
366  size_t offset;
367 
368  /* Refill ring */
369  while ( ( vnic->rq.prod - vnic->rq.cons ) < TXNIC_RQ_FILL ) {
370 
371  /* Allocate I/O buffer */
372  iobuf = alloc_iob ( TXNIC_RQE_SIZE );
373  if ( ! iobuf ) {
374  /* Wait for next refill */
375  break;
376  }
377 
378  /* Get next receive descriptor */
379  rq_idx = ( vnic->rq.prod++ % TXNIC_RQES );
380  offset = ( rq_idx * TXNIC_RQ_STRIDE );
381 
382  /* Populate receive descriptor */
383  rqe.rbdre.addr = cpu_to_le64 ( virt_to_bus ( iobuf->data ) );
384  DBGC2 ( vnic, "TXNIC %s RQE %#03x is [%08lx,%08lx)\n",
385  vnic->name, rq_idx, virt_to_bus ( iobuf->data ),
386  ( virt_to_bus ( iobuf->data ) + TXNIC_RQE_SIZE ) );
387 
388  /* Copy receive descriptor to ring */
389  copy_to_user ( vnic->rq.rqe, offset, &rqe, sizeof ( rqe ) );
390  refilled++;
391 
392  /* Record I/O buffer */
393  rq_iobuf_idx = ( rq_idx % TXNIC_RQ_FILL );
394  assert ( vnic->rq.iobuf[rq_iobuf_idx] == NULL );
395  vnic->rq.iobuf[rq_iobuf_idx] = iobuf;
396  }
397 
398  /* Ring doorbell */
399  wmb();
400  writeq ( refilled, ( vnic->regs + TXNIC_QS_RBDR_DOOR(0) ) );
401 }
402 
403 /**
404  * Complete receive queue entry
405  *
406  * @v vnic Virtual NIC
407  * @v cqe Receive completion queue entry
408  */
409 static void txnic_complete_rqe ( struct txnic *vnic,
410  struct txnic_cqe_rx *cqe ) {
411  struct net_device *netdev = vnic->netdev;
412  struct io_buffer *iobuf;
413  unsigned int errop;
414  unsigned int rq_idx;
415  unsigned int rq_iobuf_idx;
416  size_t apad_len;
417  size_t len;
418 
419  /* Parse completion */
420  errop = cqe->errop;
421  apad_len = TXNIC_CQE_RX_APAD_LEN ( cqe->apad );
422  len = le16_to_cpu ( cqe->len );
423 
424  /* Get next receive I/O buffer */
425  rq_idx = ( vnic->rq.cons++ % TXNIC_RQES );
426  rq_iobuf_idx = ( rq_idx % TXNIC_RQ_FILL );
427  iobuf = vnic->rq.iobuf[rq_iobuf_idx];
428  vnic->rq.iobuf[rq_iobuf_idx] = NULL;
429 
430  /* Populate I/O buffer */
431  iob_reserve ( iobuf, apad_len );
432  iob_put ( iobuf, len );
433 
434  /* Hand off to network stack */
435  if ( errop ) {
436  DBGC ( vnic, "TXNIC %s RQE %#03x error (length %zd, errop "
437  "%#02x)\n", vnic->name, rq_idx, len, errop );
438  netdev_rx_err ( netdev, iobuf, -EIO );
439  } else {
440  DBGC2 ( vnic, "TXNIC %s RQE %#03x complete (length %zd)\n",
441  vnic->name, rq_idx, len );
442  netdev_rx ( netdev, iobuf );
443  }
444 }
445 
446 /******************************************************************************
447  *
448  * Completion queue
449  *
450  ******************************************************************************
451  */
452 
453 /**
454  * Create completion queue
455  *
456  * @v vnic Virtual NIC
457  * @ret rc Return status code
458  */
459 static int txnic_create_cq ( struct txnic *vnic ) {
460 
461  /* Reset completion queue */
462  vnic->cq.cons = 0;
463  writeq ( TXNIC_QS_CQ_CFG_RESET, ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
464 
465  /* Configure and enable completion queue */
466  writeq ( user_to_phys ( vnic->cq.cqe, 0 ),
467  ( vnic->regs + TXNIC_QS_CQ_BASE(0) ) );
469  ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
470 
471  DBGC ( vnic, "TXNIC %s CQ at [%08lx,%08lx)\n",
472  vnic->name, user_to_phys ( vnic->cq.cqe, 0 ),
473  user_to_phys ( vnic->cq.cqe, TXNIC_CQ_SIZE ) );
474  return 0;
475 }
476 
477 /**
478  * Disable completion queue
479  *
480  * @v vnic Virtual NIC
481  * @ret rc Return status code
482  */
483 static int txnic_disable_cq ( struct txnic *vnic ) {
484  uint64_t cfg;
485  unsigned int i;
486 
487  /* Disable completion queue */
488  writeq ( 0, ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
489 
490  /* Wait for completion queue to be disabled */
491  for ( i = 0 ; i < TXNIC_CQ_DISABLE_MAX_WAIT_MS ; i++ ) {
492 
493  /* Check if completion queue is disabled */
494  cfg = readq ( vnic->regs + TXNIC_QS_CQ_CFG(0) );
495  if ( ! ( cfg & TXNIC_QS_CQ_CFG_ENA ) )
496  return 0;
497 
498  /* Delay */
499  mdelay ( 1 );
500  }
501 
502  DBGC ( vnic, "TXNIC %s CQ disable timed out\n", vnic->name );
503  return -ETIMEDOUT;
504 }
505 
506 /**
507  * Destroy completion queue
508  *
509  * @v vnic Virtual NIC
510  */
511 static void txnic_destroy_cq ( struct txnic *vnic ) {
512  int rc;
513 
514  /* Disable completion queue */
515  if ( ( rc = txnic_disable_cq ( vnic ) ) != 0 ) {
516  /* Leak memory; there's nothing else we can do */
517  return;
518  }
519 
520  /* Reset completion queue */
521  writeq ( TXNIC_QS_CQ_CFG_RESET, ( vnic->regs + TXNIC_QS_CQ_CFG(0) ) );
522 }
523 
524 /**
525  * Poll completion queue
526  *
527  * @v vnic Virtual NIC
528  */
529 static void txnic_poll_cq ( struct txnic *vnic ) {
530  union txnic_cqe cqe;
532  size_t offset;
533  unsigned int qcount;
534  unsigned int cq_idx;
535  unsigned int i;
536 
537  /* Get number of completions */
538  status = readq ( vnic->regs + TXNIC_QS_CQ_STATUS(0) );
539  qcount = TXNIC_QS_CQ_STATUS_QCOUNT ( status );
540  if ( ! qcount )
541  return;
542 
543  /* Process completion queue entries */
544  for ( i = 0 ; i < qcount ; i++ ) {
545 
546  /* Get completion queue entry */
547  cq_idx = ( vnic->cq.cons++ % TXNIC_CQES );
548  offset = ( cq_idx * TXNIC_CQ_STRIDE );
549  copy_from_user ( &cqe, vnic->cq.cqe, offset, sizeof ( cqe ) );
550 
551  /* Process completion queue entry */
552  switch ( cqe.common.cqe_type ) {
553  case TXNIC_CQE_TYPE_SEND:
554  txnic_complete_sqe ( vnic, &cqe.send );
555  break;
556  case TXNIC_CQE_TYPE_RX:
557  txnic_complete_rqe ( vnic, &cqe.rx );
558  break;
559  default:
560  DBGC ( vnic, "TXNIC %s unknown completion type %d\n",
561  vnic->name, cqe.common.cqe_type );
562  DBGC_HDA ( vnic, user_to_phys ( vnic->cq.cqe, offset ),
563  &cqe, sizeof ( cqe ) );
564  break;
565  }
566  }
567 
568  /* Ring doorbell */
569  writeq ( qcount, ( vnic->regs + TXNIC_QS_CQ_DOOR(0) ) );
570 }
571 
572 /******************************************************************************
573  *
574  * Virtual NIC
575  *
576  ******************************************************************************
577  */
578 
579 /**
580  * Open virtual NIC
581  *
582  * @v vnic Virtual NIC
583  * @ret rc Return status code
584  */
585 static int txnic_open ( struct txnic *vnic ) {
586  int rc;
587 
588  /* Create completion queue */
589  if ( ( rc = txnic_create_cq ( vnic ) ) != 0 )
590  goto err_create_cq;
591 
592  /* Create send queue */
593  if ( ( rc = txnic_create_sq ( vnic ) ) != 0 )
594  goto err_create_sq;
595 
596  /* Create receive queue */
597  if ( ( rc = txnic_create_rq ( vnic ) ) != 0 )
598  goto err_create_rq;
599 
600  /* Refill receive queue */
601  txnic_refill_rq ( vnic );
602 
603  return 0;
604 
605  txnic_destroy_rq ( vnic );
606  err_create_rq:
607  txnic_destroy_sq ( vnic );
608  err_create_sq:
609  txnic_destroy_cq ( vnic );
610  err_create_cq:
611  return rc;
612 }
613 
614 /**
615  * Close virtual NIC
616  *
617  * @v vnic Virtual NIC
618  */
619 static void txnic_close ( struct txnic *vnic ) {
620 
621  /* Destroy receive queue */
622  txnic_destroy_rq ( vnic );
623 
624  /* Destroy send queue */
625  txnic_destroy_sq ( vnic );
626 
627  /* Destroy completion queue */
628  txnic_destroy_cq ( vnic );
629 }
630 
631 /**
632  * Poll virtual NIC
633  *
634  * @v vnic Virtual NIC
635  */
636 static void txnic_poll ( struct txnic *vnic ) {
637 
638  /* Poll completion queue */
639  txnic_poll_cq ( vnic );
640 
641  /* Refill receive queue */
642  txnic_refill_rq ( vnic );
643 }
644 
645 /**
646  * Allocate virtual NIC
647  *
648  * @v pci Underlying PCI device
649  * @v membase Register base address
650  * @ret vnic Virtual NIC, or NULL on failure
651  */
652 static struct txnic * txnic_alloc ( struct pci_device *pci,
653  unsigned long membase ) {
654  struct net_device *netdev;
655  struct txnic *vnic;
656 
657  /* Allocate network device */
658  netdev = alloc_etherdev ( sizeof ( *vnic ) );
659  if ( ! netdev )
660  goto err_alloc_netdev;
661  netdev->dev = &pci->dev;
662  vnic = netdev->priv;
663  vnic->netdev = netdev;
664  vnic->name = pci->dev.name;
665 
666  /* Allow caller to reuse netdev->priv. (The generic virtual
667  * NIC code never assumes that netdev->priv==vnic.)
668  */
669  netdev->priv = NULL;
670 
671  /* Allocate completion queue */
672  vnic->cq.cqe = umalloc ( TXNIC_CQ_SIZE );
673  if ( ! vnic->cq.cqe )
674  goto err_alloc_cq;
675 
676  /* Allocate send queue */
677  vnic->sq.sqe = umalloc ( TXNIC_SQ_SIZE );
678  if ( ! vnic->sq.sqe )
679  goto err_alloc_sq;
680 
681  /* Allocate receive queue */
682  vnic->rq.rqe = umalloc ( TXNIC_RQ_SIZE );
683  if ( ! vnic->rq.rqe )
684  goto err_alloc_rq;
685 
686  /* Map registers */
687  vnic->regs = pci_ioremap ( pci, membase, TXNIC_VF_BAR_SIZE );
688  if ( ! vnic->regs )
689  goto err_ioremap;
690 
691  return vnic;
692 
693  iounmap ( vnic->regs );
694  err_ioremap:
695  ufree ( vnic->rq.rqe );
696  err_alloc_rq:
697  ufree ( vnic->sq.sqe );
698  err_alloc_sq:
699  ufree ( vnic->cq.cqe );
700  err_alloc_cq:
702  netdev_put ( netdev );
703  err_alloc_netdev:
704  return NULL;
705 }
706 
707 /**
708  * Free virtual NIC
709  *
710  * @v vnic Virtual NIC
711  */
712 static void txnic_free ( struct txnic *vnic ) {
713  struct net_device *netdev = vnic->netdev;
714 
715  /* Unmap registers */
716  iounmap ( vnic->regs );
717 
718  /* Free receive queue */
719  ufree ( vnic->rq.rqe );
720 
721  /* Free send queue */
722  ufree ( vnic->sq.sqe );
723 
724  /* Free completion queue */
725  ufree ( vnic->cq.cqe );
726 
727  /* Free network device */
729  netdev_put ( netdev );
730 }
731 
732 /******************************************************************************
733  *
734  * Logical MAC virtual NICs
735  *
736  ******************************************************************************
737  */
738 
739 /**
740  * Show LMAC diagnostics (for debugging)
741  *
742  * @v lmac Logical MAC
743  */
744 static __attribute__ (( unused )) void
745 txnic_lmac_diag ( struct txnic_lmac *lmac ) {
746  struct txnic *vnic = lmac->vnic;
749  uint64_t br_status1;
750  uint64_t br_status2;
751  uint64_t br_algn_status;
752  uint64_t br_pmd_status;
753  uint64_t an_status;
754 
755  /* Read status (clearing latching bits) */
758  status1 = readq ( lmac->regs + BGX_SPU_STATUS1 );
759  status2 = readq ( lmac->regs + BGX_SPU_STATUS2 );
760  DBGC ( vnic, "TXNIC %s SPU %02llx:%04llx%s%s%s\n",
761  vnic->name, status1, status2,
762  ( ( status1 & BGX_SPU_STATUS1_FLT ) ? " FLT" : "" ),
763  ( ( status1 & BGX_SPU_STATUS1_RCV_LNK ) ? " RCV_LNK" : "" ),
764  ( ( status2 & BGX_SPU_STATUS2_RCVFLT ) ? " RCVFLT" : "" ) );
765 
766  /* Read BASE-R status (clearing latching bits) */
769  ( lmac->regs + BGX_SPU_BR_STATUS2 ) );
770  br_status1 = readq ( lmac->regs + BGX_SPU_BR_STATUS1 );
771  br_status2 = readq ( lmac->regs + BGX_SPU_BR_STATUS2 );
772  DBGC ( vnic, "TXNIC %s BR %04llx:%04llx%s%s%s%s%s\n",
773  vnic->name, br_status2, br_status2,
774  ( ( br_status1 & BGX_SPU_BR_STATUS1_RCV_LNK ) ? " RCV_LNK" : ""),
775  ( ( br_status1 & BGX_SPU_BR_STATUS1_HI_BER ) ? " HI_BER" : "" ),
776  ( ( br_status1 & BGX_SPU_BR_STATUS1_BLK_LOCK ) ?
777  " BLK_LOCK" : "" ),
778  ( ( br_status2 & BGX_SPU_BR_STATUS2_LATCHED_LOCK ) ?
779  " LATCHED_LOCK" : "" ),
780  ( ( br_status2 & BGX_SPU_BR_STATUS2_LATCHED_BER ) ?
781  " LATCHED_BER" : "" ) );
782 
783  /* Read BASE-R alignment status */
784  br_algn_status = readq ( lmac->regs + BGX_SPU_BR_ALGN_STATUS );
785  DBGC ( vnic, "TXNIC %s BR ALGN %016llx%s\n", vnic->name, br_algn_status,
786  ( ( br_algn_status & BGX_SPU_BR_ALGN_STATUS_ALIGND ) ?
787  " ALIGND" : "" ) );
788 
789  /* Read BASE-R link training status */
790  br_pmd_status = readq ( lmac->regs + BGX_SPU_BR_PMD_STATUS );
791  DBGC ( vnic, "TXNIC %s BR PMD %04llx\n", vnic->name, br_pmd_status );
792 
793  /* Read autonegotiation status (clearing latching bits) */
795  ( lmac->regs + BGX_SPU_AN_STATUS ) );
796  an_status = readq ( lmac->regs + BGX_SPU_AN_STATUS );
797  DBGC ( vnic, "TXNIC %s BR AN %04llx%s%s%s%s%s\n", vnic->name, an_status,
798  ( ( an_status & BGX_SPU_AN_STATUS_XNP_STAT ) ? " XNP_STAT" : ""),
799  ( ( an_status & BGX_SPU_AN_STATUS_PAGE_RX ) ? " PAGE_RX" : "" ),
800  ( ( an_status & BGX_SPU_AN_STATUS_AN_COMPLETE ) ?
801  " AN_COMPLETE" : "" ),
802  ( ( an_status & BGX_SPU_AN_STATUS_LINK_STATUS ) ?
803  " LINK_STATUS" : "" ),
804  ( ( an_status & BGX_SPU_AN_STATUS_LP_AN_ABLE ) ?
805  " LP_AN_ABLE" : "" ) );
806 
807  /* Read transmit statistics */
808  DBGC ( vnic, "TXNIC %s TXF xc %#llx xd %#llx mc %#llx sc %#llx ok "
809  "%#llx bc %#llx mc %#llx un %#llx pa %#llx\n", vnic->name,
810  readq ( lmac->regs + BGX_CMR_TX_STAT0 ),
811  readq ( lmac->regs + BGX_CMR_TX_STAT1 ),
812  readq ( lmac->regs + BGX_CMR_TX_STAT2 ),
813  readq ( lmac->regs + BGX_CMR_TX_STAT3 ),
814  readq ( lmac->regs + BGX_CMR_TX_STAT5 ),
815  readq ( lmac->regs + BGX_CMR_TX_STAT14 ),
816  readq ( lmac->regs + BGX_CMR_TX_STAT15 ),
817  readq ( lmac->regs + BGX_CMR_TX_STAT16 ),
818  readq ( lmac->regs + BGX_CMR_TX_STAT17 ) );
819  DBGC ( vnic, "TXNIC %s TXB ok %#llx hist %#llx:%#llx:%#llx:%#llx:"
820  "%#llx:%#llx:%#llx:%#llx\n", vnic->name,
821  readq ( lmac->regs + BGX_CMR_TX_STAT4 ),
822  readq ( lmac->regs + BGX_CMR_TX_STAT6 ),
823  readq ( lmac->regs + BGX_CMR_TX_STAT7 ),
824  readq ( lmac->regs + BGX_CMR_TX_STAT8 ),
825  readq ( lmac->regs + BGX_CMR_TX_STAT9 ),
826  readq ( lmac->regs + BGX_CMR_TX_STAT10 ),
827  readq ( lmac->regs + BGX_CMR_TX_STAT11 ),
828  readq ( lmac->regs + BGX_CMR_TX_STAT12 ),
829  readq ( lmac->regs + BGX_CMR_TX_STAT13 ) );
830 
831  /* Read receive statistics */
832  DBGC ( vnic, "TXNIC %s RXF ok %#llx pa %#llx nm %#llx ov %#llx er "
833  "%#llx nc %#llx\n", vnic->name,
834  readq ( lmac->regs + BGX_CMR_RX_STAT0 ),
835  readq ( lmac->regs + BGX_CMR_RX_STAT2 ),
836  readq ( lmac->regs + BGX_CMR_RX_STAT4 ),
837  readq ( lmac->regs + BGX_CMR_RX_STAT6 ),
838  readq ( lmac->regs + BGX_CMR_RX_STAT8 ),
839  readq ( lmac->regs + BGX_CMR_RX_STAT9 ) );
840  DBGC ( vnic, "TXNIC %s RXB ok %#llx pa %#llx nm %#llx ov %#llx nc "
841  "%#llx\n", vnic->name,
842  readq ( lmac->regs + BGX_CMR_RX_STAT1 ),
843  readq ( lmac->regs + BGX_CMR_RX_STAT3 ),
844  readq ( lmac->regs + BGX_CMR_RX_STAT5 ),
845  readq ( lmac->regs + BGX_CMR_RX_STAT7 ),
846  readq ( lmac->regs + BGX_CMR_RX_STAT10 ) );
847 }
848 
849 /**
850  * Update LMAC link state
851  *
852  * @v lmac Logical MAC
853  */
854 static void txnic_lmac_update_link ( struct txnic_lmac *lmac ) {
855  struct txnic *vnic = lmac->vnic;
856  struct net_device *netdev = vnic->netdev;
858 
859  /* Read status (clearing latching bits) */
861  status1 = readq ( lmac->regs + BGX_SPU_STATUS1 );
862 
863  /* Report link status */
866  } else {
868  }
869 }
870 
871 /**
872  * Poll LMAC link state
873  *
874  * @v lmac Logical MAC
875  */
876 static void txnic_lmac_poll_link ( struct txnic_lmac *lmac ) {
877  struct txnic *vnic = lmac->vnic;
878  uint64_t intr;
879 
880  /* Get interrupt status */
881  intr = readq ( lmac->regs + BGX_SPU_INT );
882  if ( ! intr )
883  return;
884  DBGC ( vnic, "TXNIC %s INT %04llx%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
885  vnic->name, intr,
886  ( ( intr & BGX_SPU_INT_TRAINING_FAIL ) ? " TRAINING_FAIL" : "" ),
887  ( ( intr & BGX_SPU_INT_TRAINING_DONE ) ? " TRAINING_DONE" : "" ),
888  ( ( intr & BGX_SPU_INT_AN_COMPLETE ) ? " AN_COMPLETE" : "" ),
889  ( ( intr & BGX_SPU_INT_AN_LINK_GOOD ) ? " AN_LINK_GOOD" : "" ),
890  ( ( intr & BGX_SPU_INT_AN_PAGE_RX ) ? " AN_PAGE_RX" : "" ),
891  ( ( intr & BGX_SPU_INT_FEC_UNCORR ) ? " FEC_UNCORR" : "" ),
892  ( ( intr & BGX_SPU_INT_FEC_CORR ) ? " FEC_CORR" : "" ),
893  ( ( intr & BGX_SPU_INT_BIP_ERR ) ? " BIP_ERR" : "" ),
894  ( ( intr & BGX_SPU_INT_DBG_SYNC ) ? " DBG_SYNC" : "" ),
895  ( ( intr & BGX_SPU_INT_ALGNLOS ) ? " ALGNLOS" : "" ),
896  ( ( intr & BGX_SPU_INT_SYNLOS ) ? " SYNLOS" : "" ),
897  ( ( intr & BGX_SPU_INT_BITLCKLS ) ? " BITLCKLS" : "" ),
898  ( ( intr & BGX_SPU_INT_ERR_BLK ) ? " ERR_BLK" : "" ),
899  ( ( intr & BGX_SPU_INT_RX_LINK_DOWN ) ? " RX_LINK_DOWN" : "" ),
900  ( ( intr & BGX_SPU_INT_RX_LINK_UP ) ? " RX_LINK_UP" : "" ) );
901 
902  /* Clear interrupt status */
903  writeq ( intr, ( lmac->regs + BGX_SPU_INT ) );
904 
905  /* Update link state */
906  txnic_lmac_update_link ( lmac );
907 }
908 
909 /**
910  * Reset LMAC
911  *
912  * @v lmac Logical MAC
913  */
914 static void txnic_lmac_reset ( struct txnic_lmac *lmac ) {
915  struct txnic_bgx *bgx = lmac->bgx;
916  struct txnic_pf *pf = bgx->pf;
917  void *qsregs = ( pf->regs + TXNIC_PF_QS ( lmac->idx ) );
918 
919  /* There is no reset available for the physical function
920  * aspects of a virtual NIC; we have to explicitly reload a
921  * sensible set of default values.
922  */
923  writeq ( 0, ( qsregs + TXNIC_PF_QS_CFG ) );
924  writeq ( 0, ( qsregs + TXNIC_PF_QS_RQ_CFG(0) ) );
925  writeq ( 0, ( qsregs + TXNIC_PF_QS_RQ_DROP_CFG(0) ) );
926  writeq ( 0, ( qsregs + TXNIC_PF_QS_RQ_BP_CFG(0) ) );
927  writeq ( 0, ( qsregs + TXNIC_PF_QS_SQ_CFG(0) ) );
928 }
929 
930 /**
931  * Open network device
932  *
933  * @v netdev Network device
934  * @ret rc Return status code
935  */
936 static int txnic_lmac_open ( struct net_device *netdev ) {
937  struct txnic_lmac *lmac = netdev->priv;
938  struct txnic_bgx *bgx = lmac->bgx;
939  struct txnic_pf *pf = bgx->pf;
940  struct txnic *vnic = lmac->vnic;
941  unsigned int vnic_idx = lmac->idx;
942  unsigned int chan_idx = TXNIC_CHAN_IDX ( vnic_idx );
943  unsigned int tl4_idx = TXNIC_TL4_IDX ( vnic_idx );
944  unsigned int tl3_idx = TXNIC_TL3_IDX ( vnic_idx );
945  unsigned int tl2_idx = TXNIC_TL2_IDX ( vnic_idx );
946  void *lmregs = ( pf->regs + TXNIC_PF_LMAC ( vnic_idx ) );
947  void *chregs = ( pf->regs + TXNIC_PF_CHAN ( chan_idx ) );
948  void *qsregs = ( pf->regs + TXNIC_PF_QS ( vnic_idx ) );
949  size_t max_pkt_size;
950  int rc;
951 
952  /* Configure channel/match parse indices */
953  writeq ( ( TXNIC_PF_MPI_CFG_VNIC ( vnic_idx ) |
954  TXNIC_PF_MPI_CFG_RSSI_BASE ( vnic_idx ) ),
955  ( TXNIC_PF_MPI_CFG ( vnic_idx ) + pf->regs ) );
956  writeq ( ( TXNIC_PF_RSSI_RQ_RQ_QS ( vnic_idx ) ),
957  ( TXNIC_PF_RSSI_RQ ( vnic_idx ) + pf->regs ) );
958 
959  /* Configure LMAC */
960  max_pkt_size = ( netdev->max_pkt_len + 4 /* possible VLAN */ );
963  ( TXNIC_PF_LMAC_CFG + lmregs ) );
964  writeq ( ( TXNIC_PF_LMAC_CFG2_MAX_PKT_SIZE ( max_pkt_size ) ),
965  ( TXNIC_PF_LMAC_CFG2 + lmregs ) );
969  ( TXNIC_PF_LMAC_CREDIT + lmregs ) );
970 
971  /* Configure channels */
973  ( TXNIC_PF_CHAN_TX_CFG + chregs ) );
974  writeq ( ( TXNIC_PF_CHAN_RX_CFG_CPI_BASE ( vnic_idx ) ),
975  ( TXNIC_PF_CHAN_RX_CFG + chregs ) );
977  TXNIC_PF_CHAN_RX_BP_CFG_BPID ( vnic_idx ) ),
978  ( TXNIC_PF_CHAN_RX_BP_CFG + chregs ) );
979 
980  /* Configure traffic limiters */
982  ( TXNIC_PF_TL2_CFG ( tl2_idx ) + pf->regs ) );
984  ( TXNIC_PF_TL3_CFG ( tl3_idx ) + pf->regs ) );
985  writeq ( ( TXNIC_PF_TL3_CHAN_CHAN ( chan_idx ) ),
986  ( TXNIC_PF_TL3_CHAN ( tl3_idx ) + pf->regs ) );
987  writeq ( ( TXNIC_PF_TL4_CFG_SQ_QS ( vnic_idx ) |
989  ( TXNIC_PF_TL4_CFG ( tl4_idx ) + pf->regs ) );
990 
991  /* Configure send queue */
992  writeq ( ( TXNIC_PF_QS_SQ_CFG_CQ_QS ( vnic_idx ) ),
993  ( TXNIC_PF_QS_SQ_CFG(0) + qsregs ) );
994  writeq ( ( TXNIC_PF_QS_SQ_CFG2_TL4 ( tl4_idx ) ),
995  ( TXNIC_PF_QS_SQ_CFG2(0) + qsregs ) );
996 
997  /* Configure receive queue */
999  TXNIC_PF_QS_RQ_CFG_CQ_QS ( vnic_idx ) |
1000  TXNIC_PF_QS_RQ_CFG_RBDR_CONT_QS ( vnic_idx ) |
1001  TXNIC_PF_QS_RQ_CFG_RBDR_STRT_QS ( vnic_idx ) ),
1002  ( TXNIC_PF_QS_RQ_CFG(0) + qsregs ) );
1005  TXNIC_PF_QS_RQ_BP_CFG_BPID ( vnic_idx ) ),
1006  ( TXNIC_PF_QS_RQ_BP_CFG(0) + qsregs ) );
1007 
1008  /* Enable queue set */
1009  writeq ( ( TXNIC_PF_QS_CFG_ENA | TXNIC_PF_QS_CFG_VNIC ( vnic_idx ) ),
1010  ( TXNIC_PF_QS_CFG + qsregs ) );
1011 
1012  /* Open virtual NIC */
1013  if ( ( rc = txnic_open ( vnic ) ) != 0 )
1014  goto err_open;
1015 
1016  /* Update link state */
1017  txnic_lmac_update_link ( lmac );
1018 
1019  return 0;
1020 
1021  txnic_close ( vnic );
1022  err_open:
1023  writeq ( 0, ( qsregs + TXNIC_PF_QS_CFG ) );
1024  return rc;
1025 }
1026 
1027 /**
1028  * Close network device
1029  *
1030  * @v netdev Network device
1031  */
1032 static void txnic_lmac_close ( struct net_device *netdev ) {
1033  struct txnic_lmac *lmac = netdev->priv;
1034  struct txnic_bgx *bgx = lmac->bgx;
1035  struct txnic_pf *pf = bgx->pf;
1036  struct txnic *vnic = lmac->vnic;
1037  void *qsregs = ( pf->regs + TXNIC_PF_QS ( lmac->idx ) );
1038 
1039  /* Close virtual NIC */
1040  txnic_close ( vnic );
1041 
1042  /* Disable queue set */
1043  writeq ( 0, ( qsregs + TXNIC_PF_QS_CFG ) );
1044 }
1045 
1046 /**
1047  * Transmit packet
1048  *
1049  * @v netdev Network device
1050  * @v iobuf I/O buffer
1051  * @ret rc Return status code
1052  */
1054  struct io_buffer *iobuf ) {
1055  struct txnic_lmac *lmac = netdev->priv;
1056  struct txnic *vnic = lmac->vnic;
1057 
1058  return txnic_send ( vnic, iobuf );
1059 }
1060 
1061 /**
1062  * Poll network device
1063  *
1064  * @v netdev Network device
1065  */
1066 static void txnic_lmac_poll ( struct net_device *netdev ) {
1067  struct txnic_lmac *lmac = netdev->priv;
1068  struct txnic *vnic = lmac->vnic;
1069 
1070  /* Poll virtual NIC */
1071  txnic_poll ( vnic );
1072 
1073  /* Poll link state */
1074  txnic_lmac_poll_link ( lmac );
1075 }
1076 
1077 /** Network device operations */
1079  .open = txnic_lmac_open,
1080  .close = txnic_lmac_close,
1081  .transmit = txnic_lmac_transmit,
1082  .poll = txnic_lmac_poll,
1083 };
1084 
1085 /**
1086  * Probe logical MAC virtual NIC
1087  *
1088  * @v lmac Logical MAC
1089  * @ret rc Return status code
1090  */
1091 static int txnic_lmac_probe ( struct txnic_lmac *lmac ) {
1092  struct txnic_bgx *bgx = lmac->bgx;
1093  struct txnic_pf *pf = bgx->pf;
1094  struct txnic *vnic;
1095  struct net_device *netdev;
1096  unsigned long membase;
1097  int rc;
1098 
1099  /* Sanity check */
1100  assert ( lmac->vnic == NULL );
1101 
1102  /* Calculate register base address */
1103  membase = ( pf->vf_membase + ( lmac->idx * pf->vf_stride ) );
1104 
1105  /* Allocate and initialise network device */
1106  vnic = txnic_alloc ( bgx->pci, membase );
1107  if ( ! vnic ) {
1108  rc = -ENOMEM;
1109  goto err_alloc;
1110  }
1111  netdev = vnic->netdev;
1113  netdev->priv = lmac;
1114  lmac->vnic = vnic;
1115 
1116  /* Reset device */
1117  txnic_lmac_reset ( lmac );
1118 
1119  /* Set MAC address */
1120  memcpy ( netdev->hw_addr, lmac->mac.raw, ETH_ALEN );
1121 
1122  /* Register network device */
1123  if ( ( rc = register_netdev ( netdev ) ) != 0 )
1124  goto err_register;
1125  vnic->name = netdev->name;
1126  DBGC ( TXNICCOL ( pf ), "TXNIC %d/%d/%d is %s (%s)\n", pf->node,
1127  bgx->idx, lmac->idx, vnic->name, eth_ntoa ( lmac->mac.raw ) );
1128 
1129  /* Update link state */
1130  txnic_lmac_update_link ( lmac );
1131 
1132  return 0;
1133 
1135  err_register:
1136  txnic_lmac_reset ( lmac );
1137  txnic_free ( vnic );
1138  lmac->vnic = NULL;
1139  err_alloc:
1140  return rc;
1141 }
1142 
1143 /**
1144  * Remove logical MAC virtual NIC
1145  *
1146  * @v lmac Logical MAC
1147  */
1148 static void txnic_lmac_remove ( struct txnic_lmac *lmac ) {
1149  uint64_t config;
1150 
1151  /* Sanity check */
1152  assert ( lmac->vnic != NULL );
1153 
1154  /* Disable packet receive and transmit */
1155  config = readq ( lmac->regs + BGX_CMR_CONFIG );
1156  config &= ~( BGX_CMR_CONFIG_DATA_PKT_TX_EN |
1158  writeq ( config, ( lmac->regs + BGX_CMR_CONFIG ) );
1159 
1160  /* Unregister network device */
1161  unregister_netdev ( lmac->vnic->netdev );
1162 
1163  /* Reset device */
1164  txnic_lmac_reset ( lmac );
1165 
1166  /* Free virtual NIC */
1167  txnic_free ( lmac->vnic );
1168  lmac->vnic = NULL;
1169 }
1170 
1171 /**
1172  * Probe all LMACs on a BGX Ethernet interface
1173  *
1174  * @v pf Physical function
1175  * @v bgx BGX Ethernet interface
1176  * @ret rc Return status code
1177  */
1178 static int txnic_lmac_probe_all ( struct txnic_pf *pf, struct txnic_bgx *bgx ) {
1179  unsigned int bgx_idx;
1180  int lmac_idx;
1181  int count;
1182  int rc;
1183 
1184  /* Sanity checks */
1185  bgx_idx = bgx->idx;
1186  assert ( pf->node == bgx->node );
1187  assert ( pf->bgx[bgx_idx] == NULL );
1188  assert ( bgx->pf == NULL );
1189 
1190  /* Associate BGX with physical function */
1191  pf->bgx[bgx_idx] = bgx;
1192  bgx->pf = pf;
1193 
1194  /* Probe all LMACs */
1195  count = bgx->count;
1196  for ( lmac_idx = 0 ; lmac_idx < count ; lmac_idx++ ) {
1197  if ( ( rc = txnic_lmac_probe ( &bgx->lmac[lmac_idx] ) ) != 0 )
1198  goto err_probe;
1199  }
1200 
1201  return 0;
1202 
1203  lmac_idx = count;
1204  err_probe:
1205  for ( lmac_idx-- ; lmac_idx >= 0 ; lmac_idx-- )
1206  txnic_lmac_remove ( &bgx->lmac[lmac_idx] );
1207  pf->bgx[bgx_idx] = NULL;
1208  bgx->pf = NULL;
1209  return rc;
1210 }
1211 
1212 /**
1213  * Remove all LMACs on a BGX Ethernet interface
1214  *
1215  * @v pf Physical function
1216  * @v bgx BGX Ethernet interface
1217  */
1218 static void txnic_lmac_remove_all ( struct txnic_pf *pf,
1219  struct txnic_bgx *bgx ) {
1220  unsigned int lmac_idx;
1221 
1222  /* Sanity checks */
1223  assert ( pf->bgx[bgx->idx] == bgx );
1224  assert ( bgx->pf == pf );
1225 
1226  /* Remove all LMACs */
1227  for ( lmac_idx = 0 ; lmac_idx < bgx->count ; lmac_idx++ )
1228  txnic_lmac_remove ( &bgx->lmac[lmac_idx] );
1229 
1230  /* Disassociate BGX from physical function */
1231  pf->bgx[bgx->idx] = NULL;
1232  bgx->pf = NULL;
1233 }
1234 
1235 /******************************************************************************
1236  *
1237  * NIC physical function interface
1238  *
1239  ******************************************************************************
1240  */
1241 
1242 /**
1243  * Probe PCI device
1244  *
1245  * @v pci PCI device
1246  * @ret rc Return status code
1247  */
1248 static int txnic_pf_probe ( struct pci_device *pci ) {
1249  struct txnic_pf *pf;
1250  struct txnic_bgx *bgx;
1251  unsigned long membase;
1252  unsigned int i;
1253  int rc;
1254 
1255  /* Allocate and initialise structure */
1256  pf = zalloc ( sizeof ( *pf ) );
1257  if ( ! pf ) {
1258  rc = -ENOMEM;
1259  goto err_alloc;
1260  }
1261  pf->pci = pci;
1262  pci_set_drvdata ( pci, pf );
1263 
1264  /* Get base addresses */
1265  membase = pciea_bar_start ( pci, PCIEA_BEI_BAR_0 );
1268 
1269  /* Calculate node ID */
1270  pf->node = txnic_address_node ( membase );
1271  DBGC ( TXNICCOL ( pf ), "TXNIC %d/*/* PF %s at %#lx (VF %#lx+%#lx)\n",
1272  pf->node, pci->dev.name, membase, pf->vf_membase, pf->vf_stride);
1273 
1274  /* Fix up PCI device */
1275  adjust_pci_device ( pci );
1276 
1277  /* Map registers */
1278  pf->regs = pci_ioremap ( pci, membase, TXNIC_PF_BAR_SIZE );
1279  if ( ! pf->regs ) {
1280  rc = -ENODEV;
1281  goto err_ioremap;
1282  }
1283 
1284  /* Configure physical function */
1288  ( pf->regs + TXNIC_PF_BP_CFG ) );
1289  for ( i = 0 ; i < TXNIC_NUM_BGX ; i++ ) {
1292  ( pf->regs + TXNIC_PF_INTF_SEND_CFG ( i ) ) );
1296  ( pf->regs + TXNIC_PF_INTF_BP_CFG ( i ) ) );
1297  }
1301  ( pf->regs + TXNIC_PF_PKIND_CFG(0) ) );
1302 
1303  /* Add to list of physical functions */
1304  list_add_tail ( &pf->list, &txnic_pfs );
1305 
1306  /* Probe all LMACs, if applicable */
1307  list_for_each_entry ( bgx, &txnic_bgxs, list ) {
1308  if ( bgx->node != pf->node )
1309  continue;
1310  if ( ( rc = txnic_lmac_probe_all ( pf, bgx ) ) != 0 )
1311  goto err_probe;
1312  }
1313 
1314  return 0;
1315 
1316  err_probe:
1317  for ( i = 0 ; i < TXNIC_NUM_BGX ; i++ ) {
1318  if ( pf->bgx[i] )
1319  txnic_lmac_remove_all ( pf, pf->bgx[i] );
1320  }
1321  list_del ( &pf->list );
1322  writeq ( 0, ( pf->regs + TXNIC_PF_CFG ) );
1323  iounmap ( pf->regs );
1324  err_ioremap:
1325  free ( pf );
1326  err_alloc:
1327  return rc;
1328 }
1329 
1330 /**
1331  * Remove PCI device
1332  *
1333  * @v pci PCI device
1334  */
1335 static void txnic_pf_remove ( struct pci_device *pci ) {
1336  struct txnic_pf *pf = pci_get_drvdata ( pci );
1337  unsigned int i;
1338 
1339  /* Remove all LMACs, if applicable */
1340  for ( i = 0 ; i < TXNIC_NUM_BGX ; i++ ) {
1341  if ( pf->bgx[i] )
1342  txnic_lmac_remove_all ( pf, pf->bgx[i] );
1343  }
1344 
1345  /* Remove from list of physical functions */
1346  list_del ( &pf->list );
1347 
1348  /* Unmap registers */
1349  iounmap ( pf->regs );
1350 
1351  /* Free physical function */
1352  free ( pf );
1353 }
1354 
1355 /** NIC physical function PCI device IDs */
1356 static struct pci_device_id txnic_pf_ids[] = {
1357  PCI_ROM ( 0x177d, 0xa01e, "thunder-pf", "ThunderX NIC PF", 0 ),
1358 };
1359 
1360 /** NIC physical function PCI driver */
1361 struct pci_driver txnic_pf_driver __pci_driver = {
1362  .ids = txnic_pf_ids,
1363  .id_count = ( sizeof ( txnic_pf_ids ) / sizeof ( txnic_pf_ids[0] ) ),
1364  .probe = txnic_pf_probe,
1366 };
1367 
1368 /******************************************************************************
1369  *
1370  * BGX interface
1371  *
1372  ******************************************************************************
1373  */
1374 
1375 /** LMAC types */
1377  [TXNIC_LMAC_XAUI] = {
1378  .name = "XAUI",
1379  .count = 1,
1380  .lane_to_sds = 0xe4,
1381  },
1382  [TXNIC_LMAC_RXAUI] = {
1383  .name = "RXAUI",
1384  .count = 2,
1385  .lane_to_sds = 0x0e04,
1386  },
1387  [TXNIC_LMAC_10G_R] = {
1388  .name = "10GBASE-R",
1389  .count = 4,
1390  .lane_to_sds = 0x00000000,
1391  },
1392  [TXNIC_LMAC_40G_R] = {
1393  .name = "40GBASE-R",
1394  .count = 1,
1395  .lane_to_sds = 0xe4,
1396  },
1397 };
1398 
1399 /**
1400  * Detect BGX Ethernet interface LMAC type
1401  *
1402  * @v bgx BGX Ethernet interface
1403  * @ret type LMAC type, or negative error
1404  */
1405 static int txnic_bgx_detect ( struct txnic_bgx *bgx ) {
1406  uint64_t config;
1407  uint64_t br_pmd_control;
1408  uint64_t rx_lmacs;
1409  unsigned int type;
1410 
1411  /* We assume that the early (pre-UEFI) firmware will have
1412  * configured at least the LMAC 0 type and use of link
1413  * training, and may have overridden the number of LMACs.
1414  */
1415 
1416  /* Determine type from LMAC 0 */
1417  config = readq ( bgx->regs + BGX_CMR_CONFIG );
1418  type = BGX_CMR_CONFIG_LMAC_TYPE_GET ( config );
1419  if ( ( type >= ( sizeof ( txnic_lmac_types ) /
1420  sizeof ( txnic_lmac_types[0] ) ) ) ||
1421  ( txnic_lmac_types[type].count == 0 ) ) {
1422  DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/* BGX unknown type %d\n",
1423  bgx->node, bgx->idx, type );
1424  return -ENOTTY;
1425  }
1426  bgx->type = &txnic_lmac_types[type];
1427 
1428  /* Check whether link training is required */
1429  br_pmd_control = readq ( bgx->regs + BGX_SPU_BR_PMD_CONTROL );
1430  bgx->training =
1431  ( !! ( br_pmd_control & BGX_SPU_BR_PMD_CONTROL_TRAIN_EN ) );
1432 
1433  /* Determine number of LMACs */
1434  rx_lmacs = readq ( bgx->regs + BGX_CMR_RX_LMACS );
1435  bgx->count = BGX_CMR_RX_LMACS_LMACS_GET ( rx_lmacs );
1436  if ( ( bgx->count == TXNIC_NUM_LMAC ) &&
1437  ( bgx->type->count != TXNIC_NUM_LMAC ) ) {
1438  DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/* assuming %d LMACs\n",
1439  bgx->node, bgx->idx, bgx->type->count );
1440  bgx->count = bgx->type->count;
1441  }
1442 
1443  return type;
1444 }
1445 
1446 /**
1447  * Initialise BGX Ethernet interface
1448  *
1449  * @v bgx BGX Ethernet interface
1450  * @v type LMAC type
1451  */
1452 static void txnic_bgx_init ( struct txnic_bgx *bgx, unsigned int type ) {
1453  uint64_t global_config;
1455  unsigned int i;
1456 
1457  /* Set number of LMACs */
1459  ( bgx->regs + BGX_CMR_RX_LMACS ) );
1461  ( bgx->regs + BGX_CMR_TX_LMACS ) );
1462 
1463  /* Set LMAC types and lane mappings, and disable all LMACs */
1464  lane_to_sds = bgx->type->lane_to_sds;
1465  for ( i = 0 ; i < bgx->count ; i++ ) {
1468  ( bgx->regs + BGX_LMAC ( i ) + BGX_CMR_CONFIG ) );
1469  lane_to_sds >>= 8;
1470  }
1471 
1472  /* Reset all MAC address filtering */
1473  for ( i = 0 ; i < TXNIC_NUM_DMAC ; i++ )
1474  writeq ( 0, ( bgx->regs + BGX_CMR_RX_DMAC_CAM ( i ) ) );
1475 
1476  /* Reset NCSI steering */
1477  for ( i = 0 ; i < TXNIC_NUM_STEERING ; i++ )
1478  writeq ( 0, ( bgx->regs + BGX_CMR_RX_STEERING ( i ) ) );
1479 
1480  /* Enable backpressure to all channels */
1482  ( bgx->regs + BGX_CMR_CHAN_MSK_AND ) );
1483 
1484  /* Strip FCS */
1485  global_config = readq ( bgx->regs + BGX_CMR_GLOBAL_CONFIG );
1486  global_config |= BGX_CMR_GLOBAL_CONFIG_FCS_STRIP;
1487  writeq ( global_config, ( bgx->regs + BGX_CMR_GLOBAL_CONFIG ) );
1488 }
1489 
1490 /**
1491  * Get MAC address
1492  *
1493  * @v lmac Logical MAC
1494  */
1495 static void txnic_bgx_mac ( struct txnic_lmac *lmac ) {
1496  struct txnic_bgx *bgx = lmac->bgx;
1497  unsigned int lmac_idx = TXNIC_LMAC_IDX ( lmac->idx );
1498  uint64_t mac;
1499  EFI_STATUS efirc;
1500  int rc;
1501 
1502  /* Extract MAC from Board Configuration protocol, if available */
1503  if ( txcfg ) {
1504  if ( ( efirc = txcfg->GetLmacProp ( txcfg, bgx->node, bgx->idx,
1505  lmac_idx, MAC_ADDRESS,
1506  sizeof ( mac ),
1507  &mac ) ) == 0 ) {
1508  lmac->mac.be64 = cpu_to_be64 ( mac );
1509  } else {
1510  rc = -EEFI ( efirc );
1511  DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/%d could not get "
1512  "MAC address: %s\n", bgx->node, bgx->idx,
1513  lmac->idx, strerror ( rc ) );
1514  }
1515  } else {
1516  DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/%d has no board "
1517  "configuration protocol\n", bgx->node, bgx->idx,
1518  lmac->idx );
1519  }
1520 
1521  /* Use random MAC address if none available */
1522  if ( ! lmac->mac.be64 ) {
1523  DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/%d has no MAC address\n",
1524  bgx->node, bgx->idx, lmac->idx );
1525  eth_random_addr ( lmac->mac.raw );
1526  }
1527 }
1528 
1529 /**
1530  * Initialise Super PHY Unit (SPU)
1531  *
1532  * @v lmac Logical MAC
1533  */
1534 static void txnic_bgx_spu_init ( struct txnic_lmac *lmac ) {
1535  struct txnic_bgx *bgx = lmac->bgx;
1536 
1537  /* Reset PHY */
1540 
1541  /* Power down PHY */
1543 
1544  /* Configure training, if applicable */
1545  if ( bgx->training ) {
1546  writeq ( 0, ( lmac->regs + BGX_SPU_BR_PMD_LP_CUP ) );
1547  writeq ( 0, ( lmac->regs + BGX_SPU_BR_PMD_LD_CUP ) );
1548  writeq ( 0, ( lmac->regs + BGX_SPU_BR_PMD_LD_REP ) );
1550  ( lmac->regs + BGX_SPU_BR_PMD_CONTROL ) );
1551  }
1552 
1553  /* Disable forward error correction */
1554  writeq ( 0, ( lmac->regs + BGX_SPU_FEC_CONTROL ) );
1555 
1556  /* Disable autonegotiation */
1557  writeq ( 0, ( lmac->regs + BGX_SPU_AN_CONTROL ) );
1558 
1559  /* Power up PHY */
1560  writeq ( 0, ( lmac->regs + BGX_SPU_CONTROL1 ) );
1561 }
1562 
1563 /**
1564  * Initialise LMAC
1565  *
1566  * @v bgx BGX Ethernet interface
1567  * @v lmac_idx LMAC index
1568  */
1569 static void txnic_bgx_lmac_init ( struct txnic_bgx *bgx,
1570  unsigned int lmac_idx ) {
1571  struct txnic_lmac *lmac = &bgx->lmac[lmac_idx];
1572  uint64_t config;
1573 
1574  /* Record associated BGX */
1575  lmac->bgx = bgx;
1576 
1577  /* Set register base address (already mapped) */
1578  lmac->regs = ( bgx->regs + BGX_LMAC ( lmac_idx ) );
1579 
1580  /* Calculate virtual NIC index */
1581  lmac->idx = TXNIC_VNIC_IDX ( bgx->idx, lmac_idx );
1582 
1583  /* Set MAC address */
1584  txnic_bgx_mac ( lmac );
1585 
1586  /* Initialise PHY */
1587  txnic_bgx_spu_init ( lmac );
1588 
1589  /* Accept all multicasts and broadcasts */
1592  ( lmac->regs + BGX_CMR_RX_DMAC_CTL ) );
1593 
1594  /* Enable LMAC */
1595  config = readq ( lmac->regs + BGX_CMR_CONFIG );
1596  config |= ( BGX_CMR_CONFIG_ENABLE |
1599  writeq ( config, ( lmac->regs + BGX_CMR_CONFIG ) );
1600 }
1601 
1602 /**
1603  * Probe PCI device
1604  *
1605  * @v pci PCI device
1606  * @ret rc Return status code
1607  */
1608 static int txnic_bgx_probe ( struct pci_device *pci ) {
1609  struct txnic_bgx *bgx;
1610  struct txnic_pf *pf;
1611  unsigned long membase;
1612  unsigned int i;
1613  int type;
1614  int rc;
1615 
1616  /* Allocate and initialise structure */
1617  bgx = zalloc ( sizeof ( *bgx ) );
1618  if ( ! bgx ) {
1619  rc = -ENOMEM;
1620  goto err_alloc;
1621  }
1622  bgx->pci = pci;
1623  pci_set_drvdata ( pci, bgx );
1624 
1625  /* Get base address */
1626  membase = pciea_bar_start ( pci, PCIEA_BEI_BAR_0 );
1627 
1628  /* Calculate node ID and index */
1629  bgx->node = txnic_address_node ( membase );
1630  bgx->idx = txnic_address_bgx ( membase );
1631 
1632  /* Fix up PCI device */
1633  adjust_pci_device ( pci );
1634 
1635  /* Map registers */
1636  bgx->regs = pci_ioremap ( pci, membase, TXNIC_BGX_BAR_SIZE );
1637  if ( ! bgx->regs ) {
1638  rc = -ENODEV;
1639  goto err_ioremap;
1640  }
1641 
1642  /* Detect LMAC type */
1643  if ( ( type = txnic_bgx_detect ( bgx ) ) < 0 ) {
1644  rc = type;
1645  goto err_detect;
1646  }
1647  DBGC ( TXNICCOL ( bgx ), "TXNIC %d/%d/* BGX %s at %#lx %dx %s%s\n",
1648  bgx->node, bgx->idx, pci->dev.name, membase, bgx->count,
1649  bgx->type->name, ( bgx->training ? "(training)" : "" ) );
1650 
1651  /* Initialise interface */
1652  txnic_bgx_init ( bgx, type );
1653 
1654  /* Initialise all LMACs */
1655  for ( i = 0 ; i < bgx->count ; i++ )
1656  txnic_bgx_lmac_init ( bgx, i );
1657 
1658  /* Add to list of BGX devices */
1659  list_add_tail ( &bgx->list, &txnic_bgxs );
1660 
1661  /* Probe all LMACs, if applicable */
1662  list_for_each_entry ( pf, &txnic_pfs, list ) {
1663  if ( pf->node != bgx->node )
1664  continue;
1665  if ( ( rc = txnic_lmac_probe_all ( pf, bgx ) ) != 0 )
1666  goto err_probe;
1667  }
1668 
1669  return 0;
1670 
1671  if ( bgx->pf )
1673  list_del ( &bgx->list );
1674  err_probe:
1675  err_detect:
1676  iounmap ( bgx->regs );
1677  err_ioremap:
1678  free ( bgx );
1679  err_alloc:
1680  return rc;
1681 }
1682 
1683 /**
1684  * Remove PCI device
1685  *
1686  * @v pci PCI device
1687  */
1688 static void txnic_bgx_remove ( struct pci_device *pci ) {
1689  struct txnic_bgx *bgx = pci_get_drvdata ( pci );
1690 
1691  /* Remove all LMACs, if applicable */
1692  if ( bgx->pf )
1693  txnic_lmac_remove_all ( bgx->pf, bgx );
1694 
1695  /* Remove from list of BGX devices */
1696  list_del ( &bgx->list );
1697 
1698  /* Unmap registers */
1699  iounmap ( bgx->regs );
1700 
1701  /* Free BGX device */
1702  free ( bgx );
1703 }
1704 
1705 /** BGX PCI device IDs */
1706 static struct pci_device_id txnic_bgx_ids[] = {
1707  PCI_ROM ( 0x177d, 0xa026, "thunder-bgx", "ThunderX BGX", 0 ),
1708 };
1709 
1710 /** BGX PCI driver */
1711 struct pci_driver txnic_bgx_driver __pci_driver = {
1712  .ids = txnic_bgx_ids,
1713  .id_count = ( sizeof ( txnic_bgx_ids ) / sizeof ( txnic_bgx_ids[0] ) ),
1716 };
#define BGX_SPU_STATUS2_RCVFLT
Definition: thunderx.h:813
static void txnic_bgx_remove(struct pci_device *pci)
Remove PCI device.
Definition: thunderx.c:1688
#define TXNIC_PF_PKIND_CFG(pk)
Port kind configuration.
Definition: thunderx.h:463
#define TXNIC_PF_BAR_SIZE
Physical function BAR size.
Definition: thunderx.h:31
#define BGX_CMR_TX_STAT12
CMR transmit statistics 12.
Definition: thunderx.h:769
unsigned int idx
BGX index.
Definition: thunderx.h:931
#define BGX_CMR_TX_STAT3
CMR transmit statistics 3.
Definition: thunderx.h:742
#define TXNIC_PF_INTF_BP_CFG(in)
Interface backpressure configuration.
Definition: thunderx.h:457
#define BGX_CMR_CONFIG_LMAC_TYPE_GET(config)
Definition: thunderx.h:672
static int txnic_lmac_open(struct net_device *netdev)
Open network device.
Definition: thunderx.c:936
#define BGX_CMR_TX_STAT14
CMR transmit statistics 14.
Definition: thunderx.h:775
#define __attribute__(x)
Definition: compiler.h:10
static void txnic_lmac_close(struct net_device *netdev)
Close network device.
Definition: thunderx.c:1032
static void txnic_bgx_init(struct txnic_bgx *bgx, unsigned int type)
Initialise BGX Ethernet interface.
Definition: thunderx.c:1452
#define TXNIC_QS_RQ_CFG_ENA
Definition: thunderx.h:203
static void txnic_complete_rqe(struct txnic *vnic, struct txnic_cqe_rx *cqe)
Complete receive queue entry.
Definition: thunderx.c:409
#define BGX_SPU_AN_STATUS
SPU autonegotiation status.
Definition: thunderx.h:853
static EFI_THUNDER_CONFIG_PROTOCOL * txcfg
Board configuration protocol.
Definition: thunderx.c:61
#define TXNIC_PF_QS_SQ_CFG_CQ_QS(qs)
Definition: thunderx.h:572
A receive completion queue entry.
Definition: thunderx.h:345
struct txnic * vnic
Virtual NIC (if applicable)
Definition: thunderx.h:919
#define TXNIC_PF_TL4_CFG_RR_QUANTUM_DEFAULT
Definition: thunderx.h:541
struct arbelprm_rc_send_wqe rc
Definition: arbel.h:14
#define TXNIC_PF_INTF_BP_CFG_BP_ENA
Definition: thunderx.h:458
wmb()
static int txnic_bgx_probe(struct pci_device *pci)
Probe PCI device.
Definition: thunderx.c:1608
40GBASE-R
Definition: thunderx.h:884
uint16_t sqe_ptr
Send queue entry pointer.
Definition: thunderx.h:336
#define BGX_SPU_BR_STATUS1_RCV_LNK
Definition: thunderx.h:817
#define iob_put(iobuf, len)
Definition: iobuf.h:120
void netdev_rx_err(struct net_device *netdev, struct io_buffer *iobuf, int rc)
Discard received packet.
Definition: netdevice.c:527
#define EEFI(efirc)
Convert an EFI status code to an iPXE status code.
Definition: efi.h:167
#define BGX_CMR_CONFIG
CMR configuration.
Definition: thunderx.h:668
A physical function.
Definition: thunderx.h:579
uint8_t flags
Flags.
Definition: thunderx.h:118
#define TXNIC_PF_TL3_CFG(tl)
Traffic limiter 3 configuration.
Definition: thunderx.h:528
#define TXNIC_PF_CHAN_RX_BP_CFG_BPID(bp)
Definition: thunderx.h:519
#define TXNIC_PF_CHAN_RX_CFG_CPI_BASE(ix)
Definition: thunderx.h:514
#define TXNIC_RQE_SIZE
Receive queue entry size.
Definition: thunderx.h:263
#define BGX_SPU_BR_ALGN_STATUS_ALIGND
Definition: thunderx.h:828
A PCI driver.
Definition: pci.h:227
unsigned int count
Number of LMACs.
Definition: thunderx.h:936
#define BGX_SPU_STATUS1_RCV_LNK
Definition: thunderx.h:809
10GBASE-R
Definition: thunderx.h:883
static int txnic_create_sq(struct txnic *vnic)
Create send queue.
Definition: thunderx.c:113
#define TXNIC_PF_LMAC_CFG2_MAX_PKT_SIZE(sz)
Definition: thunderx.h:493
#define TXNIC_QS_RQ_CFG(q)
Receive queue configuration.
Definition: thunderx.h:202
#define TXNIC_QS_SQ_DOOR(q)
Send queue doorbell.
Definition: thunderx.h:97
int(* open)(struct net_device *netdev)
Open network device.
Definition: netdevice.h:222
#define TXNIC_QS_CQ_HEAD(q)
Completion queue head pointer.
Definition: thunderx.h:313
#define BGX_CMR_TX_STAT15
CMR transmit statistics 15.
Definition: thunderx.h:778
#define TXNIC_QS_RBDR_CFG_QSIZE_8K
Definition: thunderx.h:216
#define BGX_CMR_RX_STAT3
CMR receive statistics 3.
Definition: thunderx.h:691
#define BGX_SPU_INT
SPU interrupt.
Definition: thunderx.h:861
#define TXNIC_PF_RSSI_RQ(ix)
RSS indirection receive queue.
Definition: thunderx.h:478
#define BGX_CMR_TX_STAT10
CMR transmit statistics 10.
Definition: thunderx.h:763
#define BGX_CMR_CHAN_MSK_AND_ALL(count)
Definition: thunderx.h:729
Error codes.
#define BGX_SPU_BR_STATUS1_HI_BER
Definition: thunderx.h:818
#define TXNIC_NUM_DMAC
Maximum number of destination MAC addresses (per BGX)
Definition: thunderx.h:43
#define BGX_CMR_RX_STEERING(i)
CMR receive steering.
Definition: thunderx.h:725
#define TXNIC_QS_RBDR_DOOR(q)
Receive buffer descriptor ring doorbell.
Definition: thunderx.h:230
uint64_t readq(volatile uint64_t *io_addr)
Read 64-bit qword from memory-mapped device.
#define TXNIC_PF_QS_RQ_BP_CFG_CQ_BP_ENA
Definition: thunderx.h:567
#define TXNIC_QS_SQ_BASE(q)
Send queue base address.
Definition: thunderx.h:88
static struct pci_device_id txnic_bgx_ids[]
BGX PCI device IDs.
Definition: thunderx.c:1706
static void netdev_tx_complete_next(struct net_device *netdev)
Complete network transmission.
Definition: netdevice.h:764
I/O buffers.
void free_iob(struct io_buffer *iobuf)
Free I/O buffer.
Definition: iobuf.c:146
struct pci_device_id * ids
PCI ID table.
Definition: pci.h:229
#define TXNIC_LINE_SIZE
Size of a cache line.
Definition: thunderx.h:25
#define BGX_CMR_RX_STAT2
CMR receive statistics 2.
Definition: thunderx.h:688
uint8_t type
Type.
Definition: ena.h:16
const char * name
Device name (for debugging)
Definition: thunderx.h:421
#define BGX_CMR_RX_DMAC_CTL_MCST_MODE_ACCEPT
Definition: thunderx.h:717
#define BGX_SPU_FEC_CONTROL
SPU forward error correction control.
Definition: thunderx.h:847
#define BGX_SPU_AN_CONTROL
SPU autonegotation control.
Definition: thunderx.h:850
static __always_inline void copy_from_user(void *dest, userptr_t src, off_t src_off, size_t len)
Copy data from user buffer.
Definition: uaccess.h:337
static int txnic_lmac_probe(struct txnic_lmac *lmac)
Probe logical MAC virtual NIC.
Definition: thunderx.c:1091
#define TXNIC_QS_CQ_CFG_QSIZE_256
Definition: thunderx.h:300
static void txnic_lmac_reset(struct txnic_lmac *lmac)
Reset LMAC.
Definition: thunderx.c:914
A Logical MAC (LMAC)
Definition: thunderx.h:907
static void txnic_lmac_diag(struct txnic_lmac *lmac)
Show LMAC diagnostics (for debugging)
Definition: thunderx.c:745
static int txnic_disable_sq(struct txnic *vnic)
Disable send queue.
Definition: thunderx.c:138
#define TXNIC_QS_SQ_CFG_RESET
Definition: thunderx.h:82
static struct pci_device_id txnic_pf_ids[]
NIC physical function PCI device IDs.
Definition: thunderx.c:1356
unsigned long user_to_phys(userptr_t userptr, off_t offset)
Convert user pointer to physical address.
uint16_t size
Size.
Definition: thunderx.h:132
#define TXNIC_NUM_BGX
Maximum number of BGX Ethernet interfaces (per node)
Definition: thunderx.h:37
#define DBGC(...)
Definition: compiler.h:505
#define TXNIC_PF_CFG
Physical function configuration.
Definition: thunderx.h:441
#define BGX_CMR_TX_STAT0
CMR transmit statistics 0.
Definition: thunderx.h:733
char name[40]
Name.
Definition: device.h:75
static void txnic_destroy_sq(struct txnic *vnic)
Destroy send queue.
Definition: thunderx.c:166
#define BGX_SPU_BR_STATUS2_LATCHED_BER
Definition: thunderx.h:824
#define TXNIC_PF_INTF_BP_CFG_BP_ID(bp)
Definition: thunderx.h:460
static void txnic_destroy_cq(struct txnic *vnic)
Destroy completion queue.
Definition: thunderx.c:511
#define TXNIC_PF_RSSI_RQ_RQ_QS(qs)
Definition: thunderx.h:479
unsigned long long uint64_t
Definition: stdint.h:13
#define BGX_SPU_INT_FEC_UNCORR
Definition: thunderx.h:867
uint8_t errop
Error opcode.
Definition: thunderx.h:347
uint8_t flags
Flags.
Definition: thunderx.h:136
Virtual function BAR 0.
Definition: pciea.h:39
#define BGX_SPU_STATUS1_FLT
Definition: thunderx.h:808
#define BGX_SPU_BR_PMD_LD_REP
SPU local device status report.
Definition: thunderx.h:844
#define TXNIC_CQE_TYPE_SEND
Send completion queue entry type.
Definition: thunderx.h:342
#define TXNIC_PF_PKIND_CFG_MAXLEN_DISABLE
Definition: thunderx.h:466
unsigned long vf_stride
Virtual function BAR stride.
Definition: thunderx.h:590
static void txnic_poll_cq(struct txnic *vnic)
Poll completion queue.
Definition: thunderx.c:529
struct txnic_rbdr_entry rbdre
Receive buffer descriptor ring entry.
Definition: thunderx.h:244
#define cpu_to_le64(value)
Definition: byteswap.h:108
#define BGX_SPU_AN_STATUS_XNP_STAT
Definition: thunderx.h:854
#define BGX_CMR_TX_STAT7
CMR transmit statistics 7.
Definition: thunderx.h:754
void netdev_tx_complete_next_err(struct net_device *netdev, int rc)
Complete network transmission.
Definition: netdevice.c:450
void netdev_link_down(struct net_device *netdev)
Mark network device as having link down.
Definition: netdevice.c:186
#define TXNIC_QS_SQ_CFG_QSIZE_1K
Definition: thunderx.h:84
#define TXNIC_PF_PKIND_CFG_MINLEN_DISABLE
Definition: thunderx.h:469
#define BGX_SPU_INT_AN_PAGE_RX
Definition: thunderx.h:866
uint8_t mac[ETH_ALEN]
MAC address.
Definition: ena.h:24
#define TXNIC_RQ_SIZE
Receive queue size.
Definition: thunderx.h:274
struct txnic_sq sq
Send queue.
Definition: thunderx.h:426
#define TXNIC_PF_BP_CFG_BP_POLL_DLY_DEFAULT
Definition: thunderx.h:448
#define BGX_SPU_CONTROL1
SPU control 1.
Definition: thunderx.h:799
#define BGX_SPU_BR_PMD_STATUS
SPU BASE-R link training status.
Definition: thunderx.h:835
uint8_t send_status
Status.
Definition: thunderx.h:332
#define TXNIC_SEND_HDR_FLAGS
Flags for send header subdescriptor.
Definition: thunderx.h:127
#define BGX_CMR_TX_STAT6
CMR transmit statistics 6.
Definition: thunderx.h:751
unsigned int cons
Consumer counter.
Definition: thunderx.h:281
#define TXNIC_PF_TL2_CFG(tl)
Traffic limiter 2 configuration.
Definition: thunderx.h:522
#define TXNIC_QS_CQ_CFG_RESET
Definition: thunderx.h:298
void adjust_pci_device(struct pci_device *pci)
Enable PCI device.
Definition: pci.c:149
struct io_buffer * alloc_iob(size_t len)
Allocate I/O buffer.
Definition: iobuf.c:129
unsigned int prod
Producer counter.
Definition: thunderx.h:279
#define TXNIC_PF_LMAC_CFG
LMAC configuration.
Definition: thunderx.h:485
#define TXNIC_QS_RBDR_CFG_RESET
Definition: thunderx.h:214
#define BGX_SPU_INT_ERR_BLK
Definition: thunderx.h:874
static void txnic_poll(struct txnic *vnic)
Poll virtual NIC.
Definition: thunderx.c:636
#define BGX_SPU_INT_BIP_ERR
Definition: thunderx.h:869
struct device dev
Generic device.
Definition: pci.h:190
#define BGX_SPU_RESET_DELAY_MS
SPU reset delay.
Definition: thunderx.h:804
#define BGX_SPU_AN_STATUS_PAGE_RX
Definition: thunderx.h:855
#define TXNIC_PF_QS_RQ_CFG_RBDR_STRT_QS(qs)
Definition: thunderx.h:559
#define TXNIC_PF_LMAC_CFG2
LMAC configuration 2.
Definition: thunderx.h:492
#define TXNIC_SQ_STRIDE
Send queue stride.
Definition: thunderx.h:179
#define TXNIC_PF_INTF_SEND_CFG(in)
Interface send configuration.
Definition: thunderx.h:452
uint32_t lane_to_sds
Lane-to-SDS mapping.
Definition: thunderx.h:894
#define TXNIC_PF_QS_SQ_CFG2_TL4(tl)
Definition: thunderx.h:576
#define TXNIC_PF_TL4_CFG_SQ_QS(qs)
Definition: thunderx.h:539
Dynamic memory allocation.
#define BGX_SPU_BR_STATUS1_BLK_LOCK
Definition: thunderx.h:819
void * regs
Registers.
Definition: thunderx.h:909
static int txnic_send(struct txnic *vnic, struct io_buffer *iobuf)
Send packet.
Definition: thunderx.c:186
#define BGX_CMR_RX_DMAC_CTL_BCST_ACCEPT
Definition: thunderx.h:719
#define TXNIC_SEND_GATHER_FLAGS
Flags for send gather subdescriptor.
Definition: thunderx.h:145
Protocol structure.
Definition: thunderxcfg.h:148
uint8_t status
Status.
Definition: ena.h:16
#define TXNIC_SQE_SUBDESCS
Number of subdescriptors per send queue entry.
Definition: thunderx.h:160
static void netdev_init(struct net_device *netdev, struct net_device_operations *op)
Initialise a network device.
Definition: netdevice.h:504
#define BGX_SPU_BR_PMD_CONTROL_TRAIN_EN
Definition: thunderx.h:832
#define list_del(list)
Delete an entry from a list.
Definition: list.h:119
static void pci_set_drvdata(struct pci_device *pci, void *priv)
Set PCI driver-private data.
Definition: pci.h:341
An LMAC type.
Definition: thunderx.h:888
#define BGX_SPU_BR_STATUS2_LATCHED_LOCK
Definition: thunderx.h:823
#define ENOMEM
Not enough space.
Definition: errno.h:534
#define TXNIC_QS_CQ_BASE(q)
Completion queue base address.
Definition: thunderx.h:310
static struct net_device_operations txnic_lmac_operations
Network device operations.
Definition: thunderx.c:1078
#define TXNIC_PF_QS_SQ_CFG2(q)
Send queue configuration 2.
Definition: thunderx.h:575
#define TXNIC_TL2_IDX(vnic_idx)
Calculate traffic limiter 2 index.
Definition: thunderx.h:630
#define BGX_SPU_INT_TRAINING_FAIL
Definition: thunderx.h:862
static int txnic_create_cq(struct txnic *vnic)
Create completion queue.
Definition: thunderx.c:459
struct txnic_pf * pf
Physical function (if known)
Definition: thunderx.h:943
void * memcpy(void *dest, const void *src, size_t len) __nonnull
static int txnic_open(struct txnic *vnic)
Open virtual NIC.
Definition: thunderx.c:585
#define TXNIC_QS_SQ_STATUS(q)
Send queue status.
Definition: thunderx.h:100
static int txnic_disable_cq(struct txnic *vnic)
Disable completion queue.
Definition: thunderx.c:483
#define BGX_CMR_TX_STAT2
CMR transmit statistics 2.
Definition: thunderx.h:739
A receive queue entry.
Definition: thunderx.h:242
#define TXNIC_QS_CQ_DOOR(q)
Completion queue doorbell.
Definition: thunderx.h:319
#define TXNIC_PF_QS_RQ_CFG_RBDR_CONT_QS(qs)
Definition: thunderx.h:558
A send queue entry.
Definition: thunderx.h:152
unsigned int cons
Consumer counter.
Definition: thunderx.h:189
#define BGX_SPU_INT_AN_LINK_GOOD
Definition: thunderx.h:865
#define BGX_CMR_RX_DMAC_CAM(i)
CMR destination MAC CAM.
Definition: thunderx.h:722
uint64_t addr
Address.
Definition: thunderx.h:138
static __always_inline unsigned long virt_to_bus(volatile const void *addr)
Convert virtual address to a bus address.
Definition: io.h:183
#define BGX_SPU_BR_ALGN_STATUS
SPU BASE-R alignment status.
Definition: thunderx.h:827
#define TXNIC_PF_MPI_CFG(ix)
Match parse index configuration.
Definition: thunderx.h:473
unsigned int node
Node ID.
Definition: thunderx.h:929
struct pci_device * pci
PCI device.
Definition: thunderx.h:927
Assertions.
#define TXNIC_VNIC_IDX(bgx_idx, lmac_idx)
Calculate virtual NIC index.
Definition: thunderx.h:605
#define TXNIC_PF_QS_CFG_ENA
Definition: thunderx.h:549
#define TXNIC_CQE_RX_APAD_LEN(apad)
Applied padding.
Definition: thunderx.h:366
#define TXNIC_QS_CQ_STATUS_QCOUNT(status)
Definition: thunderx.h:323
assert((readw(&hdr->flags) &(GTF_reading|GTF_writing))==0)
#define TXNIC_CQE_TYPE_RX
Receive completion queue entry type.
Definition: thunderx.h:363
static void netdev_put(struct net_device *netdev)
Drop reference to network device.
Definition: netdevice.h:561
Ethernet protocol.
#define TXNIC_PF_PKIND_CFG_LENERR_EN
Definition: thunderx.h:464
#define BGX_CMR_RX_STAT5
CMR receive statistics 5.
Definition: thunderx.h:697
#define TXNIC_QS_SQ_STATUS_STOPPED
Definition: thunderx.h:101
uint8_t intr
Interrupts enabled.
Definition: ena.h:14
EFI_THUNDER_CONFIG_PROTOCOL_GET_LMAC_PROP GetLmacProp
Definition: thunderxcfg.h:151
#define TXNIC_PF_TL3_CHAN_CHAN(ch)
Definition: thunderx.h:535
#define list_for_each_entry(pos, head, member)
Iterate over entries in a list.
Definition: list.h:420
void * priv
Driver private data.
Definition: netdevice.h:431
#define TXNIC_PF_CFG_ENA
Definition: thunderx.h:442
#define DBGC_HDA(...)
Definition: compiler.h:506
static LIST_HEAD(txnic_bgxs)
List of BGX Ethernet interfaces.
#define list_add_tail(new, head)
Add a new entry to the tail of a list.
Definition: list.h:93
#define TXNIC_PF_LMAC(lm)
LMAC registers.
Definition: thunderx.h:482
static void netdev_link_up(struct net_device *netdev)
Mark network device as having link up.
Definition: netdevice.h:774
#define TXNIC_PF_CHAN_RX_BP_CFG_ENA
Definition: thunderx.h:518
#define TXNIC_PF_QS_RQ_DROP_CFG(q)
Receive queue drop configuration.
Definition: thunderx.h:562
unsigned int cons
Consumer counter.
Definition: thunderx.h:404
#define BGX_SPU_INT_SYNLOS
Definition: thunderx.h:872
static void txnic_lmac_remove_all(struct txnic_pf *pf, struct txnic_bgx *bgx)
Remove all LMACs on a BGX Ethernet interface.
Definition: thunderx.c:1218
#define TXNIC_PF_INTF_SEND_CFG_BLOCK_BGX
Definition: thunderx.h:453
Standard BAR 0.
Definition: pciea.h:32
static userptr_t size_t offset
Offset of the first segment within the content.
Definition: deflate.h:259
static struct net_device * netdev
Definition: gdbudp.c:52
#define BGX_SPU_BR_STATUS1
SPU BASE-R status 1.
Definition: thunderx.h:816
#define TXNIC_CQ_STRIDE
Completion queue stride.
Definition: thunderx.h:396
static void txnic_bgx_mac(struct txnic_lmac *lmac)
Get MAC address.
Definition: thunderx.c:1495
#define TXNIC_QS_RBDR_BASE(q)
Receive buffer descriptor ring base address.
Definition: thunderx.h:221
#define BGX_SPU_BR_PMD_LD_CUP
SPU local device coefficient update.
Definition: thunderx.h:841
#define TXNIC_PF_LMAC_CREDIT
LMAC credit.
Definition: thunderx.h:496
struct txnic_rq rq
Receive queue.
Definition: thunderx.h:428
#define TXNIC_QS_RBDR_CFG_ENA
Definition: thunderx.h:213
#define BGX_SPU_INT_AN_COMPLETE
Definition: thunderx.h:864
void unregister_netdev(struct net_device *netdev)
Unregister network device.
Definition: netdevice.c:874
static void txnic_bgx_lmac_init(struct txnic_bgx *bgx, unsigned int lmac_idx)
Initialise LMAC.
Definition: thunderx.c:1569
#define TXNIC_TL3_IDX(vnic_idx)
Calculate traffic limiter 3 index.
Definition: thunderx.h:638
u32 status1
Definition: ar9003_mac.h:68
#define cpu_to_le32(value)
Definition: byteswap.h:107
static void txnic_pf_remove(struct pci_device *pci)
Remove PCI device.
Definition: thunderx.c:1335
#define TXNIC_PF_QS_RQ_CFG_CQ_QS(qs)
Definition: thunderx.h:557
#define TXNIC_QS_CQ_CFG(q)
Completion queue configuration.
Definition: thunderx.h:296
#define TXNIC_PF_CHAN_RX_CFG
Channel receive configuration.
Definition: thunderx.h:513
#define TXNIC_PF_QS_CFG_VNIC(vn)
Definition: thunderx.h:550
A send completion queue entry.
Definition: thunderx.h:330
#define BGX_CMR_RX_LMACS_LMACS_GET(lmacs)
Definition: thunderx.h:788
#define TXNIC_PF_CHAN(ch)
Channel registers.
Definition: thunderx.h:506
char * strerror(int errno)
Retrieve string representation of error number.
Definition: strerror.c:78
static void(* free)(struct refcnt *refcnt))
Definition: refcnt.h:54
#define BGX_SPU_STATUS2
SPU status 2.
Definition: thunderx.h:812
struct list_head list
List of physical functions.
Definition: thunderx.h:593
#define TXNIC_RQ_DISABLE_MAX_WAIT_MS
Maximum time to wait for a receive queue to disable.
Definition: thunderx.h:209
void * zalloc(size_t size)
Allocate cleared memory.
Definition: malloc.c:624
#define BGX_CMR_CONFIG_DATA_PKT_RX_EN
Definition: thunderx.h:670
#define BGX_CMR_GLOBAL_CONFIG
CMR global configuration.
Definition: thunderx.h:678
void * regs
Registers.
Definition: thunderx.h:925
const char * name
Name.
Definition: thunderx.h:890
PCI bus.
#define TXNIC_PF_QS_RQ_BP_CFG_BPID(bp)
Definition: thunderx.h:568
Cavium ThunderX Board Configuration.
A PCI device.
Definition: pci.h:188
int register_netdev(struct net_device *netdev)
Register network device.
Definition: netdevice.c:697
static size_t iob_len(struct io_buffer *iobuf)
Calculate length of data in an I/O buffer.
Definition: iobuf.h:155
const char * eth_ntoa(const void *ll_addr)
Transcribe Ethernet address.
Definition: ethernet.c:175
static void txnic_free(struct txnic *vnic)
Free virtual NIC.
Definition: thunderx.c:712
#define TXNIC_PF_MPI_CFG_VNIC(vn)
Definition: thunderx.h:474
#define BGX_SPU_BR_PMD_LP_CUP
SPU link partner coefficient update.
Definition: thunderx.h:838
#define TXNICCOL(x)
Debug colour for physical function and BGX messages.
Definition: thunderx.c:58
#define BGX_LMAC(lm)
Per-LMAC registers.
Definition: thunderx.h:665
#define TXNIC_PF_QS_RQ_BP_CFG(q)
Receive queue backpressure configuration.
Definition: thunderx.h:565
struct txnic_bgx * bgx[TXNIC_NUM_BGX]
BGX Ethernet interfaces (if known)
Definition: thunderx.h:595
A virtual NIC.
Definition: thunderx.h:417
static __always_inline void copy_to_user(userptr_t dest, off_t dest_off, const void *src, size_t len)
Copy data to user buffer.
Definition: uaccess.h:324
User memory allocation.
static void txnic_lmac_remove(struct txnic_lmac *lmac)
Remove logical MAC virtual NIC.
Definition: thunderx.c:1148
static struct txnic * txnic_alloc(struct pci_device *pci, unsigned long membase)
Allocate virtual NIC.
Definition: thunderx.c:652
#define BGX_CMR_RX_STAT6
CMR receive statistics 6.
Definition: thunderx.h:700
EFI_REQUEST_PROTOCOL(EFI_THUNDER_CONFIG_PROTOCOL, &txcfg)
A network device.
Definition: netdevice.h:352
static int txnic_disable_rq(struct txnic *vnic)
Disable receive queue.
Definition: thunderx.c:302
#define TXNIC_QS_CQ_TAIL(q)
Completion queue tail pointer.
Definition: thunderx.h:316
#define ENODEV
No such device.
Definition: errno.h:509
#define BGX_SPU_BR_STATUS2
SPU BASE-R status 2.
Definition: thunderx.h:822
#define TXNIC_QS_SQ_HEAD(q)
Send queue head pointer.
Definition: thunderx.h:91
#define BGX_CMR_RX_STAT1
CMR receive statistics 1.
Definition: thunderx.h:685
static void netdev_nullify(struct net_device *netdev)
Stop using a network device.
Definition: netdevice.h:517
#define TXNIC_TL4_IDX(vnic_idx)
Calculate traffic limiter 4 index.
Definition: thunderx.h:646
#define BGX_CMR_TX_STAT13
CMR transmit statistics 13.
Definition: thunderx.h:772
#define BGX_CMR_CHAN_MSK_AND
CMR backpressure channel mask AND.
Definition: thunderx.h:728
#define BGX_CMR_TX_STAT1
CMR transmit statistics 1.
Definition: thunderx.h:736
unsigned long pciea_bar_size(struct pci_device *pci, unsigned int bei)
Find the size of a PCI Enhanced Allocation BAR equivalent.
Definition: pciea.c:144
#define TXNIC_PF_LMAC_CREDIT_CC_PACKET_CNT_DEFAULT
Definition: thunderx.h:501
userptr_t cqe
Completion queue entries.
Definition: thunderx.h:406
#define BGX_SPU_AN_STATUS_AN_COMPLETE
Definition: thunderx.h:856
#define BGX_CMR_CONFIG_LMAC_TYPE_SET(ty)
Definition: thunderx.h:674
#define TXNIC_CHAN_IDX(vnic_idx)
Calculate channel index.
Definition: thunderx.h:654
#define TXNIC_RQES
Number of receive queue entries.
Definition: thunderx.h:251
#define TXNIC_PF_INTF_SEND_CFG_BLOCK(bl)
Definition: thunderx.h:454
#define TXNIC_RQ_FILL
Receive queue maximum fill level.
Definition: thunderx.h:257
#define TXNIC_PF_BP_CFG
Backpressure configuration.
Definition: thunderx.h:445
#define BGX_SPU_INT_DBG_SYNC
Definition: thunderx.h:870
Cavium ThunderX Ethernet driver.
#define ETH_ALEN
Definition: if_ether.h:8
#define ETH_ZLEN
Definition: if_ether.h:10
A PCI device ID list entry.
Definition: pci.h:152
uint8_t raw[ETH_ALEN]
Definition: thunderx.h:901
#define le16_to_cpu(value)
Definition: byteswap.h:112
A BGX Ethernet interface.
Definition: thunderx.h:923
unsigned int uint32_t
Definition: stdint.h:12
#define BGX_CMR_TX_STAT9
CMR transmit statistics 9.
Definition: thunderx.h:760
#define TXNIC_CQES
Number of completion queue entries.
Definition: thunderx.h:390
static void txnic_bgx_spu_init(struct txnic_lmac *lmac)
Initialise Super PHY Unit (SPU)
Definition: thunderx.c:1534
userptr_t sqe
Send queue entries.
Definition: thunderx.h:191
#define BGX_CMR_RX_STAT9
CMR receive statistics 9.
Definition: thunderx.h:709
#define TXNIC_NUM_STEERING
Maximum number of steering rules (per BGX)
Definition: thunderx.h:46
#define TXNIC_PF_TL2_CFG_RR_QUANTUM_DEFAULT
Definition: thunderx.h:524
#define BGX_SPU_INT_FEC_CORR
Definition: thunderx.h:868
struct txnic_lmac_type * type
LMAC type.
Definition: thunderx.h:934
#define TXNIC_PF_CHAN_TX_CFG_BP_ENA
Definition: thunderx.h:510
#define BGX_SPU_STATUS1
SPU status 1.
Definition: thunderx.h:807
static struct xen_remove_from_physmap * remove
Definition: xenmem.h:39
#define BGX_CMR_RX_LMACS_LMACS_SET(ct)
Definition: thunderx.h:790
struct io_buffer * iobuf[TXNIC_RQ_FILL]
I/O buffers.
Definition: thunderx.h:285
#define TXNIC_LMAC_IDX(vnic_idx)
Calculate logical MAC index.
Definition: thunderx.h:622
#define TXNIC_PF_QS_RQ_CFG(q)
Receive queue configuration.
Definition: thunderx.h:553
#define BGX_CMR_TX_STAT4
CMR transmit statistics 4.
Definition: thunderx.h:745
txnic_lmac_types
LMAC types.
Definition: thunderx.h:879
Network device operations.
Definition: netdevice.h:213
struct txnic_send_header hdr
Send header descriptor.
Definition: thunderx.h:154
static unsigned int txnic_address_node(uint64_t addr)
Calculate node ID.
Definition: thunderx.h:54
void netdev_rx(struct net_device *netdev, struct io_buffer *iobuf)
Add packet to receive queue.
Definition: netdevice.c:489
struct device * dev
Underlying hardware device.
Definition: netdevice.h:364
void * regs
Registers.
Definition: thunderx.h:419
struct txnic_cqe_common common
Common fields.
Definition: thunderx.h:379
#define TXNIC_PF_BP_CFG_BP_POLL_ENA
Definition: thunderx.h:446
uint8_t count
Number of LMACs.
Definition: thunderx.h:892
#define BGX_SPU_BR_PMD_CONTROL
SPU BASE-R link training control.
Definition: thunderx.h:831
10GBASE-X/XAUI or DXAUI
Definition: thunderx.h:881
Network device management.
#define BGX_CMR_TX_STAT17
CMR transmit statistics 17.
Definition: thunderx.h:784
#define BGX_CMR_RX_STAT10
CMR receive statistics 10.
Definition: thunderx.h:712
struct txnic_lmac lmac[TXNIC_NUM_LMAC]
Logical MACs.
Definition: thunderx.h:946
#define BGX_CMR_TX_STAT16
CMR transmit statistics 16.
Definition: thunderx.h:781
static void * pci_get_drvdata(struct pci_device *pci)
Get PCI driver-private data.
Definition: pci.h:351
#define BGX_CMR_RX_STAT4
CMR receive statistics 4.
Definition: thunderx.h:694
static void txnic_lmac_update_link(struct txnic_lmac *lmac)
Update LMAC link state.
Definition: thunderx.c:854
unsigned int node
Node ID.
Definition: thunderx.h:585
uint8_t apad
Padding.
Definition: thunderx.h:355
void mdelay(unsigned long msecs)
Delay for a fixed number of milliseconds.
Definition: timer.c:78
Definition: sis900.h:23
#define iob_reserve(iobuf, len)
Definition: iobuf.h:67
uint8_t subdcnt
Subdescriptor count.
Definition: thunderx.h:116
#define TXNIC_PF_LMAC_CFG_MIN_PKT_SIZE(sz)
Definition: thunderx.h:489
#define TXNIC_QS_CQ_STATUS2(q)
Completion queue status 2.
Definition: thunderx.h:327
#define TXNIC_SQ_FILL
Send queue maximum fill level.
Definition: thunderx.h:173
#define TXNIC_QS_SQ_CFG(q)
Send queue configuration.
Definition: thunderx.h:80
char name[NETDEV_NAME_LEN]
Name of this network device.
Definition: netdevice.h:362
struct txnic_send_gather gather
Send gather descriptor.
Definition: thunderx.h:156
#define BGX_SPU_CONTROL1_LO_PWR
Definition: thunderx.h:801
#define TXNIC_QS_RBDR_CFG_LINES(sz)
Definition: thunderx.h:218
#define TXNIC_PF_QS_RQ_BP_CFG_RBDR_BP_ENA
Definition: thunderx.h:566
static __always_inline void ufree(userptr_t userptr)
Free external memory.
Definition: umalloc.h:65
uint32_t len
Length.
Definition: ena.h:14
#define TXNIC_PF_INTF_BP_CFG_BP_ID_BGX
Definition: thunderx.h:459
uint8_t unused[32]
Unused.
Definition: eltorito.h:15
#define TXNIC_PF_LMAC_CFG_ADJUST_DEFAULT
Definition: thunderx.h:487
static __always_inline userptr_t umalloc(size_t size)
Allocate external memory.
Definition: umalloc.h:54
#define BGX_SPU_INT_BITLCKLS
Definition: thunderx.h:873
#define TXNIC_QS_CQ_STATUS(q)
Completion queue status.
Definition: thunderx.h:322
#define ENOBUFS
No buffer space available.
Definition: errno.h:498
#define DBGC2(...)
Definition: compiler.h:522
unsigned long pciea_bar_start(struct pci_device *pci, unsigned int bei)
Find the start of a PCI Enhanced Allocation BAR equivalent.
Definition: pciea.c:130
#define ENOTTY
Inappropriate I/O control operation.
Definition: errno.h:594
static int txnic_create_rq(struct txnic *vnic)
Create receive queue.
Definition: thunderx.c:271
int(* probe)(struct pci_device *pci)
Probe device.
Definition: pci.h:240
#define TXNIC_QS_RBDR_STATUS0(q)
Receive buffer descriptor ring status 0.
Definition: thunderx.h:233
unsigned long vf_membase
Virtual function BAR base.
Definition: thunderx.h:588
#define BGX_SPU_INT_RX_LINK_UP
Definition: thunderx.h:876
void * regs
Registers.
Definition: thunderx.h:581
static void txnic_lmac_poll_link(struct txnic_lmac *lmac)
Poll LMAC link state.
Definition: thunderx.c:876
static void txnic_lmac_poll(struct net_device *netdev)
Poll network device.
Definition: thunderx.c:1066
void * data
Start of data.
Definition: iobuf.h:48
#define BGX_CMR_TX_STAT8
CMR transmit statistics 8.
Definition: thunderx.h:757
RETURN_STATUS EFI_STATUS
Function return status for EFI API.
Definition: UefiBaseType.h:35
struct pci_device * pci
PCI device.
Definition: thunderx.h:583
#define EIO
Input/output error.
Definition: errno.h:433
#define TXNIC_PF_CHAN_TX_CFG
Channel transmit configuration.
Definition: thunderx.h:509
#define BGX_CMR_CONFIG_LANE_TO_SDS(ls)
Definition: thunderx.h:675
userptr_t rqe
Receive queue entries.
Definition: thunderx.h:283
uint16_t count
Number of entries.
Definition: ena.h:22
#define TXNIC_QS_RBDR_TAIL(q)
Receive buffer descriptor ring tail pointer.
Definition: thunderx.h:227
struct net_device * alloc_etherdev(size_t priv_size)
Allocate Ethernet device.
Definition: ethernet.c:264
uint8_t cqe_type
Type.
Definition: thunderx.h:373
#define BGX_SPU_INT_RX_LINK_DOWN
Definition: thunderx.h:875
uint16_t len
Length.
Definition: thunderx.h:359
static void txnic_diag(struct txnic *vnic)
Show virtual NIC diagnostics (for debugging)
Definition: thunderx.c:76
PCI Enhanced Allocation.
#define cpu_to_le16(value)
Definition: byteswap.h:106
static void txnic_complete_sqe(struct txnic *vnic, struct txnic_cqe_send *cqe)
Complete send queue entry.
Definition: thunderx.c:230
struct txnic_cqe_send send
Send completion.
Definition: thunderx.h:381
void iounmap(volatile const void *io_addr)
Unmap I/O address.
#define cpu_to_be64(value)
Definition: byteswap.h:111
static void txnic_close(struct txnic *vnic)
Close virtual NIC.
Definition: thunderx.c:619
#define BGX_CMR_RX_STAT0
CMR receive statistics 0.
Definition: thunderx.h:682
#define TXNIC_PF_MPI_CFG_RSSI_BASE(ix)
Definition: thunderx.h:475
#define BGX_CMR_CONFIG_DATA_PKT_TX_EN
Definition: thunderx.h:671
uint32_t total
Total length.
Definition: thunderx.h:112
#define BGX_CMR_CONFIG_ENABLE
Definition: thunderx.h:669
unsigned int idx
Virtual NIC index.
Definition: thunderx.h:913
uint64_t addr
Address.
Definition: thunderx.h:238
#define TXNIC_PF_LMAC_CREDIT_CC_ENABLE
Definition: thunderx.h:503
static void txnic_refill_rq(struct txnic *vnic)
Refill receive queue.
Definition: thunderx.c:360
#define TXNIC_SQES
Number of send queue entries.
Definition: thunderx.h:167
#define TXNIC_SQ_SIZE
Send queue size.
Definition: thunderx.h:182
#define BGX_SPU_AN_STATUS_LINK_STATUS
Definition: thunderx.h:857
struct pci_driver txnic_pf_driver __pci_driver
NIC physical function PCI driver.
Definition: thunderx.c:1361
#define TXNIC_VF_BAR_SIZE
Virtual function BAR size.
Definition: thunderx.h:28
#define TXNIC_PF_TL3_CHAN(tl)
Traffic limiter 3 channel mapping.
Definition: thunderx.h:534
#define TXNIC_NUM_LMAC
Maximum number of Logical MACs (per BGX)
Definition: thunderx.h:40
u32 status2
Definition: ar9003_mac.h:69
#define TXNIC_PF_TL3_CFG_RR_QUANTUM_DEFAULT
Definition: thunderx.h:530
struct net_device * netdev
Network device.
Definition: thunderx.h:423
#define BGX_SPU_INT_TRAINING_DONE
Definition: thunderx.h:863
#define TXNIC_PF_CHAN_RX_BP_CFG
Channel receive backpressure configuration.
Definition: thunderx.h:517
#define BGX_CMR_RX_LMACS
CMR receive logical MACs.
Definition: thunderx.h:787
#define TXNIC_BGX_BAR_SIZE
BGX BAR size.
Definition: thunderx.h:34
#define TXNIC_PF_QS_SQ_CFG(q)
Send queue configuration.
Definition: thunderx.h:571
size_t max_pkt_len
Maximum packet length.
Definition: netdevice.h:409
#define TXNIC_PF_QS(qs)
Queue set registers.
Definition: thunderx.h:545
#define BGX_SPU_CONTROL1_RESET
Definition: thunderx.h:800
int training
Link training is in use.
Definition: thunderx.h:938
struct list_head list
List of BGX Ethernet interfaces.
Definition: thunderx.h:941
#define BGX_CMR_TX_LMACS
CMR transmit logical MACs.
Definition: thunderx.h:793
void * pci_ioremap(struct pci_device *pci, unsigned long bus_addr, size_t len)
Map PCI bus address as an I/O address.
#define TXNIC_PF_TL4_CFG(tl)
Traffic limiter 4 configuration.
Definition: thunderx.h:538
static void txnic_destroy_rq(struct txnic *vnic)
Destroy receive queue.
Definition: thunderx.c:330
struct txnic_cq cq
Completion queue.
Definition: thunderx.h:430
#define BGX_CMR_GLOBAL_CONFIG_FCS_STRIP
Definition: thunderx.h:679
#define TXNIC_PF_LMAC_CREDIT_CC_UNIT_CNT_DEFAULT
Definition: thunderx.h:498
#define BGX_CMR_RX_STAT7
CMR receive statistics 7.
Definition: thunderx.h:703
#define TXNIC_CQ_DISABLE_MAX_WAIT_MS
Maximum time to wait for a completion queue to disable.
Definition: thunderx.h:307
#define TXNIC_QS_RBDR_CFG(q)
Receive buffer descriptor ring configuration.
Definition: thunderx.h:212
A completion queue entry.
Definition: thunderx.h:377
unsigned int prod
Producer counter.
Definition: thunderx.h:187
union txnic_lmac_address mac
MAC address.
Definition: thunderx.h:916
uint8_t hw_addr[MAX_HW_ADDR_LEN]
Hardware address.
Definition: netdevice.h:381
#define NULL
NULL pointer (VOID *)
Definition: Base.h:362
#define ETIMEDOUT
Connection timed out.
Definition: errno.h:669
static int txnic_lmac_transmit(struct net_device *netdev, struct io_buffer *iobuf)
Transmit packet.
Definition: thunderx.c:1053
#define TXNIC_QS_SQ_CFG_ENA
Definition: thunderx.h:81
String functions.
#define PCI_ROM(_vendor, _device, _name, _description, _data)
Definition: pci.h:286
#define BGX_CMR_TX_STAT5
CMR transmit statistics 5.
Definition: thunderx.h:748
static int txnic_pf_probe(struct pci_device *pci)
Probe PCI device.
Definition: thunderx.c:1248
#define TXNIC_CQ_SIZE
Completion queue size.
Definition: thunderx.h:399
void writeq(uint64_t data, volatile uint64_t *io_addr)
Write 64-bit qword to memory-mapped device.
static int txnic_lmac_probe_all(struct txnic_pf *pf, struct txnic_bgx *bgx)
Probe all LMACs on a BGX Ethernet interface.
Definition: thunderx.c:1178
FILE_LICENCE(GPL2_OR_LATER_OR_UBDL)
#define TXNIC_PF_QS_CFG
Queue set configuration.
Definition: thunderx.h:548
void eth_random_addr(void *hw_addr)
Generate random Ethernet address.
Definition: ethernet.c:159
#define BGX_CMR_RX_DMAC_CTL
CMR destination MAC control.
Definition: thunderx.h:715
struct txnic_bgx * bgx
Containing BGX Ethernet interface.
Definition: thunderx.h:911
#define TXNIC_PF_QS_RQ_CFG_CACHING_ALL
Definition: thunderx.h:555
Reduced XAUI.
Definition: thunderx.h:882
static int txnic_bgx_detect(struct txnic_bgx *bgx)
Detect BGX Ethernet interface LMAC type.
Definition: thunderx.c:1405
#define BGX_CMR_TX_LMACS_LMACS_SET(ct)
Definition: thunderx.h:796
#define TXNIC_RQ_STRIDE
Receive queue stride.
Definition: thunderx.h:271
#define TXNIC_QS_RBDR_HEAD(q)
Receive buffer descriptor ring head pointer.
Definition: thunderx.h:224
#define TXNIC_QS_CQ_CFG_ENA
Definition: thunderx.h:297
#define BGX_SPU_AN_STATUS_LP_AN_ABLE
Definition: thunderx.h:858
String functions.
#define TXNIC_QS_SQ_TAIL(q)
Send queue tail pointer.
Definition: thunderx.h:94
#define BGX_SPU_INT_ALGNLOS
Definition: thunderx.h:871
struct txnic_cqe_rx rx
Receive completion.
Definition: thunderx.h:383
void * memset(void *dest, int character, size_t len) __nonnull
A persistent I/O buffer.
Definition: iobuf.h:33
#define TXNIC_SQ_STOP_MAX_WAIT_MS
Maximum time to wait for a send queue to stop.
Definition: thunderx.h:107
#define BGX_CMR_TX_STAT11
CMR transmit statistics 11.
Definition: thunderx.h:766
static unsigned int txnic_address_bgx(uint64_t addr)
Calculate BGX Ethernet interface index.
Definition: thunderx.h:66
#define BGX_CMR_RX_STAT8
CMR receive statistics 8.
Definition: thunderx.h:706