akaros/kern/drivers/net/etherigbe.c
<<
>>
Prefs
   1/* This file is part of the UCB release of Plan 9. It is subject to the license
   2 * terms in the LICENSE file found in the top-level directory of this
   3 * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
   4 * part of the UCB release of Plan 9, including this file, may be copied,
   5 * modified, propagated, or distributed except according to the terms contained
   6 * in the LICENSE file. */
   7
   8/*
   9 * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
  10 * as found on the Intel PRO/1000 series of adapters:
  11 *      82543GC Intel PRO/1000 T
  12 *      82544EI Intel PRO/1000 XT
  13 *      82540EM Intel PRO/1000 MT
  14 *      82541[GP]I
  15 *      82547GI
  16 *      82546GB
  17 *      82546EB
  18 * To Do:
  19 *      finish autonegotiation code;
  20 *      integrate fiber stuff back in (this ONLY handles
  21 *      the CAT5 cards at the moment);
  22 *      add checksum-offload;
  23 *      add tuning control via ctl file;
  24 *      this driver is little-endian specific.
  25 *
  26 * Modified by brho:
  27 *      ported to Akaros
  28 *      fixed mii bugs (allocation, startup, miirw, etc)
  29 *      fixed CLS bug (continue -> break)
  30 *      made sure igbepci only runs once, even if it fails */
  31
  32#include <arch/pci.h>
  33#include <assert.h>
  34#include <cpio.h>
  35#include <error.h>
  36#include <kmalloc.h>
  37#include <kref.h>
  38#include <net/ip.h>
  39#include <ns.h>
  40#include <pmap.h>
  41#include <slab.h>
  42#include <smp.h>
  43#include <stdio.h>
  44#include <string.h>
  45
  46#include "ethermii.h"
  47
  48#define ilock(x) spin_lock_irqsave(x)
  49#define iunlock(x) spin_unlock_irqsave(x)
  50
  51enum { i82542 = (0x1000 << 16) | 0x8086,
  52       i82543gc = (0x1004 << 16) | 0x8086,
  53       i82544ei = (0x1008 << 16) | 0x8086,
  54       i82544eif = (0x1009 << 16) | 0x8086,
  55       i82544gc = (0x100d << 16) | 0x8086,
  56       i82540em = (0x100E << 16) | 0x8086,
  57       i82540eplp = (0x101E << 16) | 0x8086,
  58       i82545em = (0x100F << 16) | 0x8086,
  59       i82545gmc = (0x1026 << 16) | 0x8086,
  60       i82547ei = (0x1019 << 16) | 0x8086,
  61       i82547gi = (0x1075 << 16) | 0x8086,
  62       i82541ei = (0x1013 << 16) | 0x8086,
  63       i82541gi = (0x1076 << 16) | 0x8086,
  64       i82541gi2 = (0x1077 << 16) | 0x8086,
  65       i82541pi = (0x107c << 16) | 0x8086,
  66       i82546gb = (0x1079 << 16) | 0x8086,
  67       i82546eb = (0x1010 << 16) | 0x8086,
  68};
  69
  70enum { Ctrl = 0x00000000,    /* Device Control */
  71       Ctrldup = 0x00000004, /* Device Control Duplicate */
  72       Status = 0x00000008,  /* Device Status */
  73       Eecd = 0x00000010,    /* EEPROM/Flash Control/Data */
  74       Ctrlext = 0x00000018, /* Extended Device Control */
  75       Mdic = 0x00000020,    /* MDI Control */
  76       Fcal = 0x00000028,    /* Flow Control Address Low */
  77       Fcah = 0x0000002C,    /* Flow Control Address High */
  78       Fct = 0x00000030,     /* Flow Control Type */
  79       Icr = 0x000000C0,     /* Interrupt Cause Read */
  80       Ics = 0x000000C8,     /* Interrupt Cause Set */
  81       Ims = 0x000000D0,     /* Interrupt Mask Set/Read */
  82       Imc = 0x000000D8,     /* Interrupt mask Clear */
  83       Rctl = 0x00000100,    /* Receive Control */
  84       Fcttv = 0x00000170,   /* Flow Control Transmit Timer Value */
  85       Txcw = 0x00000178,    /* Transmit Configuration Word */
  86       Rxcw = 0x00000180,    /* Receive Configuration Word */
  87       /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
  88       Tctl = 0x00000400,   /* Transmit Control */
  89       Tipg = 0x00000410,   /* Transmit IPG */
  90       Tbt = 0x00000448,    /* Transmit Burst Timer */
  91       Ait = 0x00000458,    /* Adaptive IFS Throttle */
  92       Fcrtl = 0x00002160,  /* Flow Control RX Threshold Low */
  93       Fcrth = 0x00002168,  /* Flow Control Rx Threshold High */
  94       Rdfh = 0x00002410,   /* Receive data fifo head */
  95       Rdft = 0x00002418,   /* Receive data fifo tail */
  96       Rdfhs = 0x00002420,  /* Receive data fifo head saved */
  97       Rdfts = 0x00002428,  /* Receive data fifo tail saved */
  98       Rdfpc = 0x00002430,  /* Receive data fifo packet count */
  99       Rdbal = 0x00002800,  /* Rd Base Address Low */
 100       Rdbah = 0x00002804,  /* Rd Base Address High */
 101       Rdlen = 0x00002808,  /* Receive Descriptor Length */
 102       Rdh = 0x00002810,    /* Receive Descriptor Head */
 103       Rdt = 0x00002818,    /* Receive Descriptor Tail */
 104       Rdtr = 0x00002820,   /* Receive Descriptor Timer Ring */
 105       Rxdctl = 0x00002828, /* Receive Descriptor Control */
 106       Radv = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
 107       Txdmac = 0x00003000, /* Transfer DMA Control */
 108       Ett = 0x00003008,    /* Early Transmit Control */
 109       Tdfh = 0x00003410,   /* Transmit data fifo head */
 110       Tdft = 0x00003418,   /* Transmit data fifo tail */
 111       Tdfhs = 0x00003420,  /* Transmit data Fifo Head saved */
 112       Tdfts = 0x00003428,  /* Transmit data fifo tail saved */
 113       Tdfpc = 0x00003430,  /* Trasnmit data Fifo packet count */
 114       Tdbal = 0x00003800,  /* Td Base Address Low */
 115       Tdbah = 0x00003804,  /* Td Base Address High */
 116       Tdlen = 0x00003808,  /* Transmit Descriptor Length */
 117       Tdh = 0x00003810,    /* Transmit Descriptor Head */
 118       Tdt = 0x00003818,    /* Transmit Descriptor Tail */
 119       Tidv = 0x00003820,   /* Transmit Interrupt Delay Value */
 120       Txdctl = 0x00003828, /* Transmit Descriptor Control */
 121       Tadv = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
 122
 123       Statistics = 0x00004000, /* Start of Statistics Area */
 124       Gorcl = 0x88 / 4,        /* Good Octets Received Count */
 125       Gotcl = 0x90 / 4,        /* Good Octets Transmitted Count */
 126       Torl = 0xC0 / 4,         /* Total Octets Received */
 127       Totl = 0xC8 / 4,         /* Total Octets Transmitted */
 128       Nstatistics = 64,
 129
 130       Rxcsum = 0x00005000, /* Receive Checksum Control */
 131       Mta = 0x00005200,    /* Multicast Table Array */
 132       Ral = 0x00005400,    /* Receive Address Low */
 133       Rah = 0x00005404,    /* Receive Address High */
 134       Manc = 0x00005820,   /* Management Control */
 135};
 136
 137enum {                          /* Ctrl */
 138       Bem = 0x00000002,        /* Big Endian Mode */
 139       Prior = 0x00000004,      /* Priority on the PCI bus */
 140       Lrst = 0x00000008,       /* Link Reset */
 141       Asde = 0x00000020,       /* Auto-Speed Detection Enable */
 142       Slu = 0x00000040,        /* Set Link Up */
 143       Ilos = 0x00000080,       /* Invert Loss of Signal (LOS) */
 144       SspeedMASK = 0x00000300, /* Speed Selection */
 145       SspeedSHIFT = 8,
 146       Sspeed10 = 0x00000000,      /* 10Mb/s */
 147       Sspeed100 = 0x00000100,     /* 100Mb/s */
 148       Sspeed1000 = 0x00000200,    /* 1000Mb/s */
 149       Frcspd = 0x00000800,        /* Force Speed */
 150       Frcdplx = 0x00001000,       /* Force Duplex */
 151       SwdpinsloMASK = 0x003C0000, /* Software Defined Pins - lo nibble */
 152       SwdpinsloSHIFT = 18,
 153       SwdpioloMASK = 0x03C00000, /* Software Defined Pins - I or O */
 154       SwdpioloSHIFT = 22,
 155       Devrst = 0x04000000, /* Device Reset */
 156       Rfce = 0x08000000,   /* Receive Flow Control Enable */
 157       Tfce = 0x10000000,   /* Transmit Flow Control Enable */
 158       Vme = 0x40000000,    /* VLAN Mode Enable */
 159};
 160
 161/*
 162 * can't find Tckok nor Rbcok in any Intel docs,
 163 * but even 82543gc docs define Lanid.
 164 */
 165enum {                     /* Status */
 166       Lu = 0x00000002,    /* Link Up */
 167       Lanid = 0x0000000C, /* mask for Lan ID. (function id) */
 168       //       Tckok           = 0x00000004,   /* Transmit clock is running */
 169       //       Rbcok           = 0x00000008,   /* Receive clock is running */
 170       Txoff = 0x00000010,      /* Transmission Paused */
 171       Tbimode = 0x00000020,    /* TBI Mode Indication */
 172       LspeedMASK = 0x000000C0, /* Link Speed Setting */
 173       LspeedSHIFT = 6,
 174       Lspeed10 = 0x00000000,      /* 10Mb/s */
 175       Lspeed100 = 0x00000040,     /* 100Mb/s */
 176       Lspeed1000 = 0x00000080,    /* 1000Mb/s */
 177       Mtxckok = 0x00000400,       /* MTX clock is running */
 178       Pci66 = 0x00000800,         /* PCI Bus speed indication */
 179       Bus64 = 0x00001000,         /* PCI Bus width indication */
 180       Pcixmode = 0x00002000,      /* PCI-X mode */
 181       PcixspeedMASK = 0x0000C000, /* PCI-X bus speed */
 182       PcixspeedSHIFT = 14,
 183       Pcix66 = 0x00000000,  /* 50-66MHz */
 184       Pcix100 = 0x00004000, /* 66-100MHz */
 185       Pcix133 = 0x00008000, /* 100-133MHz */
 186};
 187
 188enum {                  /* Ctrl and Status */
 189       Fd = 0x00000001, /* Full-Duplex */
 190       AsdvMASK = 0x00000300,
 191       AsdvSHIFT = 8,
 192       Asdv10 = 0x00000000,   /* 10Mb/s */
 193       Asdv100 = 0x00000100,  /* 100Mb/s */
 194       Asdv1000 = 0x00000200, /* 1000Mb/s */
 195};
 196
 197enum {                         /* Eecd */
 198       Sk = 0x00000001,        /* Clock input to the EEPROM */
 199       Cs = 0x00000002,        /* Chip Select */
 200       Di = 0x00000004,        /* Data Input to the EEPROM */
 201       Do = 0x00000008,        /* Data Output from the EEPROM */
 202       Areq = 0x00000040,      /* EEPROM Access Request */
 203       Agnt = 0x00000080,      /* EEPROM Access Grant */
 204       Eepresent = 0x00000100, /* EEPROM Present */
 205       Eesz256 = 0x00000200,   /* EEPROM is 256 words not 64 */
 206       Eeszaddr = 0x00000400,  /* EEPROM size for 8254[17] */
 207       Spi = 0x00002000,       /* EEPROM is SPI not Microwire */
 208};
 209
 210enum {                             /* Ctrlext */
 211       Gpien = 0x0000000F,         /* General Purpose Interrupt Enables */
 212       SwdpinshiMASK = 0x000000F0, /* Software Defined Pins - hi nibble */
 213       SwdpinshiSHIFT = 4,
 214       SwdpiohiMASK = 0x00000F00, /* Software Defined Pins - I or O */
 215       SwdpiohiSHIFT = 8,
 216       Asdchk = 0x00001000,  /* ASD Check */
 217       Eerst = 0x00002000,   /* EEPROM Reset */
 218       Ips = 0x00004000,     /* Invert Power State */
 219       Spdbyps = 0x00008000, /* Speed Select Bypass */
 220};
 221
 222enum {              /* EEPROM content offsets */
 223       Ea = 0x00,   /* Ethernet Address */
 224       Cf = 0x03,   /* Compatibility Field */
 225       Pba = 0x08,  /* Printed Board Assembly number */
 226       Icw1 = 0x0A, /* Initialization Control Word 1 */
 227       Sid = 0x0B,  /* Subsystem ID */
 228       Svid = 0x0C, /* Subsystem Vendor ID */
 229       Did = 0x0D,  /* Device ID */
 230       Vid = 0x0E,  /* Vendor ID */
 231       Icw2 = 0x0F, /* Initialization Control Word 2 */
 232};
 233
 234enum {                        /* Mdic */
 235       MDIdMASK = 0x0000FFFF, /* Data */
 236       MDIdSHIFT = 0,
 237       MDIrMASK = 0x001F0000, /* PHY Register Address */
 238       MDIrSHIFT = 16,
 239       MDIpMASK = 0x03E00000, /* PHY Address */
 240       MDIpSHIFT = 21,
 241       MDIwop = 0x04000000,   /* Write Operation */
 242       MDIrop = 0x08000000,   /* Read Operation */
 243       MDIready = 0x10000000, /* End of Transaction */
 244       MDIie = 0x20000000,    /* Interrupt Enable */
 245       MDIe = 0x40000000,     /* Error */
 246};
 247
 248enum {                      /* Icr, Ics, Ims, Imc */
 249       Txdw = 0x00000001,   /* Transmit Descriptor Written Back */
 250       Txqe = 0x00000002,   /* Transmit Queue Empty */
 251       Lsc = 0x00000004,    /* Link Status Change */
 252       Rxseq = 0x00000008,  /* Receive Sequence Error */
 253       Rxdmt0 = 0x00000010, /* Rd Minimum Threshold Reached */
 254       Rxo = 0x00000040,    /* Receiver Overrun */
 255       Rxt0 = 0x00000080,   /* Receiver Timer Interrupt */
 256       Mdac = 0x00000200,   /* MDIO Access Completed */
 257       Rxcfg = 0x00000400,  /* Receiving /C/ ordered sets */
 258       Gpi0 = 0x00000800,   /* General Purpose Interrupts */
 259       Gpi1 = 0x00001000,
 260       Gpi2 = 0x00002000,
 261       Gpi3 = 0x00004000,
 262};
 263
 264/*
 265 * The Mdic register isn't implemented on the 82543GC,
 266 * the software defined pins are used instead.
 267 * These definitions work for the Intel PRO/1000 T Server Adapter.
 268 * The direction pin bits are read from the EEPROM.
 269 */
 270enum { Mdd = ((1 << 2) << SwdpinsloSHIFT), /* data */
 271       Mddo = ((1 << 2) << SwdpioloSHIFT), /* pin direction */
 272       Mdc = ((1 << 3) << SwdpinsloSHIFT), /* clock */
 273       Mdco = ((1 << 3) << SwdpioloSHIFT), /* pin direction */
 274       Mdr = ((1 << 0) << SwdpinshiSHIFT), /* reset */
 275       Mdro = ((1 << 0) << SwdpiohiSHIFT), /* pin direction */
 276};
 277
 278enum {                             /* Txcw */
 279       TxcwFd = 0x00000020,        /* Full Duplex */
 280       TxcwHd = 0x00000040,        /* Half Duplex */
 281       TxcwPauseMASK = 0x00000180, /* Pause */
 282       TxcwPauseSHIFT = 7,
 283       TxcwPs = (1 << TxcwPauseSHIFT), /* Pause Supported */
 284       TxcwAs = (2 << TxcwPauseSHIFT), /* Asymmetric FC desired */
 285       TxcwRfiMASK = 0x00003000,       /* Remote Fault Indication */
 286       TxcwRfiSHIFT = 12,
 287       TxcwNpr = 0x00008000,    /* Next Page Request */
 288       TxcwConfig = 0x40000000, /* Transmit COnfig Control */
 289       TxcwAne = 0x80000000,    /* Auto-Negotiation Enable */
 290};
 291
 292enum {                           /* Rxcw */
 293       Rxword = 0x0000FFFF,      /* Data from auto-negotiation process */
 294       Rxnocarrier = 0x04000000, /* Carrier Sense indication */
 295       Rxinvalid = 0x08000000,   /* Invalid Symbol during configuration */
 296       Rxchange = 0x10000000,    /* Change to the Rxword indication */
 297       Rxconfig = 0x20000000,    /* /C/ order set reception indication */
 298       Rxsync = 0x40000000,      /* Lost bit synchronization indication */
 299       Anc = 0x80000000,         /* Auto Negotiation Complete */
 300};
 301
 302enum {                            /* Rctl */
 303       Rrst = 0x00000001,         /* Receiver Software Reset */
 304       Ren = 0x00000002,          /* Receiver Enable */
 305       Sbp = 0x00000004,          /* Store Bad Packets */
 306       Upe = 0x00000008,          /* Unicast Promiscuous Enable */
 307       Mpe = 0x00000010,          /* Multicast Promiscuous Enable */
 308       Lpe = 0x00000020,          /* Long Packet Reception Enable */
 309       LbmMASK = 0x000000C0,      /* Loopback Mode */
 310       LbmOFF = 0x00000000,       /* No Loopback */
 311       LbmTBI = 0x00000040,       /* TBI Loopback */
 312       LbmMII = 0x00000080,       /* GMII/MII Loopback */
 313       LbmXCVR = 0x000000C0,      /* Transceiver Loopback */
 314       RdtmsMASK = 0x00000300,    /* Rd Minimum Threshold Size */
 315       RdtmsHALF = 0x00000000,    /* Threshold is 1/2 Rdlen */
 316       RdtmsQUARTER = 0x00000100, /* Threshold is 1/4 Rdlen */
 317       RdtmsEIGHTH = 0x00000200,  /* Threshold is 1/8 Rdlen */
 318       MoMASK = 0x00003000,       /* Multicast Offset */
 319       Mo47b36 = 0x00000000,      /* bits [47:36] of received address */
 320       Mo46b35 = 0x00001000,      /* bits [46:35] of received address */
 321       Mo45b34 = 0x00002000,      /* bits [45:34] of received address */
 322       Mo43b32 = 0x00003000,      /* bits [43:32] of received address */
 323       Bam = 0x00008000,          /* Broadcast Accept Mode */
 324       BsizeMASK = 0x00030000,    /* Receive Buffer Size */
 325       Bsize2048 = 0x00000000,    /* Bsex = 0 */
 326       Bsize1024 = 0x00010000,    /* Bsex = 0 */
 327       Bsize512 = 0x00020000,     /* Bsex = 0 */
 328       Bsize256 = 0x00030000,     /* Bsex = 0 */
 329       Bsize16384 = 0x00010000,   /* Bsex = 1 */
 330       Vfe = 0x00040000,          /* VLAN Filter Enable */
 331       Cfien = 0x00080000,        /* Canonical Form Indicator Enable */
 332       Cfi = 0x00100000,          /* Canonical Form Indicator value */
 333       Dpf = 0x00400000,          /* Discard Pause Frames */
 334       Pmcf = 0x00800000,         /* Pass MAC Control Frames */
 335       Bsex = 0x02000000,         /* Buffer Size Extension */
 336       Secrc = 0x04000000,        /* Strip CRC from incoming packet */
 337};
 338
 339enum {                      /* Tctl */
 340       Trst = 0x00000001,   /* Transmitter Software Reset */
 341       Ten = 0x00000002,    /* Transmit Enable */
 342       Psp = 0x00000008,    /* Pad Short Packets */
 343       CtMASK = 0x00000FF0, /* Collision Threshold */
 344       CtSHIFT = 4,
 345       ColdMASK = 0x003FF000, /* Collision Distance */
 346       ColdSHIFT = 12,
 347       Swxoff = 0x00400000, /* Sofware XOFF Transmission */
 348       Pbe = 0x00800000,    /* Packet Burst Enable */
 349       Rtlc = 0x01000000,   /* Re-transmit on Late Collision */
 350       Nrtu = 0x02000000,   /* No Re-transmit on Underrrun */
 351};
 352
 353enum {                           /* [RT]xdctl */
 354       PthreshMASK = 0x0000003F, /* Prefetch Threshold */
 355       PthreshSHIFT = 0,
 356       HthreshMASK = 0x00003F00, /* Host Threshold */
 357       HthreshSHIFT = 8,
 358       WthreshMASK = 0x003F0000, /* Writeback Threshold */
 359       WthreshSHIFT = 16,
 360       Gran = 0x01000000,        /* Granularity */
 361       LthreshMASK = 0xFE000000, /* Low Threshold */
 362       LthreshSHIFT = 25,
 363};
 364
 365enum {                        /* Rxcsum */
 366       PcssMASK = 0x000000FF, /* Packet Checksum Start */
 367       PcssSHIFT = 0,
 368       Ipofl = 0x00000100, /* IP Checksum Off-load Enable */
 369       Tuofl = 0x00000200, /* TCP/UDP Checksum Off-load Enable */
 370};
 371
 372enum {                     /* Manc */
 373       Arpen = 0x00002000, /* Enable ARP Request Filtering */
 374};
 375
 376enum {                         /* Receive Delay Timer Ring */
 377       DelayMASK = 0x0000FFFF, /* delay timer in 1.024nS increments */
 378       DelaySHIFT = 0,
 379       Fpd = 0x80000000, /* Flush partial Descriptor Block */
 380};
 381
 382typedef struct Rd { /* Receive Descriptor */
 383        unsigned int addr[2];
 384        uint16_t length;
 385        uint16_t checksum;
 386        uint8_t status;
 387        uint8_t errors;
 388        uint16_t special;
 389} Rd;
 390
 391enum {               /* Rd status */
 392       Rdd = 0x01,   /* Descriptor Done */
 393       Reop = 0x02,  /* End of Packet */
 394       Ixsm = 0x04,  /* Ignore Checksum Indication */
 395       Vp = 0x08,    /* Packet is 802.1Q (matched VET) */
 396       Tcpcs = 0x20, /* TCP Checksum Calculated on Packet */
 397       Ipcs = 0x40,  /* IP Checksum Calculated on Packet */
 398       Pif = 0x80,   /* Passed in-exact filter */
 399};
 400
 401enum {              /* Rd errors */
 402       Ce = 0x01,   /* CRC Error or Alignment Error */
 403       Se = 0x02,   /* Symbol Error */
 404       Seq = 0x04,  /* Sequence Error */
 405       Cxe = 0x10,  /* Carrier Extension Error */
 406       Tcpe = 0x20, /* TCP/UDP Checksum Error */
 407       Ipe = 0x40,  /* IP Checksum Error */
 408       Rxe = 0x80,  /* RX Data Error */
 409};
 410
 411typedef struct Td Td;
 412struct Td { /* Transmit Descriptor */
 413        union {
 414                unsigned int addr[2]; /* Data */
 415                struct {              /* Context */
 416                        uint8_t ipcss;
 417                        uint8_t ipcso;
 418                        uint16_t ipcse;
 419                        uint8_t tucss;
 420                        uint8_t tucso;
 421                        uint16_t tucse;
 422                };
 423        };
 424        unsigned int control;
 425        unsigned int status;
 426};
 427
 428enum {                       /* Td control */
 429       LenMASK = 0x000FFFFF, /* Data/Packet Length Field */
 430       LenSHIFT = 0,
 431       DtypeCD = 0x00000000,  /* Data Type 'Context Descriptor' */
 432       DtypeDD = 0x00100000,  /* Data Type 'Data Descriptor' */
 433       PtypeTCP = 0x01000000, /* TCP/UDP Packet Type (CD) */
 434       Teop = 0x01000000,     /* End of Packet (DD) */
 435       PtypeIP = 0x02000000,  /* IP Packet Type (CD) */
 436       Ifcs = 0x02000000,     /* Insert FCS (DD) */
 437       Tse = 0x04000000,      /* TCP Segmentation Enable */
 438       Rs = 0x08000000,       /* Report Status */
 439       Rps = 0x10000000,      /* Report Status Sent */
 440       Dext = 0x20000000,     /* Descriptor Extension */
 441       Vle = 0x40000000,      /* VLAN Packet Enable */
 442       Ide = 0x80000000,      /* Interrupt Delay Enable */
 443};
 444
 445enum {                          /* Td status */
 446       Tdd = 0x00000001,        /* Descriptor Done */
 447       Ec = 0x00000002,         /* Excess Collisions */
 448       Lc = 0x00000004,         /* Late Collision */
 449       Tu = 0x00000008,         /* Transmit Underrun */
 450       Iixsm = 0x00000100,      /* Insert IP Checksum */
 451       Itxsm = 0x00000200,      /* Insert TCP/UDP Checksum */
 452       HdrlenMASK = 0x0000FF00, /* Header Length (Tse) */
 453       HdrlenSHIFT = 8,
 454       VlanMASK = 0x0FFF0000, /* VLAN Identifier */
 455       VlanSHIFT = 16,
 456       Tcfi = 0x10000000,    /* Canonical Form Indicator */
 457       PriMASK = 0xE0000000, /* User Priority */
 458       PriSHIFT = 29,
 459       MssMASK = 0xFFFF0000, /* Maximum Segment Size (Tse) */
 460       MssSHIFT = 16,
 461};
 462
 463enum { Nrd = 256, /* multiple of 8 */
 464       Ntd = 64,  /* multiple of 8 */
 465       Rbsz = 2048,
 466};
 467
 468struct ctlr {
 469        int port;
 470        struct pci_device *pci;
 471        struct ctlr *next;
 472        struct ether *edev;
 473        int active;
 474        int started;
 475        int id;
 476        int cls;
 477        uint16_t eeprom[0x40];
 478
 479        qlock_t alock; /* attach */
 480        void *alloc;   /* receive/transmit descriptors */
 481        int nrd;
 482        int ntd;
 483
 484        int *nic;
 485        spinlock_t imlock;
 486        int im; /* interrupt mask */
 487
 488        struct mii *mii;
 489        struct rendez lrendez;
 490        int lim;
 491
 492        int link;
 493
 494        qlock_t slock;
 495        unsigned int statistics[Nstatistics];
 496        unsigned int lsleep;
 497        unsigned int lintr;
 498        unsigned int rsleep;
 499        unsigned int rintr;
 500        unsigned int txdw;
 501        unsigned int tintr;
 502        unsigned int ixsm;
 503        unsigned int ipcs;
 504        unsigned int tcpcs;
 505
 506        uint8_t ra[Eaddrlen]; /* receive address */
 507        uint32_t mta[128];    /* multicast table array */
 508
 509        struct rendez rrendez;
 510        int rim;
 511        int rdfree;
 512        Rd *rdba;          /* receive descriptor base address */
 513        struct block **rb; /* receive buffers */
 514        int rdh;           /* receive descriptor head */
 515        int rdt;           /* receive descriptor tail */
 516        int rdtr;          /* receive delay timer ring value */
 517
 518        spinlock_t tlock;
 519        int tbusy;
 520        int tdfree;
 521        Td *tdba;          /* transmit descriptor base address */
 522        struct block **tb; /* transmit buffers */
 523        int tdh;           /* transmit descriptor head */
 524        int tdt;           /* transmit descriptor tail */
 525
 526        int txcw;
 527        int fcrtl;
 528        int fcrth;
 529};
 530
 531static inline uint32_t csr32r(struct ctlr *c, uintptr_t reg)
 532{
 533        return read_mmreg32((uintptr_t)(c->nic + (reg / 4)));
 534}
 535
 536static inline void csr32w(struct ctlr *c, uintptr_t reg, uint32_t val)
 537{
 538        write_mmreg32((uintptr_t)(c->nic + (reg / 4)), val);
 539}
 540
 541static struct ctlr *igbectlrhead;
 542static struct ctlr *igbectlrtail;
 543
 544static char *statistics[Nstatistics] = {
 545    "CRC Error",
 546    "Alignment Error",
 547    "Symbol Error",
 548    "RX Error",
 549    "Missed Packets",
 550    "Single Collision",
 551    "Excessive Collisions",
 552    "Multiple Collision",
 553    "Late Collisions",
 554    NULL,
 555    "Collision",
 556    "Transmit Underrun",
 557    "Defer",
 558    "Transmit - No CRS",
 559    "Sequence Error",
 560    "Carrier Extension Error",
 561    "Receive Error Length",
 562    NULL,
 563    "XON Received",
 564    "XON Transmitted",
 565    "XOFF Received",
 566    "XOFF Transmitted",
 567    "FC Received Unsupported",
 568    "Packets Received (64 Bytes)",
 569    "Packets Received (65-127 Bytes)",
 570    "Packets Received (128-255 Bytes)",
 571    "Packets Received (256-511 Bytes)",
 572    "Packets Received (512-1023 Bytes)",
 573    "Packets Received (1024-1522 Bytes)",
 574    "Good Packets Received",
 575    "Broadcast Packets Received",
 576    "Multicast Packets Received",
 577    "Good Packets Transmitted",
 578    NULL,
 579    "Good Octets Received",
 580    NULL,
 581    "Good Octets Transmitted",
 582    NULL,
 583    NULL,
 584    NULL,
 585    "Receive No Buffers",
 586    "Receive Undersize",
 587    "Receive Fragment",
 588    "Receive Oversize",
 589    "Receive Jabber",
 590    NULL,
 591    NULL,
 592    NULL,
 593    "Total Octets Received",
 594    NULL,
 595    "Total Octets Transmitted",
 596    NULL,
 597    "Total Packets Received",
 598    "Total Packets Transmitted",
 599    "Packets Transmitted (64 Bytes)",
 600    "Packets Transmitted (65-127 Bytes)",
 601    "Packets Transmitted (128-255 Bytes)",
 602    "Packets Transmitted (256-511 Bytes)",
 603    "Packets Transmitted (512-1023 Bytes)",
 604    "Packets Transmitted (1024-1522 Bytes)",
 605    "Multicast Packets Transmitted",
 606    "Broadcast Packets Transmitted",
 607    "TCP Segmentation Context Transmitted",
 608    "TCP Segmentation Context Fail",
 609};
 610
 611static void igbe_print_rd(struct Rd *rd)
 612{
 613        printk("Rd %p: stat 0x%02x, err 0x%02x, len 0x%04x, check 0x%04x, "
 614               "spec 0x%04x, addr[1] 0x%08x, addr[0] 0x%08x\n",
 615               rd, rd->status, rd->errors, rd->length, rd->checksum,
 616               rd->special, rd->addr[1], rd->addr[0]);
 617}
 618
 619static long igbeifstat(struct ether *edev, void *a, long n, uint32_t offset)
 620{
 621        struct ctlr *ctlr;
 622        char *p, *s;
 623        int i, l, r;
 624        uint64_t tuvl, ruvl;
 625
 626        ctlr = edev->ctlr;
 627        qlock(&ctlr->slock);
 628        p = kzmalloc(READSTR, 0);
 629        if (p == NULL) {
 630                qunlock(&ctlr->slock);
 631                error(ENOMEM, ERROR_FIXME);
 632        }
 633        l = 0;
 634        for (i = 0; i < Nstatistics; i++) {
 635                r = csr32r(ctlr, Statistics + i * 4);
 636                if ((s = statistics[i]) == NULL)
 637                        continue;
 638                switch (i) {
 639                case Gorcl:
 640                case Gotcl:
 641                case Torl:
 642                case Totl:
 643                        ruvl = r;
 644                        ruvl +=
 645                            ((uint64_t)csr32r(ctlr, Statistics + (i + 1) * 4))
 646                            << 32;
 647                        tuvl = ruvl;
 648                        tuvl += ctlr->statistics[i];
 649                        tuvl += ((uint64_t)ctlr->statistics[i + 1]) << 32;
 650                        if (tuvl == 0)
 651                                continue;
 652                        ctlr->statistics[i] = tuvl;
 653                        ctlr->statistics[i + 1] = tuvl >> 32;
 654                        l += snprintf(p + l, READSTR - l, "%s: %llud %llud\n",
 655                                      s, tuvl, ruvl);
 656                        i++;
 657                        break;
 658
 659                default:
 660                        ctlr->statistics[i] += r;
 661                        if (ctlr->statistics[i] == 0)
 662                                continue;
 663                        l += snprintf(p + l, READSTR - l, "%s: %ud %ud\n", s,
 664                                      ctlr->statistics[i], r);
 665                        break;
 666                }
 667        }
 668
 669        l += snprintf(p + l, READSTR - l, "lintr: %ud %ud\n", ctlr->lintr,
 670                      ctlr->lsleep);
 671        l += snprintf(p + l, READSTR - l, "rintr: %ud %ud\n", ctlr->rintr,
 672                      ctlr->rsleep);
 673        l += snprintf(p + l, READSTR - l, "tintr: %ud %ud\n", ctlr->tintr,
 674                      ctlr->txdw);
 675        l += snprintf(p + l, READSTR - l, "ixcs: %ud %ud %ud\n", ctlr->ixsm,
 676                      ctlr->ipcs, ctlr->tcpcs);
 677        l += snprintf(p + l, READSTR - l, "rdtr: %ud\n", ctlr->rdtr);
 678        l += snprintf(p + l, READSTR - l, "Ctrlext: %08x\n",
 679                      csr32r(ctlr, Ctrlext));
 680
 681        l += snprintf(p + l, READSTR - l, "eeprom:");
 682        for (i = 0; i < 0x40; i++) {
 683                if (i && ((i & 0x07) == 0))
 684                        l += snprintf(p + l, READSTR - l, "\n       ");
 685                l += snprintf(p + l, READSTR - l, " %4.4uX", ctlr->eeprom[i]);
 686        }
 687        l += snprintf(p + l, READSTR - l, "\n");
 688
 689        if (ctlr->mii != NULL && ctlr->mii->curphy != NULL) {
 690                l += snprintf(p + l, READSTR - l, "phy:   ");
 691                for (i = 0; i < NMiiPhyr; i++) {
 692                        if (i && ((i & 0x07) == 0))
 693                                l += snprintf(p + l, READSTR - l, "\n       ");
 694                        r = miimir(ctlr->mii, i);
 695                        l += snprintf(p + l, READSTR - l, " %4.4uX", r);
 696                }
 697                snprintf(p + l, READSTR - l, "\n");
 698        }
 699        n = readstr(offset, a, n, p);
 700        kfree(p);
 701        qunlock(&ctlr->slock);
 702
 703        return n;
 704}
 705
 706enum { CMrdtr,
 707};
 708
 709static struct cmdtab igbectlmsg[] = {
 710    {CMrdtr, "rdtr", 2},
 711};
 712
 713static long igbectl(struct ether *edev, void *buf, size_t n)
 714{
 715        ERRSTACK(2);
 716        int v;
 717        char *p;
 718        struct ctlr *ctlr;
 719        struct cmdbuf *cb;
 720        struct cmdtab *ct;
 721
 722        if ((ctlr = edev->ctlr) == NULL)
 723                error(ENODEV, ERROR_FIXME);
 724
 725        cb = parsecmd(buf, n);
 726        if (waserror()) {
 727                kfree(cb);
 728                nexterror();
 729        }
 730
 731        ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
 732        switch (ct->index) {
 733        case CMrdtr:
 734                v = strtol(cb->f[1], &p, 0);
 735                if (v < 0 || p == cb->f[1] || v > 0xFFFF)
 736                        error(EINVAL, ERROR_FIXME);
 737                ctlr->rdtr = v;
 738                csr32w(ctlr, Rdtr, Fpd | v);
 739                break;
 740        }
 741        kfree(cb);
 742        poperror();
 743
 744        return n;
 745}
 746
 747static void igbepromiscuous(void *arg, int on)
 748{
 749        int rctl;
 750        struct ctlr *ctlr;
 751        struct ether *edev;
 752
 753        edev = arg;
 754        ctlr = edev->ctlr;
 755
 756        rctl = csr32r(ctlr, Rctl);
 757        rctl &= ~MoMASK;
 758        rctl |= Mo47b36;
 759        if (on)
 760                rctl |= Upe | Mpe;
 761        else
 762                rctl &= ~(Upe | Mpe);
 763        csr32w(ctlr, Rctl, rctl | Mpe); /* temporarily keep Mpe on */
 764}
 765
 766static void igbemulticast(void *arg, uint8_t *addr, int add)
 767{
 768        int bit, x;
 769        struct ctlr *ctlr;
 770        struct ether *edev;
 771
 772        edev = arg;
 773        ctlr = edev->ctlr;
 774
 775        x = addr[5] >> 1;
 776        bit = ((addr[5] & 1) << 4) | (addr[4] >> 4);
 777        /*
 778         * multiple ether addresses can hash to the same filter bit,
 779         * so it's never safe to clear a filter bit.
 780         * if we want to clear filter bits, we need to keep track of
 781         * all the multicast addresses in use, clear all the filter bits,
 782         * then set the ones corresponding to in-use addresses.
 783         */
 784        if (add)
 785                ctlr->mta[x] |= 1 << bit;
 786        //      else
 787        //              ctlr->mta[x] &= ~(1<<bit);
 788
 789        csr32w(ctlr, Mta + x * 4, ctlr->mta[x]);
 790}
 791
 792static void igbeim(struct ctlr *ctlr, int im)
 793{
 794        ilock(&ctlr->imlock);
 795        ctlr->im |= im;
 796        csr32w(ctlr, Ims, ctlr->im);
 797        iunlock(&ctlr->imlock);
 798}
 799
 800static int igbelim(void *ctlr)
 801{
 802        return ((struct ctlr *)ctlr)->lim != 0;
 803}
 804
 805static void igbelproc(void *arg)
 806{
 807        struct ctlr *ctlr;
 808        struct ether *edev;
 809        struct miiphy *phy;
 810        int ctrl, r;
 811
 812        edev = arg;
 813        ctlr = edev->ctlr;
 814        for (;;) {
 815                /* plan9 originally had a busy loop here (just called continue).
 816                 * though either you have the mii or you don't.  i don't think
 817                 * it'll magically show up later (it should have been
 818                 * initialized during pnp/pci, which
 819                 * is before attach, which is before lproc).  -brho */
 820                if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
 821                        printk("[kernel] igbelproc can't find a mii/curphy, "
 822                               "aborting!\n");
 823                        /* name alloc'd in attach */
 824                        kfree(per_cpu_info[core_id()].cur_kthread->name);
 825                        return;
 826                }
 827                /*
 828                 * To do:
 829                 *      logic to manage status change,
 830                 *      this is incomplete but should work
 831                 *      one time to set up the hardware.
 832                 *
 833                 *      MiiPhy.speed, etc. should be in Mii.
 834                 */
 835                if (miistatus(ctlr->mii) < 0)
 836                        // continue;    /* this comment out was plan9, not brho
 837                        // */
 838                        goto enable;
 839
 840                phy = ctlr->mii->curphy;
 841                ctrl = csr32r(ctlr, Ctrl);
 842
 843                switch (ctlr->id) {
 844                case i82543gc:
 845                case i82544ei:
 846                case i82544eif:
 847                default:
 848                        if (!(ctrl & Asde)) {
 849                                ctrl &= ~(SspeedMASK | Ilos | Fd);
 850                                ctrl |= Frcdplx | Frcspd;
 851                                if (phy->speed == 1000)
 852                                        ctrl |= Sspeed1000;
 853                                else if (phy->speed == 100)
 854                                        ctrl |= Sspeed100;
 855                                if (phy->fd)
 856                                        ctrl |= Fd;
 857                        }
 858                        break;
 859
 860                case i82540em:
 861                case i82540eplp:
 862                case i82547gi:
 863                case i82541gi:
 864                case i82541gi2:
 865                case i82541pi:
 866                        break;
 867                }
 868
 869                /*
 870                 * Collision Distance.
 871                 */
 872                r = csr32r(ctlr, Tctl);
 873                r &= ~ColdMASK;
 874                if (phy->fd)
 875                        r |= 64 << ColdSHIFT;
 876                else
 877                        r |= 512 << ColdSHIFT;
 878                csr32w(ctlr, Tctl, r);
 879
 880                /*
 881                 * Flow control.
 882                 */
 883                if (phy->rfc)
 884                        ctrl |= Rfce;
 885                if (phy->tfc)
 886                        ctrl |= Tfce;
 887                csr32w(ctlr, Ctrl, ctrl);
 888
 889        enable:
 890                netif_carrier_on(edev);
 891                ctlr->lim = 0;
 892                igbeim(ctlr, Lsc);
 893
 894                ctlr->lsleep++;
 895                rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
 896        }
 897}
 898
 899static void igbetxinit(struct ctlr *ctlr)
 900{
 901        int i, r;
 902        struct block *bp;
 903
 904        csr32w(ctlr, Tctl, (0x0F << CtSHIFT) | Psp | (66 << ColdSHIFT));
 905        switch (ctlr->id) {
 906        default:
 907                r = 6;
 908                break;
 909        case i82543gc:
 910        case i82544ei:
 911        case i82544eif:
 912        case i82544gc:
 913        case i82540em:
 914        case i82540eplp:
 915        case i82541ei:
 916        case i82541gi:
 917        case i82541gi2:
 918        case i82541pi:
 919        case i82545em:
 920        case i82545gmc:
 921        case i82546gb:
 922        case i82546eb:
 923        case i82547ei:
 924        case i82547gi:
 925                r = 8;
 926                break;
 927        }
 928        csr32w(ctlr, Tipg, (6 << 20) | (8 << 10) | r);
 929        csr32w(ctlr, Ait, 0);
 930        csr32w(ctlr, Txdmac, 0);
 931        csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
 932        csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
 933        csr32w(ctlr, Tdlen, ctlr->ntd * sizeof(Td));
 934        ctlr->tdh = PREV_RING(0, ctlr->ntd);
 935        csr32w(ctlr, Tdh, 0);
 936        ctlr->tdt = 0;
 937        csr32w(ctlr, Tdt, 0);
 938
 939        for (i = 0; i < ctlr->ntd; i++) {
 940                if ((bp = ctlr->tb[i]) != NULL) {
 941                        ctlr->tb[i] = NULL;
 942                        freeb(bp);
 943                }
 944                memset(&ctlr->tdba[i], 0, sizeof(Td));
 945        }
 946        ctlr->tdfree = ctlr->ntd;
 947
 948        csr32w(ctlr, Tidv, 128);
 949        r = (4 << WthreshSHIFT) | (4 << HthreshSHIFT) | (8 << PthreshSHIFT);
 950
 951        switch (ctlr->id) {
 952        default:
 953                break;
 954        case i82540em:
 955        case i82540eplp:
 956        case i82547gi:
 957        case i82545em:
 958        case i82545gmc:
 959        case i82546gb:
 960        case i82546eb:
 961        case i82541gi:
 962        case i82541gi2:
 963        case i82541pi:
 964                r = csr32r(ctlr, Txdctl);
 965                r &= ~WthreshMASK;
 966                r |= Gran | (4 << WthreshSHIFT);
 967
 968                csr32w(ctlr, Tadv, 64);
 969                break;
 970        }
 971
 972        csr32w(ctlr, Txdctl, r);
 973
 974        r = csr32r(ctlr, Tctl);
 975        r |= Ten;
 976        csr32w(ctlr, Tctl, r);
 977}
 978
 979static void igbetransmit(struct ether *edev)
 980{
 981        Td *td;
 982        struct block *bp;
 983        struct ctlr *ctlr;
 984        int tdh, tdt;
 985
 986        ctlr = edev->ctlr;
 987
 988        ilock(&ctlr->tlock);
 989
 990        /*
 991         * Free any completed packets
 992         */
 993        tdh = ctlr->tdh;
 994        while (NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)) {
 995                if ((bp = ctlr->tb[tdh]) != NULL) {
 996                        ctlr->tb[tdh] = NULL;
 997                        freeb(bp);
 998                }
 999                memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1000                tdh = NEXT_RING(tdh, ctlr->ntd);
1001        }
1002        ctlr->tdh = tdh;
1003
1004        /*
1005         * Try to fill the ring back up.
1006         */
1007        tdt = ctlr->tdt;
1008        while (NEXT_RING(tdt, ctlr->ntd) != tdh) {
1009                if ((bp = qget(edev->oq)) == NULL)
1010                        break;
1011                td = &ctlr->tdba[tdt];
1012                td->addr[0] = paddr_low32(bp->rp);
1013                td->addr[1] = paddr_high32(bp->rp);
1014                td->control = ((BLEN(bp) & LenMASK) << LenSHIFT);
1015                td->control |= Dext | Ifcs | Teop | DtypeDD;
1016                ctlr->tb[tdt] = bp;
1017                tdt = NEXT_RING(tdt, ctlr->ntd);
1018                if (NEXT_RING(tdt, ctlr->ntd) == tdh) {
1019                        td->control |= Rs;
1020                        ctlr->txdw++;
1021                        ctlr->tdt = tdt;
1022                        csr32w(ctlr, Tdt, tdt);
1023                        igbeim(ctlr, Txdw);
1024                        break;
1025                }
1026                ctlr->tdt = tdt;
1027                csr32w(ctlr, Tdt, tdt);
1028        }
1029
1030        iunlock(&ctlr->tlock);
1031}
1032
1033static void igbereplenish(struct ctlr *ctlr)
1034{
1035        Rd *rd;
1036        int rdt;
1037        struct block *bp;
1038
1039        rdt = ctlr->rdt;
1040        while (NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh) {
1041                rd = &ctlr->rdba[rdt];
1042                if (ctlr->rb[rdt] == NULL) {
1043                        bp = block_alloc(Rbsz, MEM_ATOMIC);
1044                        if (bp == NULL) {
1045                                /* needs to be a safe print for interrupt level
1046                                 */
1047                                printk("#l%d: igbereplenish: no available "
1048                                       "buffers\n",
1049                                       ctlr->edev->ctlrno);
1050                                break;
1051                        }
1052                        ctlr->rb[rdt] = bp;
1053                        rd->addr[0] = paddr_low32(bp->rp);
1054                        rd->addr[1] = paddr_high32(bp->rp);
1055                }
1056                wmb(); /* ensure prev rd writes come before status = 0. */
1057                rd->status = 0;
1058                rdt = NEXT_RING(rdt, ctlr->nrd);
1059                ctlr->rdfree++;
1060        }
1061        ctlr->rdt = rdt;
1062        csr32w(ctlr, Rdt, rdt);
1063}
1064
1065static void igberxinit(struct ctlr *ctlr)
1066{
1067        int i;
1068        struct block *bp;
1069
1070        /* temporarily keep Mpe on */
1071        csr32w(ctlr, Rctl, Dpf | Bsize2048 | Bam | RdtmsHALF | Mpe);
1072        csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1073        csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1074        csr32w(ctlr, Rdlen, ctlr->nrd * sizeof(Rd));
1075        ctlr->rdh = 0;
1076        csr32w(ctlr, Rdh, 0);
1077        ctlr->rdt = 0;
1078        csr32w(ctlr, Rdt, 0);
1079        ctlr->rdtr = 0;
1080        csr32w(ctlr, Rdtr, Fpd | 0);
1081
1082        for (i = 0; i < ctlr->nrd; i++) {
1083                if ((bp = ctlr->rb[i]) != NULL) {
1084                        ctlr->rb[i] = NULL;
1085                        freeb(bp);
1086                }
1087        }
1088        igbereplenish(ctlr);
1089
1090        switch (ctlr->id) {
1091        case i82540em:
1092        case i82540eplp:
1093        case i82541gi:
1094        case i82541gi2:
1095        case i82541pi:
1096        case i82545em:
1097        case i82545gmc:
1098        case i82546gb:
1099        case i82546eb:
1100        case i82547gi:
1101                csr32w(ctlr, Radv, 64);
1102                break;
1103        }
1104        csr32w(ctlr, Rxdctl, (8 << WthreshSHIFT) | (8 << HthreshSHIFT) | 4);
1105
1106        /*
1107         * Enable checksum offload.
1108         */
1109        csr32w(ctlr, Rxcsum, Tuofl | Ipofl | (ETHERHDRSIZE << PcssSHIFT));
1110}
1111
1112static int igberim(void *ctlr)
1113{
1114        return ((struct ctlr *)ctlr)->rim != 0;
1115}
1116
1117static void igberproc(void *arg)
1118{
1119        Rd *rd;
1120        struct block *bp;
1121        struct ctlr *ctlr;
1122        int r, rdh;
1123        struct ether *edev;
1124
1125        edev = arg;
1126        ctlr = edev->ctlr;
1127
1128        igberxinit(ctlr);
1129        r = csr32r(ctlr, Rctl);
1130        r |= Ren;
1131        csr32w(ctlr, Rctl, r);
1132
1133        for (;;) {
1134                ctlr->rim = 0;
1135                igbeim(ctlr, Rxt0 | Rxo | Rxdmt0 | Rxseq);
1136                ctlr->rsleep++;
1137                rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1138
1139                rdh = ctlr->rdh;
1140                for (;;) {
1141                        rd = &ctlr->rdba[rdh];
1142
1143                        if (!(rd->status & Rdd))
1144                                break;
1145
1146                        /*
1147                         * Accept eop packets with no errors.
1148                         * With no errors and the Ixsm bit set,
1149                         * the descriptor status Tpcs and Ipcs bits give
1150                         * an indication of whether the checksums were
1151                         * calculated and valid.
1152                         */
1153                        if ((rd->status & Reop) && rd->errors == 0) {
1154                                bp = ctlr->rb[rdh];
1155                                ctlr->rb[rdh] = NULL;
1156                                bp->wp += rd->length;
1157                                bp->next = NULL;
1158                                if (!(rd->status & Ixsm)) {
1159                                        ctlr->ixsm++;
1160                                        if (rd->status & Ipcs) {
1161                                                /*
1162                                                 * IP checksum calculated
1163                                                 * (and valid as errors == 0).
1164                                                 */
1165                                                ctlr->ipcs++;
1166                                                bp->flag |= Bipck;
1167                                        }
1168                                        if (rd->status & Tcpcs) {
1169                                                /*
1170                                                 * TCP/UDP checksum calculated
1171                                                 * (and valid as errors == 0).
1172                                                 */
1173                                                ctlr->tcpcs++;
1174                                                bp->flag |= Btcpck | Budpck;
1175                                        }
1176                                        bp->flag |= Bpktck;
1177                                }
1178                                etheriq(edev, bp, 1);
1179                        } else if (ctlr->rb[rdh] != NULL) {
1180                                freeb(ctlr->rb[rdh]);
1181                                ctlr->rb[rdh] = NULL;
1182                        }
1183
1184                        memset(rd, 0, sizeof(Rd));
1185                        /* make sure the zeroing happens before free (i think)
1186                         */
1187                        wmb();
1188                        ctlr->rdfree--;
1189                        rdh = NEXT_RING(rdh, ctlr->nrd);
1190                }
1191                ctlr->rdh = rdh;
1192
1193                if (ctlr->rdfree < ctlr->nrd / 2 || (ctlr->rim & Rxdmt0))
1194                        igbereplenish(ctlr);
1195        }
1196}
1197
1198static void igbeattach(struct ether *edev)
1199{
1200        ERRSTACK(1);
1201        struct block *bp;
1202        struct ctlr *ctlr;
1203        char *name;
1204
1205        ctlr = edev->ctlr;
1206        ctlr->edev = edev; /* point back to Ether* */
1207        qlock(&ctlr->alock);
1208        if (ctlr->alloc != NULL) { /* already allocated? */
1209                qunlock(&ctlr->alock);
1210                return;
1211        }
1212
1213        ctlr->tb = NULL;
1214        ctlr->rb = NULL;
1215        ctlr->alloc = NULL;
1216        if (waserror()) {
1217                kfree(ctlr->tb);
1218                ctlr->tb = NULL;
1219                kfree(ctlr->rb);
1220                ctlr->rb = NULL;
1221                kfree(ctlr->alloc);
1222                ctlr->alloc = NULL;
1223                qunlock(&ctlr->alock);
1224                nexterror();
1225        }
1226
1227        ctlr->nrd = Nrd;
1228        ctlr->ntd = Ntd;
1229        ctlr->alloc =
1230            kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1231        if (ctlr->alloc == NULL) {
1232                printd("igbe: can't allocate ctlr->alloc\n");
1233                error(ENOMEM, ERROR_FIXME);
1234        }
1235        ctlr->rdba = (Rd *)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1236        ctlr->tdba = (Td *)(ctlr->rdba + ctlr->nrd);
1237
1238        ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1239        ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1240        if (ctlr->rb == NULL || ctlr->tb == NULL) {
1241                printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1242                error(ENOMEM, ERROR_FIXME);
1243        }
1244
1245        /* the ktasks should free these names, if they ever exit */
1246        name = kmalloc(KNAMELEN, MEM_WAIT);
1247        snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1248        ktask(name, igbelproc, edev);
1249
1250        name = kmalloc(KNAMELEN, MEM_WAIT);
1251        snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1252        ktask(name, igberproc, edev);
1253
1254        igbetxinit(ctlr);
1255
1256        qunlock(&ctlr->alock);
1257        poperror();
1258}
1259
1260static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1261{
1262        struct ctlr *ctlr;
1263        struct ether *edev;
1264        int icr, im, txdw;
1265
1266        edev = arg;
1267        ctlr = edev->ctlr;
1268
1269        ilock(&ctlr->imlock);
1270        csr32w(ctlr, Imc, ~0);
1271        im = ctlr->im;
1272        txdw = 0;
1273
1274        while ((icr = csr32r(ctlr, Icr) & ctlr->im) != 0) {
1275                if (icr & Lsc) {
1276                        im &= ~Lsc;
1277                        ctlr->lim = icr & Lsc;
1278                        rendez_wakeup(&ctlr->lrendez);
1279                        ctlr->lintr++;
1280                }
1281                if (icr & (Rxt0 | Rxo | Rxdmt0 | Rxseq)) {
1282                        im &= ~(Rxt0 | Rxo | Rxdmt0 | Rxseq);
1283                        ctlr->rim = icr & (Rxt0 | Rxo | Rxdmt0 | Rxseq);
1284                        rendez_wakeup(&ctlr->rrendez);
1285                        ctlr->rintr++;
1286                }
1287                if (icr & Txdw) {
1288                        im &= ~Txdw;
1289                        txdw++;
1290                        ctlr->tintr++;
1291                }
1292        }
1293
1294        ctlr->im = im;
1295        csr32w(ctlr, Ims, im);
1296        iunlock(&ctlr->imlock);
1297
1298        if (txdw)
1299                igbetransmit(edev);
1300}
1301
1302static int i82543mdior(struct ctlr *ctlr, int n)
1303{
1304        int ctrl, data, i, r;
1305
1306        /*
1307         * Read n bits from the Management Data I/O Interface.
1308         */
1309        ctrl = csr32r(ctlr, Ctrl);
1310        r = (ctrl & ~Mddo) | Mdco;
1311        data = 0;
1312        for (i = n - 1; i >= 0; i--) {
1313                if (csr32r(ctlr, Ctrl) & Mdd)
1314                        data |= (1 << i);
1315                csr32w(ctlr, Ctrl, Mdc | r);
1316                csr32w(ctlr, Ctrl, r);
1317        }
1318        csr32w(ctlr, Ctrl, ctrl);
1319
1320        return data;
1321}
1322
1323static int i82543mdiow(struct ctlr *ctlr, int bits, int n)
1324{
1325        int ctrl, i, r;
1326
1327        /*
1328         * Write n bits to the Management Data I/O Interface.
1329         */
1330        ctrl = csr32r(ctlr, Ctrl);
1331        r = Mdco | Mddo | ctrl;
1332        for (i = n - 1; i >= 0; i--) {
1333                if (bits & (1 << i))
1334                        r |= Mdd;
1335                else
1336                        r &= ~Mdd;
1337                csr32w(ctlr, Ctrl, Mdc | r);
1338                csr32w(ctlr, Ctrl, r);
1339        }
1340        csr32w(ctlr, Ctrl, ctrl);
1341
1342        return 0;
1343}
1344
1345static int i82543miimir(struct mii *mii, int pa, int ra)
1346{
1347        int data;
1348        struct ctlr *ctlr;
1349
1350        ctlr = mii->ctlr;
1351
1352        /*
1353         * MII Management Interface Read.
1354         *
1355         * Preamble;
1356         * ST+OP+PHYAD+REGAD;
1357         * TA + 16 data bits.
1358         */
1359        i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1360        i82543mdiow(ctlr, 0x1800 | (pa << 5) | ra, 14);
1361        data = i82543mdior(ctlr, 18);
1362
1363        if (data & 0x10000)
1364                return -1;
1365
1366        return data & 0xFFFF;
1367}
1368
1369static int i82543miimiw(struct mii *mii, int pa, int ra, int data)
1370{
1371        struct ctlr *ctlr;
1372
1373        ctlr = mii->ctlr;
1374
1375        /*
1376         * MII Management Interface Write.
1377         *
1378         * Preamble;
1379         * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1380         * Z.
1381         */
1382        i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1383        data &= 0xFFFF;
1384        data |= (0x05 << (5 + 5 + 2 + 16)) | (pa << (5 + 2 + 16)) |
1385                (ra << (2 + 16)) | (0x02 << 16);
1386        i82543mdiow(ctlr, data, 32);
1387
1388        return 0;
1389}
1390
1391static int igbemiimir(struct mii *mii, int pa, int ra)
1392{
1393        struct ctlr *ctlr;
1394        int mdic, timo;
1395
1396        ctlr = mii->ctlr;
1397
1398        csr32w(ctlr, Mdic, MDIrop | (pa << MDIpSHIFT) | (ra << MDIrSHIFT));
1399        mdic = 0;
1400        for (timo = 64; timo; timo--) {
1401                mdic = csr32r(ctlr, Mdic);
1402                if (mdic & (MDIe | MDIready))
1403                        break;
1404                udelay(1);
1405        }
1406
1407        if ((mdic & (MDIe | MDIready)) == MDIready)
1408                return mdic & 0xFFFF;
1409        return -1;
1410}
1411
1412static int igbemiimiw(struct mii *mii, int pa, int ra, int data)
1413{
1414        struct ctlr *ctlr;
1415        int mdic, timo;
1416
1417        ctlr = mii->ctlr;
1418
1419        data &= MDIdMASK;
1420        csr32w(ctlr, Mdic,
1421               MDIwop | (pa << MDIpSHIFT) | (ra << MDIrSHIFT) | data);
1422        mdic = 0;
1423        for (timo = 64; timo; timo--) {
1424                mdic = csr32r(ctlr, Mdic);
1425                if (mdic & (MDIe | MDIready))
1426                        break;
1427                udelay(1);
1428        }
1429        if ((mdic & (MDIe | MDIready)) == MDIready)
1430                return 0;
1431        return -1;
1432}
1433
1434static int i82543miirw(struct mii *mii, int write, int pa, int ra, int data)
1435{
1436        if (write)
1437                return i82543miimiw(mii, pa, ra, data);
1438
1439        return i82543miimir(mii, pa, ra);
1440}
1441
1442static int igbemiirw(struct mii *mii, int write, int pa, int ra, int data)
1443{
1444        if (write)
1445                return igbemiimiw(mii, pa, ra, data);
1446
1447        return igbemiimir(mii, pa, ra);
1448}
1449
1450static int igbemii(struct ctlr *ctlr)
1451{
1452        int ctrl, p, r;
1453        int (*rw)(struct mii *, int unused_int, int, int, int);
1454
1455        r = csr32r(ctlr, Status);
1456        if (r & Tbimode)
1457                return -1;
1458
1459        ctrl = csr32r(ctlr, Ctrl);
1460        ctrl |= Slu;
1461
1462        switch (ctlr->id) {
1463        case i82543gc:
1464                ctrl |= Frcdplx | Frcspd;
1465                csr32w(ctlr, Ctrl, ctrl);
1466
1467                /*
1468                 * The reset pin direction (Mdro) should already
1469                 * be set from the EEPROM load.
1470                 * If it's not set this configuration is unexpected
1471                 * so bail.
1472                 */
1473                r = csr32r(ctlr, Ctrlext);
1474                if (!(r & Mdro))
1475                        return -1;
1476                csr32w(ctlr, Ctrlext, r);
1477                udelay(20 * 1000);
1478                r = csr32r(ctlr, Ctrlext);
1479                r &= ~Mdr;
1480                csr32w(ctlr, Ctrlext, r);
1481                udelay(20 * 1000);
1482                r = csr32r(ctlr, Ctrlext);
1483                r |= Mdr;
1484                csr32w(ctlr, Ctrlext, r);
1485                udelay(20 * 1000);
1486
1487                rw = i82543miirw;
1488                break;
1489        case i82544ei:
1490        case i82544eif:
1491        case i82544gc:
1492        case i82540em:
1493        case i82540eplp:
1494        case i82547ei:
1495        case i82547gi:
1496        case i82541ei:
1497        case i82541gi:
1498        case i82541gi2:
1499        case i82541pi:
1500        case i82545em:
1501        case i82545gmc:
1502        case i82546gb:
1503        case i82546eb:
1504                ctrl &= ~(Frcdplx | Frcspd);
1505                csr32w(ctlr, Ctrl, ctrl);
1506                rw = igbemiirw;
1507                break;
1508        default:
1509                return -1;
1510        }
1511
1512        if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1513                return -1;
1514        // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1515
1516        /*
1517         * 8254X-specific PHY registers not in 802.3:
1518         *      0x10    PHY specific control
1519         *      0x14    extended PHY specific control
1520         * Set appropriate values then reset the PHY to have
1521         * changes noted.
1522         */
1523        switch (ctlr->id) {
1524        case i82547gi:
1525        case i82541gi:
1526        case i82541gi2:
1527        case i82541pi:
1528        case i82545em:
1529        case i82545gmc:
1530        case i82546gb:
1531        case i82546eb:
1532                break;
1533        default:
1534                r = miimir(ctlr->mii, 16);
1535                r |= 0x0800; /* assert CRS on Tx */
1536                r |= 0x0060; /* auto-crossover all speeds */
1537                r |= 0x0002; /* polarity reversal enabled */
1538                miimiw(ctlr->mii, 16, r);
1539
1540                r = miimir(ctlr->mii, 20);
1541                r |= 0x0070; /* +25MHz clock */
1542                r &= ~0x0F00;
1543                r |= 0x0100; /* 1x downshift */
1544                miimiw(ctlr->mii, 20, r);
1545
1546                miireset(ctlr->mii);
1547                p = 0;
1548                if (ctlr->txcw & TxcwPs)
1549                        p |= AnaP;
1550                if (ctlr->txcw & TxcwAs)
1551                        p |= AnaAP;
1552                miiane(ctlr->mii, ~0, p, ~0);
1553                break;
1554        }
1555        return 0;
1556}
1557
1558static int at93c46io(struct ctlr *ctlr, char *op, int data)
1559{
1560        char *lp, *p;
1561        int i, loop, eecd, r;
1562
1563        eecd = csr32r(ctlr, Eecd);
1564
1565        r = 0;
1566        loop = -1;
1567        lp = NULL;
1568        for (p = op; *p != '\0'; p++) {
1569                switch (*p) {
1570                default:
1571                        return -1;
1572                case ' ':
1573                        continue;
1574                case ':': /* start of loop */
1575                        loop = strtol(p + 1, &lp, 0) - 1;
1576                        lp--;
1577                        if (p == lp)
1578                                loop = 7;
1579                        p = lp;
1580                        continue;
1581                case ';': /* end of loop */
1582                        if (lp == NULL)
1583                                return -1;
1584                        loop--;
1585                        if (loop >= 0)
1586                                p = lp;
1587                        else
1588                                lp = NULL;
1589                        continue;
1590                case 'C': /* assert clock */
1591                        eecd |= Sk;
1592                        break;
1593                case 'c': /* deassert clock */
1594                        eecd &= ~Sk;
1595                        break;
1596                case 'D': /* next bit in 'data' byte */
1597                        if (loop < 0)
1598                                return -1;
1599                        if (data & (1 << loop))
1600                                eecd |= Di;
1601                        else
1602                                eecd &= ~Di;
1603                        break;
1604                case 'O': /* collect data output */
1605                        i = (csr32r(ctlr, Eecd) & Do) != 0;
1606                        if (loop >= 0)
1607                                r |= (i << loop);
1608                        else
1609                                r = i;
1610                        continue;
1611                case 'I': /* assert data input */
1612                        eecd |= Di;
1613                        break;
1614                case 'i': /* deassert data input */
1615                        eecd &= ~Di;
1616                        break;
1617                case 'S': /* enable chip select */
1618                        eecd |= Cs;
1619                        break;
1620                case 's': /* disable chip select */
1621                        eecd &= ~Cs;
1622                        break;
1623                }
1624                csr32w(ctlr, Eecd, eecd);
1625                udelay(50);
1626        }
1627        if (loop >= 0)
1628                return -1;
1629        return r;
1630}
1631
1632static int at93c46r(struct ctlr *ctlr)
1633{
1634        uint16_t sum;
1635        char rop[20];
1636        int addr, areq, bits, data, eecd, i;
1637
1638        eecd = csr32r(ctlr, Eecd);
1639        if (eecd & Spi) {
1640                printd("igbe: SPI EEPROM access not implemented\n");
1641                return 0;
1642        }
1643        if (eecd & (Eeszaddr | Eesz256))
1644                bits = 8;
1645        else
1646                bits = 6;
1647
1648        sum = 0;
1649
1650        switch (ctlr->id) {
1651        default:
1652                areq = 0;
1653                break;
1654        case i82540em:
1655        case i82540eplp:
1656        case i82541ei:
1657        case i82541gi:
1658        case i82541gi2:
1659        case i82541pi:
1660        case i82545em:
1661        case i82545gmc:
1662        case i82546gb:
1663        case i82546eb:
1664        case i82547ei:
1665        case i82547gi:
1666                areq = 1;
1667                csr32w(ctlr, Eecd, eecd | Areq);
1668                for (i = 0; i < 1000; i++) {
1669                        if ((eecd = csr32r(ctlr, Eecd)) & Agnt)
1670                                break;
1671                        udelay(5);
1672                }
1673                if (!(eecd & Agnt)) {
1674                        printd("igbe: not granted EEPROM access\n");
1675                        goto release;
1676                }
1677                break;
1678        }
1679        snprintf(rop, sizeof(rop), "S :%dDCc;", bits + 3);
1680
1681        for (addr = 0; addr < 0x40; addr++) {
1682                /*
1683                 * Read a word at address 'addr' from the Atmel AT93C46
1684                 * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1685                 * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1686                 * for protocol details.
1687                 */
1688                if (at93c46io(ctlr, rop, (0x06 << bits) | addr) != 0) {
1689                        printd("igbe: can't set EEPROM address 0x%2.2X\n",
1690                               addr);
1691                        goto release;
1692                }
1693                data = at93c46io(ctlr, ":16COc;", 0);
1694                at93c46io(ctlr, "sic", 0);
1695                ctlr->eeprom[addr] = data;
1696                sum += data;
1697        }
1698
1699release:
1700        if (areq)
1701                csr32w(ctlr, Eecd, eecd & ~Areq);
1702        return sum;
1703}
1704
1705static int igbedetach(struct ctlr *ctlr)
1706{
1707        int r, timeo;
1708
1709        /*
1710         * Perform a device reset to get the chip back to the
1711         * power-on state, followed by an EEPROM reset to read
1712         * the defaults for some internal registers.
1713         */
1714        csr32w(ctlr, Imc, ~0);
1715        csr32w(ctlr, Rctl, 0);
1716        csr32w(ctlr, Tctl, 0);
1717
1718        udelay(10 * 1000);
1719
1720        csr32w(ctlr, Ctrl, Devrst);
1721        udelay(1 * 1000);
1722        for (timeo = 0; timeo < 1000; timeo++) {
1723                if (!(csr32r(ctlr, Ctrl) & Devrst))
1724                        break;
1725                udelay(1 * 1000);
1726        }
1727        if (csr32r(ctlr, Ctrl) & Devrst)
1728                return -1;
1729        r = csr32r(ctlr, Ctrlext);
1730        csr32w(ctlr, Ctrlext, r | Eerst);
1731        udelay(1 * 1000);
1732
1733        for (timeo = 0; timeo < 1000; timeo++) {
1734                if (!(csr32r(ctlr, Ctrlext) & Eerst))
1735                        break;
1736                udelay(1 * 1000);
1737        }
1738        if (csr32r(ctlr, Ctrlext) & Eerst)
1739                return -1;
1740
1741        switch (ctlr->id) {
1742        default:
1743                break;
1744        case i82540em:
1745        case i82540eplp:
1746        case i82541gi:
1747        case i82541gi2:
1748        case i82541pi:
1749        case i82545em:
1750        case i82545gmc:
1751        case i82547gi:
1752        case i82546gb:
1753        case i82546eb:
1754                r = csr32r(ctlr, Manc);
1755                r &= ~Arpen;
1756                csr32w(ctlr, Manc, r);
1757                break;
1758        }
1759
1760        csr32w(ctlr, Imc, ~0);
1761        udelay(1 * 1000);
1762        for (timeo = 0; timeo < 1000; timeo++) {
1763                if (!csr32r(ctlr, Icr))
1764                        break;
1765                udelay(1 * 1000);
1766        }
1767        if (csr32r(ctlr, Icr))
1768                return -1;
1769
1770        return 0;
1771}
1772
1773static void igbeshutdown(struct ether *ether)
1774{
1775        igbedetach(ether->ctlr);
1776}
1777
1778static int igbereset(struct ctlr *ctlr)
1779{
1780        int ctrl, i, pause, r, swdpio, txcw;
1781
1782        if (igbedetach(ctlr))
1783                return -1;
1784
1785        /*
1786         * Read the EEPROM, validate the checksum
1787         * then get the device back to a power-on state.
1788         */
1789        if ((r = at93c46r(ctlr)) != 0xBABA) {
1790                printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1791                return -1;
1792        }
1793
1794        /*
1795         * Snarf and set up the receive addresses.
1796         * There are 16 addresses. The first should be the MAC address.
1797         * The others are cleared and not marked valid (MS bit of Rah).
1798         */
1799        if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1800            (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1801             pci_config_addr(0, 1, 0, 0)))
1802                ctlr->eeprom[Ea + 2] += 0x100; /* second interface */
1803        if (ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1804                ctlr->eeprom[Ea] = 0xD000;
1805        for (i = Ea; i < Eaddrlen / 2; i++) {
1806                ctlr->ra[2 * i] = ctlr->eeprom[i];
1807                ctlr->ra[2 * i + 1] = ctlr->eeprom[i] >> 8;
1808        }
1809        /* lan id seems to vary on 82543gc; don't use it */
1810        if (ctlr->id != i82543gc) {
1811                r = (csr32r(ctlr, Status) & Lanid) >> 2;
1812                ctlr->ra[5] += r; /* ea ctlr[1] = ea ctlr[0]+1 */
1813        }
1814
1815        r = (ctlr->ra[3] << 24) | (ctlr->ra[2] << 16) | (ctlr->ra[1] << 8) |
1816            ctlr->ra[0];
1817        csr32w(ctlr, Ral, r);
1818        r = 0x80000000 | (ctlr->ra[5] << 8) | ctlr->ra[4];
1819        csr32w(ctlr, Rah, r);
1820        for (i = 1; i < 16; i++) {
1821                csr32w(ctlr, Ral + i * 8, 0);
1822                csr32w(ctlr, Rah + i * 8, 0);
1823        }
1824
1825        /*
1826         * Clear the Multicast Table Array.
1827         * It's a 4096 bit vector accessed as 128 32-bit registers.
1828         */
1829        memset(ctlr->mta, 0, sizeof(ctlr->mta));
1830        for (i = 0; i < 128; i++)
1831                csr32w(ctlr, Mta + i * 4, 0);
1832
1833        /*
1834         * Just in case the Eerst didn't load the defaults
1835         * (doesn't appear to fully on the 82543GC), do it manually.
1836         */
1837        if (ctlr->id == i82543gc) {
1838                txcw = csr32r(ctlr, Txcw);
1839                txcw &= ~(TxcwAne | TxcwPauseMASK | TxcwFd);
1840                ctrl = csr32r(ctlr, Ctrl);
1841                ctrl &= ~(SwdpioloMASK | Frcspd | Ilos | Lrst | Fd);
1842
1843                if (ctlr->eeprom[Icw1] & 0x0400) {
1844                        ctrl |= Fd;
1845                        txcw |= TxcwFd;
1846                }
1847                if (ctlr->eeprom[Icw1] & 0x0200)
1848                        ctrl |= Lrst;
1849                if (ctlr->eeprom[Icw1] & 0x0010)
1850                        ctrl |= Ilos;
1851                if (ctlr->eeprom[Icw1] & 0x0800)
1852                        ctrl |= Frcspd;
1853                swdpio = (ctlr->eeprom[Icw1] & 0x01E0) >> 5;
1854                ctrl |= swdpio << SwdpioloSHIFT;
1855                csr32w(ctlr, Ctrl, ctrl);
1856
1857                ctrl = csr32r(ctlr, Ctrlext);
1858                ctrl &= ~(Ips | SwdpiohiMASK);
1859                swdpio = (ctlr->eeprom[Icw2] & 0x00F0) >> 4;
1860                if (ctlr->eeprom[Icw1] & 0x1000)
1861                        ctrl |= Ips;
1862                ctrl |= swdpio << SwdpiohiSHIFT;
1863                csr32w(ctlr, Ctrlext, ctrl);
1864
1865                if (ctlr->eeprom[Icw2] & 0x0800)
1866                        txcw |= TxcwAne;
1867                pause = (ctlr->eeprom[Icw2] & 0x3000) >> 12;
1868                txcw |= pause << TxcwPauseSHIFT;
1869                switch (pause) {
1870                default:
1871                        ctlr->fcrtl = 0x00002000;
1872                        ctlr->fcrth = 0x00004000;
1873                        txcw |= TxcwAs | TxcwPs;
1874                        break;
1875                case 0:
1876                        ctlr->fcrtl = 0x00002000;
1877                        ctlr->fcrth = 0x00004000;
1878                        break;
1879                case 2:
1880                        ctlr->fcrtl = 0;
1881                        ctlr->fcrth = 0;
1882                        txcw |= TxcwAs;
1883                        break;
1884                }
1885                ctlr->txcw = txcw;
1886                csr32w(ctlr, Txcw, txcw);
1887        }
1888
1889        /*
1890         * Flow control - values from the datasheet.
1891         */
1892        csr32w(ctlr, Fcal, 0x00C28001);
1893        csr32w(ctlr, Fcah, 0x00000100);
1894        csr32w(ctlr, Fct, 0x00008808);
1895        csr32w(ctlr, Fcttv, 0x00000100);
1896
1897        csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1898        csr32w(ctlr, Fcrth, ctlr->fcrth);
1899
1900        /* FYI, igbemii checks status right away too. */
1901        if (!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1902                printk("igbemii failed!  igbe failing to reset!\n");
1903                return -1;
1904        }
1905
1906        return 0;
1907}
1908
1909static void igbepci(void)
1910{
1911        int id;
1912        struct pci_device *pcidev;
1913        struct ctlr *ctlr;
1914        void *mem;
1915        uintptr_t mmio_paddr;
1916
1917        STAILQ_FOREACH (pcidev, &pci_devices, all_dev) {
1918                /* This checks that pcidev is a Network Controller for Ethernet
1919                 */
1920                if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1921                        continue;
1922                id = pcidev->dev_id << 16 | pcidev->ven_id;
1923                switch (id) {
1924                default:
1925                        continue;
1926                case i82543gc:
1927                case i82544ei:
1928                case i82544eif:
1929                case i82544gc:
1930                case i82547ei:
1931                case i82547gi:
1932                case i82540em:
1933                case i82540eplp:
1934                case i82541ei:
1935                case i82541gi:
1936                case i82541gi2:
1937                case i82541pi:
1938                case i82545em:
1939                case i82545gmc:
1940                case i82546gb:
1941                case i82546eb:
1942                        break;
1943                }
1944                printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
1945                       pcidev->ven_id, pcidev->dev_id, pcidev->bus, pcidev->dev,
1946                       pcidev->func);
1947
1948                mmio_paddr = pcidev->bar[0].mmio_base32
1949                                 ? pcidev->bar[0].mmio_base32
1950                                 : pcidev->bar[0].mmio_base64;
1951                mem = (void *)vmap_pmem_nocache(mmio_paddr,
1952                                                pcidev->bar[0].mmio_sz);
1953                if (mem == NULL) {
1954                        printd("igbe: can't map %p\n",
1955                               pcidev->bar[0].mmio_base32);
1956                        continue;
1957                }
1958                pci_set_cacheline_size(pcidev);
1959                ctlr = kzmalloc(sizeof(struct ctlr), 0);
1960                if (ctlr == NULL) {
1961                        vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
1962                        error(ENOMEM, ERROR_FIXME);
1963                }
1964                spinlock_init_irqsave(&ctlr->imlock);
1965                spinlock_init_irqsave(&ctlr->tlock);
1966                qlock_init(&ctlr->alock);
1967                qlock_init(&ctlr->slock);
1968                rendez_init(&ctlr->lrendez);
1969                rendez_init(&ctlr->rrendez);
1970                /* port seems to be unused, and only used for some comparison
1971                 * with edev.  plan9 just used the top of the raw bar,
1972                 * regardless of the type. */
1973                ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
1974                ctlr->pci = pcidev;
1975                ctlr->id = id;
1976                ctlr->cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
1977                ctlr->nic = mem;
1978
1979                if (igbereset(ctlr)) {
1980                        kfree(ctlr);
1981                        vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
1982                        continue;
1983                }
1984                pci_set_bus_master(pcidev);
1985
1986                if (igbectlrhead != NULL)
1987                        igbectlrtail->next = ctlr;
1988                else
1989                        igbectlrhead = ctlr;
1990                igbectlrtail = ctlr;
1991        }
1992}
1993
1994static int igbepnp(struct ether *edev)
1995{
1996        struct ctlr *ctlr;
1997
1998        run_once(igbepci());
1999
2000        /*
2001         * Any adapter matches if no edev->port is supplied,
2002         * otherwise the ports must match.
2003         */
2004        for (ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next) {
2005                if (ctlr->active)
2006                        continue;
2007                if (edev->port == 0 || edev->port == ctlr->port) {
2008                        ctlr->active = 1;
2009                        break;
2010                }
2011        }
2012        if (ctlr == NULL)
2013                return -1;
2014
2015        edev->ctlr = ctlr;
2016        strlcpy(edev->drv_name, "igbe", KNAMELEN);
2017        edev->port = ctlr->port;
2018        edev->irq = ctlr->pci->irqline;
2019        edev->tbdf = pci_to_tbdf(ctlr->pci);
2020        edev->mbps = 1000;
2021        memmove(edev->ea, ctlr->ra, Eaddrlen);
2022        /* Jim or whoever have this turned on already.  We might be capable of
2023         * other features. */
2024        edev->feat = NETF_RXCSUM;
2025
2026        /*
2027         * Linkage to the generic ethernet driver.
2028         */
2029        edev->attach = igbeattach;
2030        edev->transmit = igbetransmit;
2031        edev->ifstat = igbeifstat;
2032        edev->ctl = igbectl;
2033        edev->shutdown = igbeshutdown;
2034
2035        edev->arg = edev;
2036        edev->promiscuous = igbepromiscuous;
2037        edev->multicast = igbemulticast;
2038
2039        register_irq(edev->irq, igbeinterrupt, edev, edev->tbdf);
2040        return 0;
2041}
2042
2043static void __init etherigbelink()
2044{
2045        addethercard("i82543", igbepnp);
2046        addethercard("igbe", igbepnp);
2047}
2048init_func_3(etherigbelink);
2049