net: Remove block->checksum
[akaros.git] / kern / drivers / net / etherigbe.c
1 /* This file is part of the UCB release of Plan 9. It is subject to the license
2  * terms in the LICENSE file found in the top-level directory of this
3  * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
4  * part of the UCB release of Plan 9, including this file, may be copied,
5  * modified, propagated, or distributed except according to the terms contained
6  * in the LICENSE file. */
7
8 /*
9  * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
10  * as found on the Intel PRO/1000 series of adapters:
11  *      82543GC Intel PRO/1000 T
12  *      82544EI Intel PRO/1000 XT
13  *      82540EM Intel PRO/1000 MT
14  *      82541[GP]I
15  *      82547GI
16  *      82546GB
17  *      82546EB
18  * To Do:
19  *      finish autonegotiation code;
20  *      integrate fiber stuff back in (this ONLY handles
21  *      the CAT5 cards at the moment);
22  *      add checksum-offload;
23  *      add tuning control via ctl file;
24  *      this driver is little-endian specific.
25  *
26  * Modified by brho:
27  *      ported to Akaros
28  *      fixed mii bugs (allocation, startup, miirw, etc)
29  *      fixed CLS bug (continue -> break)
30  *      made sure igbepci only runs once, even if it fails */
31
32 #include <vfs.h>
33 #include <kfs.h>
34 #include <slab.h>
35 #include <kmalloc.h>
36 #include <kref.h>
37 #include <string.h>
38 #include <stdio.h>
39 #include <assert.h>
40 #include <error.h>
41 #include <cpio.h>
42 #include <pmap.h>
43 #include <smp.h>
44 #include <arch/pci.h>
45 #include <net/ip.h>
46 #include <ns.h>
47 #include "ethermii.h"
48
49 #define ilock(x) spin_lock_irqsave(x)
50 #define iunlock(x) spin_unlock_irqsave(x)
51
52 enum {
53         i82542          = (0x1000<<16)|0x8086,
54         i82543gc        = (0x1004<<16)|0x8086,
55         i82544ei        = (0x1008<<16)|0x8086,
56         i82544eif       = (0x1009<<16)|0x8086,
57         i82544gc        = (0x100d<<16)|0x8086,
58         i82540em        = (0x100E<<16)|0x8086,
59         i82540eplp      = (0x101E<<16)|0x8086,
60         i82545em        = (0x100F<<16)|0x8086,
61         i82545gmc       = (0x1026<<16)|0x8086,
62         i82547ei        = (0x1019<<16)|0x8086,
63         i82547gi        = (0x1075<<16)|0x8086,
64         i82541ei        = (0x1013<<16)|0x8086,
65         i82541gi        = (0x1076<<16)|0x8086,
66         i82541gi2       = (0x1077<<16)|0x8086,
67         i82541pi        = (0x107c<<16)|0x8086,
68         i82546gb        = (0x1079<<16)|0x8086,
69         i82546eb        = (0x1010<<16)|0x8086,
70 };
71
72 enum {
73         Ctrl            = 0x00000000,   /* Device Control */
74         Ctrldup         = 0x00000004,   /* Device Control Duplicate */
75         Status          = 0x00000008,   /* Device Status */
76         Eecd            = 0x00000010,   /* EEPROM/Flash Control/Data */
77         Ctrlext         = 0x00000018,   /* Extended Device Control */
78         Mdic            = 0x00000020,   /* MDI Control */
79         Fcal            = 0x00000028,   /* Flow Control Address Low */
80         Fcah            = 0x0000002C,   /* Flow Control Address High */
81         Fct             = 0x00000030,   /* Flow Control Type */
82         Icr             = 0x000000C0,   /* Interrupt Cause Read */
83         Ics             = 0x000000C8,   /* Interrupt Cause Set */
84         Ims             = 0x000000D0,   /* Interrupt Mask Set/Read */
85         Imc             = 0x000000D8,   /* Interrupt mask Clear */
86         Rctl            = 0x00000100,   /* Receive Control */
87         Fcttv           = 0x00000170,   /* Flow Control Transmit Timer Value */
88         Txcw            = 0x00000178,   /* Transmit Configuration Word */
89         Rxcw            = 0x00000180,   /* Receive Configuration Word */
90         /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
91         Tctl            = 0x00000400,   /* Transmit Control */
92         Tipg            = 0x00000410,   /* Transmit IPG */
93         Tbt             = 0x00000448,   /* Transmit Burst Timer */
94         Ait             = 0x00000458,   /* Adaptive IFS Throttle */
95         Fcrtl           = 0x00002160,   /* Flow Control RX Threshold Low */
96         Fcrth           = 0x00002168,   /* Flow Control Rx Threshold High */
97         Rdfh            = 0x00002410,   /* Receive data fifo head */
98         Rdft            = 0x00002418,   /* Receive data fifo tail */
99         Rdfhs           = 0x00002420,   /* Receive data fifo head saved */
100         Rdfts           = 0x00002428,   /* Receive data fifo tail saved */
101         Rdfpc           = 0x00002430,   /* Receive data fifo packet count */
102         Rdbal           = 0x00002800,   /* Rd Base Address Low */
103         Rdbah           = 0x00002804,   /* Rd Base Address High */
104         Rdlen           = 0x00002808,   /* Receive Descriptor Length */
105         Rdh             = 0x00002810,   /* Receive Descriptor Head */
106         Rdt             = 0x00002818,   /* Receive Descriptor Tail */
107         Rdtr            = 0x00002820,   /* Receive Descriptor Timer Ring */
108         Rxdctl          = 0x00002828,   /* Receive Descriptor Control */
109         Radv            = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
110         Txdmac          = 0x00003000,   /* Transfer DMA Control */
111         Ett             = 0x00003008,   /* Early Transmit Control */
112         Tdfh            = 0x00003410,   /* Transmit data fifo head */
113         Tdft            = 0x00003418,   /* Transmit data fifo tail */
114         Tdfhs           = 0x00003420,   /* Transmit data Fifo Head saved */
115         Tdfts           = 0x00003428,   /* Transmit data fifo tail saved */
116         Tdfpc           = 0x00003430,   /* Trasnmit data Fifo packet count */
117         Tdbal           = 0x00003800,   /* Td Base Address Low */
118         Tdbah           = 0x00003804,   /* Td Base Address High */
119         Tdlen           = 0x00003808,   /* Transmit Descriptor Length */
120         Tdh             = 0x00003810,   /* Transmit Descriptor Head */
121         Tdt             = 0x00003818,   /* Transmit Descriptor Tail */
122         Tidv            = 0x00003820,   /* Transmit Interrupt Delay Value */
123         Txdctl          = 0x00003828,   /* Transmit Descriptor Control */
124         Tadv            = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
125
126         Statistics      = 0x00004000,   /* Start of Statistics Area */
127         Gorcl           = 0x88/4,       /* Good Octets Received Count */
128         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
129         Torl            = 0xC0/4,       /* Total Octets Received */
130         Totl            = 0xC8/4,       /* Total Octets Transmitted */
131         Nstatistics     = 64,
132
133         Rxcsum          = 0x00005000,   /* Receive Checksum Control */
134         Mta             = 0x00005200,   /* Multicast Table Array */
135         Ral             = 0x00005400,   /* Receive Address Low */
136         Rah             = 0x00005404,   /* Receive Address High */
137         Manc            = 0x00005820,   /* Management Control */
138 };
139
140 enum {                                  /* Ctrl */
141         Bem             = 0x00000002,   /* Big Endian Mode */
142         Prior           = 0x00000004,   /* Priority on the PCI bus */
143         Lrst            = 0x00000008,   /* Link Reset */
144         Asde            = 0x00000020,   /* Auto-Speed Detection Enable */
145         Slu             = 0x00000040,   /* Set Link Up */
146         Ilos            = 0x00000080,   /* Invert Loss of Signal (LOS) */
147         SspeedMASK      = 0x00000300,   /* Speed Selection */
148         SspeedSHIFT     = 8,
149         Sspeed10        = 0x00000000,   /* 10Mb/s */
150         Sspeed100       = 0x00000100,   /* 100Mb/s */
151         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
152         Frcspd          = 0x00000800,   /* Force Speed */
153         Frcdplx         = 0x00001000,   /* Force Duplex */
154         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
155         SwdpinsloSHIFT  = 18,
156         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
157         SwdpioloSHIFT   = 22,
158         Devrst          = 0x04000000,   /* Device Reset */
159         Rfce            = 0x08000000,   /* Receive Flow Control Enable */
160         Tfce            = 0x10000000,   /* Transmit Flow Control Enable */
161         Vme             = 0x40000000,   /* VLAN Mode Enable */
162 };
163
164 /*
165  * can't find Tckok nor Rbcok in any Intel docs,
166  * but even 82543gc docs define Lanid.
167  */
168 enum {                                  /* Status */
169         Lu              = 0x00000002,   /* Link Up */
170         Lanid           = 0x0000000C,   /* mask for Lan ID. (function id) */
171 //      Tckok           = 0x00000004,   /* Transmit clock is running */
172 //      Rbcok           = 0x00000008,   /* Receive clock is running */
173         Txoff           = 0x00000010,   /* Transmission Paused */
174         Tbimode         = 0x00000020,   /* TBI Mode Indication */
175         LspeedMASK      = 0x000000C0,   /* Link Speed Setting */
176         LspeedSHIFT     = 6,
177         Lspeed10        = 0x00000000,   /* 10Mb/s */
178         Lspeed100       = 0x00000040,   /* 100Mb/s */
179         Lspeed1000      = 0x00000080,   /* 1000Mb/s */
180         Mtxckok         = 0x00000400,   /* MTX clock is running */
181         Pci66           = 0x00000800,   /* PCI Bus speed indication */
182         Bus64           = 0x00001000,   /* PCI Bus width indication */
183         Pcixmode        = 0x00002000,   /* PCI-X mode */
184         PcixspeedMASK   = 0x0000C000,   /* PCI-X bus speed */
185         PcixspeedSHIFT  = 14,
186         Pcix66          = 0x00000000,   /* 50-66MHz */
187         Pcix100         = 0x00004000,   /* 66-100MHz */
188         Pcix133         = 0x00008000,   /* 100-133MHz */
189 };
190
191 enum {                                  /* Ctrl and Status */
192         Fd              = 0x00000001,   /* Full-Duplex */
193         AsdvMASK        = 0x00000300,
194         AsdvSHIFT       = 8,
195         Asdv10          = 0x00000000,   /* 10Mb/s */
196         Asdv100         = 0x00000100,   /* 100Mb/s */
197         Asdv1000        = 0x00000200,   /* 1000Mb/s */
198 };
199
200 enum {                                  /* Eecd */
201         Sk              = 0x00000001,   /* Clock input to the EEPROM */
202         Cs              = 0x00000002,   /* Chip Select */
203         Di              = 0x00000004,   /* Data Input to the EEPROM */
204         Do              = 0x00000008,   /* Data Output from the EEPROM */
205         Areq            = 0x00000040,   /* EEPROM Access Request */
206         Agnt            = 0x00000080,   /* EEPROM Access Grant */
207         Eepresent       = 0x00000100,   /* EEPROM Present */
208         Eesz256         = 0x00000200,   /* EEPROM is 256 words not 64 */
209         Eeszaddr        = 0x00000400,   /* EEPROM size for 8254[17] */
210         Spi             = 0x00002000,   /* EEPROM is SPI not Microwire */
211 };
212
213 enum {                                  /* Ctrlext */
214         Gpien           = 0x0000000F,   /* General Purpose Interrupt Enables */
215         SwdpinshiMASK   = 0x000000F0,   /* Software Defined Pins - hi nibble */
216         SwdpinshiSHIFT  = 4,
217         SwdpiohiMASK    = 0x00000F00,   /* Software Defined Pins - I or O */
218         SwdpiohiSHIFT   = 8,
219         Asdchk          = 0x00001000,   /* ASD Check */
220         Eerst           = 0x00002000,   /* EEPROM Reset */
221         Ips             = 0x00004000,   /* Invert Power State */
222         Spdbyps         = 0x00008000,   /* Speed Select Bypass */
223 };
224
225 enum {                                  /* EEPROM content offsets */
226         Ea              = 0x00,         /* Ethernet Address */
227         Cf              = 0x03,         /* Compatibility Field */
228         Pba             = 0x08,         /* Printed Board Assembly number */
229         Icw1            = 0x0A,         /* Initialization Control Word 1 */
230         Sid             = 0x0B,         /* Subsystem ID */
231         Svid            = 0x0C,         /* Subsystem Vendor ID */
232         Did             = 0x0D,         /* Device ID */
233         Vid             = 0x0E,         /* Vendor ID */
234         Icw2            = 0x0F,         /* Initialization Control Word 2 */
235 };
236
237 enum {                                  /* Mdic */
238         MDIdMASK        = 0x0000FFFF,   /* Data */
239         MDIdSHIFT       = 0,
240         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
241         MDIrSHIFT       = 16,
242         MDIpMASK        = 0x03E00000,   /* PHY Address */
243         MDIpSHIFT       = 21,
244         MDIwop          = 0x04000000,   /* Write Operation */
245         MDIrop          = 0x08000000,   /* Read Operation */
246         MDIready        = 0x10000000,   /* End of Transaction */
247         MDIie           = 0x20000000,   /* Interrupt Enable */
248         MDIe            = 0x40000000,   /* Error */
249 };
250
251 enum {                                  /* Icr, Ics, Ims, Imc */
252         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
253         Txqe            = 0x00000002,   /* Transmit Queue Empty */
254         Lsc             = 0x00000004,   /* Link Status Change */
255         Rxseq           = 0x00000008,   /* Receive Sequence Error */
256         Rxdmt0          = 0x00000010,   /* Rd Minimum Threshold Reached */
257         Rxo             = 0x00000040,   /* Receiver Overrun */
258         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
259         Mdac            = 0x00000200,   /* MDIO Access Completed */
260         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
261         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
262         Gpi1            = 0x00001000,
263         Gpi2            = 0x00002000,
264         Gpi3            = 0x00004000,
265 };
266
267 /*
268  * The Mdic register isn't implemented on the 82543GC,
269  * the software defined pins are used instead.
270  * These definitions work for the Intel PRO/1000 T Server Adapter.
271  * The direction pin bits are read from the EEPROM.
272  */
273 enum {
274         Mdd             = ((1<<2)<<SwdpinsloSHIFT),     /* data */
275         Mddo            = ((1<<2)<<SwdpioloSHIFT),      /* pin direction */
276         Mdc             = ((1<<3)<<SwdpinsloSHIFT),     /* clock */
277         Mdco            = ((1<<3)<<SwdpioloSHIFT),      /* pin direction */
278         Mdr             = ((1<<0)<<SwdpinshiSHIFT),     /* reset */
279         Mdro            = ((1<<0)<<SwdpiohiSHIFT),      /* pin direction */
280 };
281
282 enum {                                  /* Txcw */
283         TxcwFd          = 0x00000020,   /* Full Duplex */
284         TxcwHd          = 0x00000040,   /* Half Duplex */
285         TxcwPauseMASK   = 0x00000180,   /* Pause */
286         TxcwPauseSHIFT  = 7,
287         TxcwPs          = (1<<TxcwPauseSHIFT),  /* Pause Supported */
288         TxcwAs          = (2<<TxcwPauseSHIFT),  /* Asymmetric FC desired */
289         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
290         TxcwRfiSHIFT    = 12,
291         TxcwNpr         = 0x00008000,   /* Next Page Request */
292         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
293         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
294 };
295
296 enum {                                  /* Rxcw */
297         Rxword          = 0x0000FFFF,   /* Data from auto-negotiation process */
298         Rxnocarrier     = 0x04000000,   /* Carrier Sense indication */
299         Rxinvalid       = 0x08000000,   /* Invalid Symbol during configuration */
300         Rxchange        = 0x10000000,   /* Change to the Rxword indication */
301         Rxconfig        = 0x20000000,   /* /C/ order set reception indication */
302         Rxsync          = 0x40000000,   /* Lost bit synchronization indication */
303         Anc             = 0x80000000,   /* Auto Negotiation Complete */
304 };
305
306 enum {                                  /* Rctl */
307         Rrst            = 0x00000001,   /* Receiver Software Reset */
308         Ren             = 0x00000002,   /* Receiver Enable */
309         Sbp             = 0x00000004,   /* Store Bad Packets */
310         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
311         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
312         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
313         LbmMASK         = 0x000000C0,   /* Loopback Mode */
314         LbmOFF          = 0x00000000,   /* No Loopback */
315         LbmTBI          = 0x00000040,   /* TBI Loopback */
316         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
317         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
318         RdtmsMASK       = 0x00000300,   /* Rd Minimum Threshold Size */
319         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
320         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
321         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
322         MoMASK          = 0x00003000,   /* Multicast Offset */
323         Mo47b36         = 0x00000000,   /* bits [47:36] of received address */
324         Mo46b35         = 0x00001000,   /* bits [46:35] of received address */
325         Mo45b34         = 0x00002000,   /* bits [45:34] of received address */
326         Mo43b32         = 0x00003000,   /* bits [43:32] of received address */
327         Bam             = 0x00008000,   /* Broadcast Accept Mode */
328         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
329         Bsize2048       = 0x00000000,   /* Bsex = 0 */
330         Bsize1024       = 0x00010000,   /* Bsex = 0 */
331         Bsize512        = 0x00020000,   /* Bsex = 0 */
332         Bsize256        = 0x00030000,   /* Bsex = 0 */
333         Bsize16384      = 0x00010000,   /* Bsex = 1 */
334         Vfe             = 0x00040000,   /* VLAN Filter Enable */
335         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
336         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
337         Dpf             = 0x00400000,   /* Discard Pause Frames */
338         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
339         Bsex            = 0x02000000,   /* Buffer Size Extension */
340         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
341 };
342
343 enum {                                  /* Tctl */
344         Trst            = 0x00000001,   /* Transmitter Software Reset */
345         Ten             = 0x00000002,   /* Transmit Enable */
346         Psp             = 0x00000008,   /* Pad Short Packets */
347         CtMASK          = 0x00000FF0,   /* Collision Threshold */
348         CtSHIFT         = 4,
349         ColdMASK        = 0x003FF000,   /* Collision Distance */
350         ColdSHIFT       = 12,
351         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
352         Pbe             = 0x00800000,   /* Packet Burst Enable */
353         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
354         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
355 };
356
357 enum {                                  /* [RT]xdctl */
358         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
359         PthreshSHIFT    = 0,
360         HthreshMASK     = 0x00003F00,   /* Host Threshold */
361         HthreshSHIFT    = 8,
362         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
363         WthreshSHIFT    = 16,
364         Gran            = 0x01000000,   /* Granularity */
365         LthreshMASK     = 0xFE000000,   /* Low Threshold */
366         LthreshSHIFT    = 25,
367 };
368
369 enum {                                  /* Rxcsum */
370         PcssMASK        = 0x000000FF,   /* Packet Checksum Start */
371         PcssSHIFT       = 0,
372         Ipofl           = 0x00000100,   /* IP Checksum Off-load Enable */
373         Tuofl           = 0x00000200,   /* TCP/UDP Checksum Off-load Enable */
374 };
375
376 enum {                                  /* Manc */
377         Arpen           = 0x00002000,   /* Enable ARP Request Filtering */
378 };
379
380 enum {                                  /* Receive Delay Timer Ring */
381         DelayMASK       = 0x0000FFFF,   /* delay timer in 1.024nS increments */
382         DelaySHIFT      = 0,
383         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
384 };
385
386 typedef struct Rd {                     /* Receive Descriptor */
387         unsigned int    addr[2];
388         uint16_t        length;
389         uint16_t        checksum;
390         uint8_t status;
391         uint8_t errors;
392         uint16_t        special;
393 } Rd;
394
395 enum {                                  /* Rd status */
396         Rdd             = 0x01,         /* Descriptor Done */
397         Reop            = 0x02,         /* End of Packet */
398         Ixsm            = 0x04,         /* Ignore Checksum Indication */
399         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
400         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
401         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
402         Pif             = 0x80,         /* Passed in-exact filter */
403 };
404
405 enum {                                  /* Rd errors */
406         Ce              = 0x01,         /* CRC Error or Alignment Error */
407         Se              = 0x02,         /* Symbol Error */
408         Seq             = 0x04,         /* Sequence Error */
409         Cxe             = 0x10,         /* Carrier Extension Error */
410         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
411         Ipe             = 0x40,         /* IP Checksum Error */
412         Rxe             = 0x80,         /* RX Data Error */
413 };
414
415 typedef struct Td Td;
416 struct Td {                             /* Transmit Descriptor */
417         union {
418                 unsigned int    addr[2];        /* Data */
419                 struct {                /* Context */
420                         uint8_t ipcss;
421                         uint8_t ipcso;
422                         uint16_t        ipcse;
423                         uint8_t tucss;
424                         uint8_t tucso;
425                         uint16_t        tucse;
426                 };
427         };
428         unsigned int    control;
429         unsigned int    status;
430 };
431
432 enum {                                  /* Td control */
433         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
434         LenSHIFT        = 0,
435         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
436         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
437         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
438         Teop            = 0x01000000,   /* End of Packet (DD) */
439         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
440         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
441         Tse             = 0x04000000,   /* TCP Segmentation Enable */
442         Rs              = 0x08000000,   /* Report Status */
443         Rps             = 0x10000000,   /* Report Status Sent */
444         Dext            = 0x20000000,   /* Descriptor Extension */
445         Vle             = 0x40000000,   /* VLAN Packet Enable */
446         Ide             = 0x80000000,   /* Interrupt Delay Enable */
447 };
448
449 enum {                                  /* Td status */
450         Tdd             = 0x00000001,   /* Descriptor Done */
451         Ec              = 0x00000002,   /* Excess Collisions */
452         Lc              = 0x00000004,   /* Late Collision */
453         Tu              = 0x00000008,   /* Transmit Underrun */
454         Iixsm           = 0x00000100,   /* Insert IP Checksum */
455         Itxsm           = 0x00000200,   /* Insert TCP/UDP Checksum */
456         HdrlenMASK      = 0x0000FF00,   /* Header Length (Tse) */
457         HdrlenSHIFT     = 8,
458         VlanMASK        = 0x0FFF0000,   /* VLAN Identifier */
459         VlanSHIFT       = 16,
460         Tcfi            = 0x10000000,   /* Canonical Form Indicator */
461         PriMASK         = 0xE0000000,   /* User Priority */
462         PriSHIFT        = 29,
463         MssMASK         = 0xFFFF0000,   /* Maximum Segment Size (Tse) */
464         MssSHIFT        = 16,
465 };
466
467 enum {
468         Nrd             = 256,          /* multiple of 8 */
469         Ntd             = 64,           /* multiple of 8 */
470         Rbsz            = 2048,
471 };
472
473 struct ctlr {
474         int     port;
475         struct pci_device *pci;
476         struct ctlr*    next;
477         struct ether*   edev;
478         int     active;
479         int     started;
480         int     id;
481         int     cls;
482         uint16_t        eeprom[0x40];
483
484         qlock_t alock;                  /* attach */
485         void*   alloc;                  /* receive/transmit descriptors */
486         int     nrd;
487         int     ntd;
488
489         int*    nic;
490         spinlock_t      imlock;
491         int     im;                     /* interrupt mask */
492
493         struct mii*     mii;
494         struct rendez   lrendez;
495         int     lim;
496
497         int     link;
498
499         qlock_t slock;
500         unsigned int    statistics[Nstatistics];
501         unsigned int    lsleep;
502         unsigned int    lintr;
503         unsigned int    rsleep;
504         unsigned int    rintr;
505         unsigned int    txdw;
506         unsigned int    tintr;
507         unsigned int    ixsm;
508         unsigned int    ipcs;
509         unsigned int    tcpcs;
510
511         uint8_t ra[Eaddrlen];           /* receive address */
512         uint32_t        mta[128];               /* multicast table array */
513
514         struct rendez   rrendez;
515         int     rim;
516         int     rdfree;
517         Rd*     rdba;                   /* receive descriptor base address */
518         struct block**  rb;                     /* receive buffers */
519         int     rdh;                    /* receive descriptor head */
520         int     rdt;                    /* receive descriptor tail */
521         int     rdtr;                   /* receive delay timer ring value */
522
523         spinlock_t      tlock;
524         int     tbusy;
525         int     tdfree;
526         Td*     tdba;                   /* transmit descriptor base address */
527         struct block**  tb;                     /* transmit buffers */
528         int     tdh;                    /* transmit descriptor head */
529         int     tdt;                    /* transmit descriptor tail */
530
531         int     txcw;
532         int     fcrtl;
533         int     fcrth;
534 };
535
536 static inline uint32_t csr32r(struct ctlr *c, uintptr_t reg)
537 {
538         return read_mmreg32((uintptr_t)(c->nic + (reg / 4)));
539 }
540
541 static inline void csr32w(struct ctlr *c, uintptr_t reg, uint32_t val)
542 {
543         write_mmreg32((uintptr_t)(c->nic + (reg / 4)), val);
544 }
545
546 static struct ctlr* igbectlrhead;
547 static struct ctlr* igbectlrtail;
548
549 static char* statistics[Nstatistics] = {
550         "CRC Error",
551         "Alignment Error",
552         "Symbol Error",
553         "RX Error",
554         "Missed Packets",
555         "Single Collision",
556         "Excessive Collisions",
557         "Multiple Collision",
558         "Late Collisions",
559         NULL,
560         "Collision",
561         "Transmit Underrun",
562         "Defer",
563         "Transmit - No CRS",
564         "Sequence Error",
565         "Carrier Extension Error",
566         "Receive Error Length",
567         NULL,
568         "XON Received",
569         "XON Transmitted",
570         "XOFF Received",
571         "XOFF Transmitted",
572         "FC Received Unsupported",
573         "Packets Received (64 Bytes)",
574         "Packets Received (65-127 Bytes)",
575         "Packets Received (128-255 Bytes)",
576         "Packets Received (256-511 Bytes)",
577         "Packets Received (512-1023 Bytes)",
578         "Packets Received (1024-1522 Bytes)",
579         "Good Packets Received",
580         "Broadcast Packets Received",
581         "Multicast Packets Received",
582         "Good Packets Transmitted",
583         NULL,
584         "Good Octets Received",
585         NULL,
586         "Good Octets Transmitted",
587         NULL,
588         NULL,
589         NULL,
590         "Receive No Buffers",
591         "Receive Undersize",
592         "Receive Fragment",
593         "Receive Oversize",
594         "Receive Jabber",
595         NULL,
596         NULL,
597         NULL,
598         "Total Octets Received",
599         NULL,
600         "Total Octets Transmitted",
601         NULL,
602         "Total Packets Received",
603         "Total Packets Transmitted",
604         "Packets Transmitted (64 Bytes)",
605         "Packets Transmitted (65-127 Bytes)",
606         "Packets Transmitted (128-255 Bytes)",
607         "Packets Transmitted (256-511 Bytes)",
608         "Packets Transmitted (512-1023 Bytes)",
609         "Packets Transmitted (1024-1522 Bytes)",
610         "Multicast Packets Transmitted",
611         "Broadcast Packets Transmitted",
612         "TCP Segmentation Context Transmitted",
613         "TCP Segmentation Context Fail",
614 };
615
616 static void igbe_print_rd(struct Rd *rd)
617 {
618         printk("Rd %p: stat 0x%02x, err 0x%02x, len 0x%04x, check 0x%04x, "
619                "spec 0x%04x, addr[1] 0x%08x, addr[0] 0x%08x\n", rd,
620                    rd->status,
621                    rd->errors,
622                    rd->length,
623                    rd->checksum,
624                    rd->special,
625                    rd->addr[1],
626                    rd->addr[0]);
627 }
628
629 static long
630 igbeifstat(struct ether* edev, void* a, long n, uint32_t offset)
631 {
632         struct ctlr *ctlr;
633         char *p, *s;
634         int i, l, r;
635         uint64_t tuvl, ruvl;
636
637         ctlr = edev->ctlr;
638         qlock(&ctlr->slock);
639         p = kzmalloc(READSTR, 0);
640         if(p == NULL) {
641                 qunlock(&ctlr->slock);
642                 error(ENOMEM, ERROR_FIXME);
643         }
644         l = 0;
645         for(i = 0; i < Nstatistics; i++){
646                 r = csr32r(ctlr, Statistics+i*4);
647                 if((s = statistics[i]) == NULL)
648                         continue;
649                 switch(i){
650                 case Gorcl:
651                 case Gotcl:
652                 case Torl:
653                 case Totl:
654                         ruvl = r;
655                         ruvl += ((uint64_t)csr32r(ctlr, Statistics+(i+1)*4))<<32;
656                         tuvl = ruvl;
657                         tuvl += ctlr->statistics[i];
658                         tuvl += ((uint64_t)ctlr->statistics[i+1])<<32;
659                         if(tuvl == 0)
660                                 continue;
661                         ctlr->statistics[i] = tuvl;
662                         ctlr->statistics[i+1] = tuvl>>32;
663                         l += snprintf(p+l, READSTR-l, "%s: %llud %llud\n",
664                                 s, tuvl, ruvl);
665                         i++;
666                         break;
667
668                 default:
669                         ctlr->statistics[i] += r;
670                         if(ctlr->statistics[i] == 0)
671                                 continue;
672                         l += snprintf(p+l, READSTR-l, "%s: %ud %ud\n",
673                                 s, ctlr->statistics[i], r);
674                         break;
675                 }
676         }
677
678         l += snprintf(p+l, READSTR-l, "lintr: %ud %ud\n",
679                 ctlr->lintr, ctlr->lsleep);
680         l += snprintf(p+l, READSTR-l, "rintr: %ud %ud\n",
681                 ctlr->rintr, ctlr->rsleep);
682         l += snprintf(p+l, READSTR-l, "tintr: %ud %ud\n",
683                 ctlr->tintr, ctlr->txdw);
684         l += snprintf(p+l, READSTR-l, "ixcs: %ud %ud %ud\n",
685                 ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
686         l += snprintf(p+l, READSTR-l, "rdtr: %ud\n", ctlr->rdtr);
687         l += snprintf(p+l, READSTR-l, "Ctrlext: %08x\n", csr32r(ctlr, Ctrlext));
688
689         l += snprintf(p+l, READSTR-l, "eeprom:");
690         for(i = 0; i < 0x40; i++){
691                 if(i && ((i & 0x07) == 0))
692                         l += snprintf(p+l, READSTR-l, "\n       ");
693                 l += snprintf(p+l, READSTR-l, " %4.4uX", ctlr->eeprom[i]);
694         }
695         l += snprintf(p+l, READSTR-l, "\n");
696
697         if(ctlr->mii != NULL && ctlr->mii->curphy != NULL){
698                 l += snprintf(p+l, READSTR-l, "phy:   ");
699                 for(i = 0; i < NMiiPhyr; i++){
700                         if(i && ((i & 0x07) == 0))
701                                 l += snprintf(p+l, READSTR-l, "\n       ");
702                         r = miimir(ctlr->mii, i);
703                         l += snprintf(p+l, READSTR-l, " %4.4uX", r);
704                 }
705                 snprintf(p+l, READSTR-l, "\n");
706         }
707         n = readstr(offset, a, n, p);
708         kfree(p);
709         qunlock(&ctlr->slock);
710
711         return n;
712 }
713
714 enum {
715         CMrdtr,
716 };
717
718 static struct cmdtab igbectlmsg[] = {
719         {CMrdtr,        "rdtr", 2},
720 };
721
722 static long
723 igbectl(struct ether* edev, void* buf, long n)
724 {
725         ERRSTACK(2);
726         int v;
727         char *p;
728         struct ctlr *ctlr;
729         struct cmdbuf *cb;
730         struct cmdtab *ct;
731
732         if((ctlr = edev->ctlr) == NULL)
733                 error(ENODEV, ERROR_FIXME);
734
735         cb = parsecmd(buf, n);
736         if(waserror()){
737                 kfree(cb);
738                 nexterror();
739         }
740
741         ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
742         switch(ct->index){
743         case CMrdtr:
744                 v = strtol(cb->f[1], &p, 0);
745                 if(v < 0 || p == cb->f[1] || v > 0xFFFF)
746                         error(EINVAL, ERROR_FIXME);
747                 ctlr->rdtr = v;
748                 csr32w(ctlr, Rdtr, Fpd|v);
749                 break;
750         }
751         kfree(cb);
752         poperror();
753
754         return n;
755 }
756
757 static void
758 igbepromiscuous(void* arg, int on)
759 {
760         int rctl;
761         struct ctlr *ctlr;
762         struct ether *edev;
763
764         edev = arg;
765         ctlr = edev->ctlr;
766
767         rctl = csr32r(ctlr, Rctl);
768         rctl &= ~MoMASK;
769         rctl |= Mo47b36;
770         if(on)
771                 rctl |= Upe|Mpe;
772         else
773                 rctl &= ~(Upe|Mpe);
774         csr32w(ctlr, Rctl, rctl|Mpe);   /* temporarily keep Mpe on */
775 }
776
777 static void
778 igbemulticast(void* arg, uint8_t* addr, int add)
779 {
780         int bit, x;
781         struct ctlr *ctlr;
782         struct ether *edev;
783
784         edev = arg;
785         ctlr = edev->ctlr;
786
787         x = addr[5]>>1;
788         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
789         /*
790          * multiple ether addresses can hash to the same filter bit,
791          * so it's never safe to clear a filter bit.
792          * if we want to clear filter bits, we need to keep track of
793          * all the multicast addresses in use, clear all the filter bits,
794          * then set the ones corresponding to in-use addresses.
795          */
796         if(add)
797                 ctlr->mta[x] |= 1<<bit;
798 //      else
799 //              ctlr->mta[x] &= ~(1<<bit);
800
801         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
802 }
803
804 static void
805 igbeim(struct ctlr* ctlr, int im)
806 {
807         ilock(&ctlr->imlock);
808         ctlr->im |= im;
809         csr32w(ctlr, Ims, ctlr->im);
810         iunlock(&ctlr->imlock);
811 }
812
813 static int
814 igbelim(void* ctlr)
815 {
816         return ((struct ctlr*)ctlr)->lim != 0;
817 }
818
819 static void
820 igbelproc(void* arg)
821 {
822         struct ctlr *ctlr;
823         struct ether *edev;
824         struct miiphy *phy;
825         int ctrl, r;
826
827         edev = arg;
828         ctlr = edev->ctlr;
829         for(;;){
830                 /* plan9 originally had a busy loop here (just called continue).  though
831                  * either you have the mii or you don't.  i don't think it'll magically
832                  * show up later (it should have been initialized during pnp/pci, which
833                  * is before attach, which is before lproc).  -brho */
834                 if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
835                         printk("[kernel] igbelproc can't find a mii/curphy, aborting!\n");
836                         /* name alloc'd in attach */
837                         kfree(per_cpu_info[core_id()].cur_kthread->name);
838                         return;
839                 }
840                 /*
841                  * To do:
842                  *      logic to manage status change,
843                  *      this is incomplete but should work
844                  *      one time to set up the hardware.
845                  *
846                  *      MiiPhy.speed, etc. should be in Mii.
847                  */
848                 if(miistatus(ctlr->mii) < 0)
849                         //continue;     /* this comment out was plan9, not brho */
850                         goto enable;
851
852                 phy = ctlr->mii->curphy;
853                 ctrl = csr32r(ctlr, Ctrl);
854
855                 switch(ctlr->id){
856                 case i82543gc:
857                 case i82544ei:
858                 case i82544eif:
859                 default:
860                         if(!(ctrl & Asde)){
861                                 ctrl &= ~(SspeedMASK|Ilos|Fd);
862                                 ctrl |= Frcdplx|Frcspd;
863                                 if(phy->speed == 1000)
864                                         ctrl |= Sspeed1000;
865                                 else if(phy->speed == 100)
866                                         ctrl |= Sspeed100;
867                                 if(phy->fd)
868                                         ctrl |= Fd;
869                         }
870                         break;
871
872                 case i82540em:
873                 case i82540eplp:
874                 case i82547gi:
875                 case i82541gi:
876                 case i82541gi2:
877                 case i82541pi:
878                         break;
879                 }
880
881                 /*
882                  * Collision Distance.
883                  */
884                 r = csr32r(ctlr, Tctl);
885                 r &= ~ColdMASK;
886                 if(phy->fd)
887                         r |= 64<<ColdSHIFT;
888                 else
889                         r |= 512<<ColdSHIFT;
890                 csr32w(ctlr, Tctl, r);
891
892                 /*
893                  * Flow control.
894                  */
895                 if(phy->rfc)
896                         ctrl |= Rfce;
897                 if(phy->tfc)
898                         ctrl |= Tfce;
899                 csr32w(ctlr, Ctrl, ctrl);
900
901 enable:
902                 ctlr->lim = 0;
903                 igbeim(ctlr, Lsc);
904
905                 ctlr->lsleep++;
906                 rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
907         }
908 }
909
910 static void
911 igbetxinit(struct ctlr* ctlr)
912 {
913         int i, r;
914         struct block *bp;
915
916         csr32w(ctlr, Tctl, (0x0F<<CtSHIFT)|Psp|(66<<ColdSHIFT));
917         switch(ctlr->id){
918         default:
919                 r = 6;
920                 break;
921         case i82543gc:
922         case i82544ei:
923         case i82544eif:
924         case i82544gc:
925         case i82540em:
926         case i82540eplp:
927         case i82541ei:
928         case i82541gi:
929         case i82541gi2:
930         case i82541pi:
931         case i82545em:
932         case i82545gmc:
933         case i82546gb:
934         case i82546eb:
935         case i82547ei:
936         case i82547gi:
937                 r = 8;
938                 break;
939         }
940         csr32w(ctlr, Tipg, (6<<20)|(8<<10)|r);
941         csr32w(ctlr, Ait, 0);
942         csr32w(ctlr, Txdmac, 0);
943         csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
944         csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
945         csr32w(ctlr, Tdlen, ctlr->ntd*sizeof(Td));
946         ctlr->tdh = PREV_RING(0, ctlr->ntd);
947         csr32w(ctlr, Tdh, 0);
948         ctlr->tdt = 0;
949         csr32w(ctlr, Tdt, 0);
950
951         for(i = 0; i < ctlr->ntd; i++){
952                 if((bp = ctlr->tb[i]) != NULL){
953                         ctlr->tb[i] = NULL;
954                         freeb(bp);
955                 }
956                 memset(&ctlr->tdba[i], 0, sizeof(Td));
957         }
958         ctlr->tdfree = ctlr->ntd;
959
960         csr32w(ctlr, Tidv, 128);
961         r = (4<<WthreshSHIFT)|(4<<HthreshSHIFT)|(8<<PthreshSHIFT);
962
963         switch(ctlr->id){
964         default:
965                 break;
966         case i82540em:
967         case i82540eplp:
968         case i82547gi:
969         case i82545em:
970         case i82545gmc:
971         case i82546gb:
972         case i82546eb:
973         case i82541gi:
974         case i82541gi2:
975         case i82541pi:
976                 r = csr32r(ctlr, Txdctl);
977                 r &= ~WthreshMASK;
978                 r |= Gran|(4<<WthreshSHIFT);
979
980                 csr32w(ctlr, Tadv, 64);
981                 break;
982         }
983
984         csr32w(ctlr, Txdctl, r);
985
986         r = csr32r(ctlr, Tctl);
987         r |= Ten;
988         csr32w(ctlr, Tctl, r);
989 }
990
991 static void
992 igbetransmit(struct ether* edev)
993 {
994         Td *td;
995         struct block *bp;
996         struct ctlr *ctlr;
997         int tdh, tdt;
998
999         ctlr = edev->ctlr;
1000
1001         ilock(&ctlr->tlock);
1002
1003         /*
1004          * Free any completed packets
1005          */
1006         tdh = ctlr->tdh;
1007         while(NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)){
1008                 if((bp = ctlr->tb[tdh]) != NULL){
1009                         ctlr->tb[tdh] = NULL;
1010                         freeb(bp);
1011                 }
1012                 memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1013                 tdh = NEXT_RING(tdh, ctlr->ntd);
1014         }
1015         ctlr->tdh = tdh;
1016
1017         /*
1018          * Try to fill the ring back up.
1019          */
1020         tdt = ctlr->tdt;
1021         while(NEXT_RING(tdt, ctlr->ntd) != tdh){
1022                 if((bp = qget(edev->oq)) == NULL)
1023                         break;
1024                 td = &ctlr->tdba[tdt];
1025                 td->addr[0] = paddr_low32(bp->rp);
1026                 td->addr[1] = paddr_high32(bp->rp);
1027                 td->control = ((BLEN(bp) & LenMASK)<<LenSHIFT);
1028                 td->control |= Dext|Ifcs|Teop|DtypeDD;
1029                 ctlr->tb[tdt] = bp;
1030                 tdt = NEXT_RING(tdt, ctlr->ntd);
1031                 if(NEXT_RING(tdt, ctlr->ntd) == tdh){
1032                         td->control |= Rs;
1033                         ctlr->txdw++;
1034                         ctlr->tdt = tdt;
1035                         csr32w(ctlr, Tdt, tdt);
1036                         igbeim(ctlr, Txdw);
1037                         break;
1038                 }
1039                 ctlr->tdt = tdt;
1040                 csr32w(ctlr, Tdt, tdt);
1041         }
1042
1043         iunlock(&ctlr->tlock);
1044 }
1045
1046 static void
1047 igbereplenish(struct ctlr* ctlr)
1048 {
1049         Rd *rd;
1050         int rdt;
1051         struct block *bp;
1052
1053         rdt = ctlr->rdt;
1054         while(NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh){
1055                 rd = &ctlr->rdba[rdt];
1056                 if(ctlr->rb[rdt] == NULL){
1057                         bp = block_alloc(Rbsz, MEM_ATOMIC);
1058                         if(bp == NULL){
1059                                 /* needs to be a safe print for interrupt level */
1060                                 printk("#l%d: igbereplenish: no available buffers\n",
1061                                         ctlr->edev->ctlrno);
1062                                 break;
1063                         }
1064                         ctlr->rb[rdt] = bp;
1065                         rd->addr[0] = paddr_low32(bp->rp);
1066                         rd->addr[1] = paddr_high32(bp->rp);
1067                 }
1068                 wmb();  /* ensure prev rd writes come before status = 0. */
1069                 rd->status = 0;
1070                 rdt = NEXT_RING(rdt, ctlr->nrd);
1071                 ctlr->rdfree++;
1072         }
1073         ctlr->rdt = rdt;
1074         csr32w(ctlr, Rdt, rdt);
1075 }
1076
1077 static void
1078 igberxinit(struct ctlr* ctlr)
1079 {
1080         int i;
1081         struct block *bp;
1082
1083         /* temporarily keep Mpe on */
1084         csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF|Mpe);
1085         csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1086         csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1087         csr32w(ctlr, Rdlen, ctlr->nrd*sizeof(Rd));
1088         ctlr->rdh = 0;
1089         csr32w(ctlr, Rdh, 0);
1090         ctlr->rdt = 0;
1091         csr32w(ctlr, Rdt, 0);
1092         ctlr->rdtr = 0;
1093         csr32w(ctlr, Rdtr, Fpd|0);
1094
1095         for(i = 0; i < ctlr->nrd; i++){
1096                 if((bp = ctlr->rb[i]) != NULL){
1097                         ctlr->rb[i] = NULL;
1098                         freeb(bp);
1099                 }
1100         }
1101         igbereplenish(ctlr);
1102
1103         switch(ctlr->id){
1104         case i82540em:
1105         case i82540eplp:
1106         case i82541gi:
1107         case i82541gi2:
1108         case i82541pi:
1109         case i82545em:
1110         case i82545gmc:
1111         case i82546gb:
1112         case i82546eb:
1113         case i82547gi:
1114                 csr32w(ctlr, Radv, 64);
1115                 break;
1116         }
1117         csr32w(ctlr, Rxdctl, (8<<WthreshSHIFT)|(8<<HthreshSHIFT)|4);
1118
1119         /*
1120          * Enable checksum offload.
1121          */
1122         csr32w(ctlr, Rxcsum, Tuofl|Ipofl|(ETHERHDRSIZE<<PcssSHIFT));
1123 }
1124
1125 static int
1126 igberim(void* ctlr)
1127 {
1128         return ((struct ctlr*)ctlr)->rim != 0;
1129 }
1130
1131 static void
1132 igberproc(void* arg)
1133 {
1134         Rd *rd;
1135         struct block *bp;
1136         struct ctlr *ctlr;
1137         int r, rdh;
1138         struct ether *edev;
1139
1140         edev = arg;
1141         ctlr = edev->ctlr;
1142
1143         igberxinit(ctlr);
1144         r = csr32r(ctlr, Rctl);
1145         r |= Ren;
1146         csr32w(ctlr, Rctl, r);
1147
1148         for(;;){
1149                 ctlr->rim = 0;
1150                 igbeim(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq);
1151                 ctlr->rsleep++;
1152                 rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1153
1154                 rdh = ctlr->rdh;
1155                 for(;;){
1156                         rd = &ctlr->rdba[rdh];
1157
1158                         if(!(rd->status & Rdd))
1159                                 break;
1160
1161                         /*
1162                          * Accept eop packets with no errors.
1163                          * With no errors and the Ixsm bit set,
1164                          * the descriptor status Tpcs and Ipcs bits give
1165                          * an indication of whether the checksums were
1166                          * calculated and valid.
1167                          */
1168                         if((rd->status & Reop) && rd->errors == 0){
1169                                 bp = ctlr->rb[rdh];
1170                                 ctlr->rb[rdh] = NULL;
1171                                 bp->wp += rd->length;
1172                                 bp->next = NULL;
1173                                 if(!(rd->status & Ixsm)){
1174                                         ctlr->ixsm++;
1175                                         if(rd->status & Ipcs){
1176                                                 /*
1177                                                  * IP checksum calculated
1178                                                  * (and valid as errors == 0).
1179                                                  */
1180                                                 ctlr->ipcs++;
1181                                                 bp->flag |= Bipck;
1182                                         }
1183                                         if(rd->status & Tcpcs){
1184                                                 /*
1185                                                  * TCP/UDP checksum calculated
1186                                                  * (and valid as errors == 0).
1187                                                  */
1188                                                 ctlr->tcpcs++;
1189                                                 bp->flag |= Btcpck|Budpck;
1190                                         }
1191                                         bp->flag |= Bpktck;
1192                                 }
1193                                 etheriq(edev, bp, 1);
1194                         }
1195                         else if(ctlr->rb[rdh] != NULL){
1196                                 freeb(ctlr->rb[rdh]);
1197                                 ctlr->rb[rdh] = NULL;
1198                         }
1199
1200                         memset(rd, 0, sizeof(Rd));
1201                         wmb();  /* make sure the zeroing happens before free (i think) */
1202                         ctlr->rdfree--;
1203                         rdh = NEXT_RING(rdh, ctlr->nrd);
1204                 }
1205                 ctlr->rdh = rdh;
1206
1207                 if(ctlr->rdfree < ctlr->nrd/2 || (ctlr->rim & Rxdmt0))
1208                         igbereplenish(ctlr);
1209         }
1210 }
1211
1212 static void
1213 igbeattach(struct ether* edev)
1214 {
1215         ERRSTACK(1);
1216         struct block *bp;
1217         struct ctlr *ctlr;
1218         char *name;
1219
1220         ctlr = edev->ctlr;
1221         ctlr->edev = edev;                      /* point back to Ether* */
1222         qlock(&ctlr->alock);
1223         if(ctlr->alloc != NULL){                        /* already allocated? */
1224                 qunlock(&ctlr->alock);
1225                 return;
1226         }
1227
1228         ctlr->tb = NULL;
1229         ctlr->rb = NULL;
1230         ctlr->alloc = NULL;
1231         if(waserror()){
1232                 kfree(ctlr->tb);
1233                 ctlr->tb = NULL;
1234                 kfree(ctlr->rb);
1235                 ctlr->rb = NULL;
1236                 kfree(ctlr->alloc);
1237                 ctlr->alloc = NULL;
1238                 qunlock(&ctlr->alock);
1239                 nexterror();
1240         }
1241
1242         ctlr->nrd = Nrd;
1243         ctlr->ntd = Ntd;
1244         ctlr->alloc = kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1245         if(ctlr->alloc == NULL) {
1246                 printd("igbe: can't allocate ctlr->alloc\n");
1247                 error(ENOMEM, ERROR_FIXME);
1248         }
1249         ctlr->rdba = (Rd*)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1250         ctlr->tdba = (Td*)(ctlr->rdba+ctlr->nrd);
1251
1252         ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1253         ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1254         if (ctlr->rb == NULL || ctlr->tb == NULL) {
1255                 printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1256                 error(ENOMEM, ERROR_FIXME);
1257         }
1258
1259         /* the ktasks should free these names, if they ever exit */
1260         name = kmalloc(KNAMELEN, MEM_WAIT);
1261         snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1262         ktask(name, igbelproc, edev);
1263
1264         name = kmalloc(KNAMELEN, MEM_WAIT);
1265         snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1266         ktask(name, igberproc, edev);
1267
1268         igbetxinit(ctlr);
1269
1270         qunlock(&ctlr->alock);
1271         poperror();
1272 }
1273
1274 static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1275 {
1276         struct ctlr *ctlr;
1277         struct ether *edev;
1278         int icr, im, txdw;
1279
1280         edev = arg;
1281         ctlr = edev->ctlr;
1282
1283         ilock(&ctlr->imlock);
1284         csr32w(ctlr, Imc, ~0);
1285         im = ctlr->im;
1286         txdw = 0;
1287
1288         while((icr = csr32r(ctlr, Icr) & ctlr->im) != 0){
1289                 if(icr & Lsc){
1290                         im &= ~Lsc;
1291                         ctlr->lim = icr & Lsc;
1292                         rendez_wakeup(&ctlr->lrendez);
1293                         ctlr->lintr++;
1294                 }
1295                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq)){
1296                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq);
1297                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq);
1298                         rendez_wakeup(&ctlr->rrendez);
1299                         ctlr->rintr++;
1300                 }
1301                 if(icr & Txdw){
1302                         im &= ~Txdw;
1303                         txdw++;
1304                         ctlr->tintr++;
1305                 }
1306         }
1307
1308         ctlr->im = im;
1309         csr32w(ctlr, Ims, im);
1310         iunlock(&ctlr->imlock);
1311
1312         if(txdw)
1313                 igbetransmit(edev);
1314 }
1315
1316 static int
1317 i82543mdior(struct ctlr* ctlr, int n)
1318 {
1319         int ctrl, data, i, r;
1320
1321         /*
1322          * Read n bits from the Management Data I/O Interface.
1323          */
1324         ctrl = csr32r(ctlr, Ctrl);
1325         r = (ctrl & ~Mddo)|Mdco;
1326         data = 0;
1327         for(i = n-1; i >= 0; i--){
1328                 if(csr32r(ctlr, Ctrl) & Mdd)
1329                         data |= (1<<i);
1330                 csr32w(ctlr, Ctrl, Mdc|r);
1331                 csr32w(ctlr, Ctrl, r);
1332         }
1333         csr32w(ctlr, Ctrl, ctrl);
1334
1335         return data;
1336 }
1337
1338 static int
1339 i82543mdiow(struct ctlr* ctlr, int bits, int n)
1340 {
1341         int ctrl, i, r;
1342
1343         /*
1344          * Write n bits to the Management Data I/O Interface.
1345          */
1346         ctrl = csr32r(ctlr, Ctrl);
1347         r = Mdco|Mddo|ctrl;
1348         for(i = n-1; i >= 0; i--){
1349                 if(bits & (1<<i))
1350                         r |= Mdd;
1351                 else
1352                         r &= ~Mdd;
1353                 csr32w(ctlr, Ctrl, Mdc|r);
1354                 csr32w(ctlr, Ctrl, r);
1355         }
1356         csr32w(ctlr, Ctrl, ctrl);
1357
1358         return 0;
1359 }
1360
1361 static int
1362 i82543miimir(struct mii* mii, int pa, int ra)
1363 {
1364         int data;
1365         struct ctlr *ctlr;
1366
1367         ctlr = mii->ctlr;
1368
1369         /*
1370          * MII Management Interface Read.
1371          *
1372          * Preamble;
1373          * ST+OP+PHYAD+REGAD;
1374          * TA + 16 data bits.
1375          */
1376         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1377         i82543mdiow(ctlr, 0x1800|(pa<<5)|ra, 14);
1378         data = i82543mdior(ctlr, 18);
1379
1380         if(data & 0x10000)
1381                 return -1;
1382
1383         return data & 0xFFFF;
1384 }
1385
1386 static int
1387 i82543miimiw(struct mii* mii, int pa, int ra, int data)
1388 {
1389         struct ctlr *ctlr;
1390
1391         ctlr = mii->ctlr;
1392
1393         /*
1394          * MII Management Interface Write.
1395          *
1396          * Preamble;
1397          * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1398          * Z.
1399          */
1400         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1401         data &= 0xFFFF;
1402         data |= (0x05<<(5+5+2+16))|(pa<<(5+2+16))|(ra<<(2+16))|(0x02<<16);
1403         i82543mdiow(ctlr, data, 32);
1404
1405         return 0;
1406 }
1407
1408 static int
1409 igbemiimir(struct mii* mii, int pa, int ra)
1410 {
1411         struct ctlr *ctlr;
1412         int mdic, timo;
1413
1414         ctlr = mii->ctlr;
1415
1416         csr32w(ctlr, Mdic, MDIrop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT));
1417         mdic = 0;
1418         for(timo = 64; timo; timo--){
1419                 mdic = csr32r(ctlr, Mdic);
1420                 if(mdic & (MDIe|MDIready))
1421                         break;
1422                 udelay(1);
1423         }
1424
1425         if((mdic & (MDIe|MDIready)) == MDIready)
1426                 return mdic & 0xFFFF;
1427         return -1;
1428 }
1429
1430 static int
1431 igbemiimiw(struct mii* mii, int pa, int ra, int data)
1432 {
1433         struct ctlr *ctlr;
1434         int mdic, timo;
1435
1436         ctlr = mii->ctlr;
1437
1438         data &= MDIdMASK;
1439         csr32w(ctlr, Mdic, MDIwop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT)|data);
1440         mdic = 0;
1441         for(timo = 64; timo; timo--){
1442                 mdic = csr32r(ctlr, Mdic);
1443                 if(mdic & (MDIe|MDIready))
1444                         break;
1445                 udelay(1);
1446         }
1447         if((mdic & (MDIe|MDIready)) == MDIready)
1448                 return 0;
1449         return -1;
1450 }
1451
1452 static int
1453 i82543miirw(struct mii* mii, int write, int pa, int ra, int data)
1454 {
1455         if(write)
1456                 return i82543miimiw(mii, pa, ra, data);
1457
1458         return i82543miimir(mii, pa, ra);
1459 }
1460
1461 static int
1462 igbemiirw(struct mii* mii, int write, int pa, int ra, int data)
1463 {
1464         if(write)
1465                 return igbemiimiw(mii, pa, ra, data);
1466
1467         return igbemiimir(mii, pa, ra);
1468 }
1469
1470 static int
1471 igbemii(struct ctlr* ctlr)
1472 {
1473         int ctrl, p, r;
1474         int (*rw)(struct mii*, int unused_int, int, int, int);
1475
1476         r = csr32r(ctlr, Status);
1477         if(r & Tbimode)
1478                 return -1;
1479
1480         ctrl = csr32r(ctlr, Ctrl);
1481         ctrl |= Slu;
1482
1483         switch(ctlr->id){
1484         case i82543gc:
1485                 ctrl |= Frcdplx|Frcspd;
1486                 csr32w(ctlr, Ctrl, ctrl);
1487
1488                 /*
1489                  * The reset pin direction (Mdro) should already
1490                  * be set from the EEPROM load.
1491                  * If it's not set this configuration is unexpected
1492                  * so bail.
1493                  */
1494                 r = csr32r(ctlr, Ctrlext);
1495                 if(!(r & Mdro))
1496                         return -1;
1497                 csr32w(ctlr, Ctrlext, r);
1498                 udelay(20*1000);
1499                 r = csr32r(ctlr, Ctrlext);
1500                 r &= ~Mdr;
1501                 csr32w(ctlr, Ctrlext, r);
1502                 udelay(20*1000);
1503                 r = csr32r(ctlr, Ctrlext);
1504                 r |= Mdr;
1505                 csr32w(ctlr, Ctrlext, r);
1506                 udelay(20*1000);
1507
1508                 rw = i82543miirw;
1509                 break;
1510         case i82544ei:
1511         case i82544eif:
1512         case i82544gc:
1513         case i82540em:
1514         case i82540eplp:
1515         case i82547ei:
1516         case i82547gi:
1517         case i82541ei:
1518         case i82541gi:
1519         case i82541gi2:
1520         case i82541pi:
1521         case i82545em:
1522         case i82545gmc:
1523         case i82546gb:
1524         case i82546eb:
1525                 ctrl &= ~(Frcdplx|Frcspd);
1526                 csr32w(ctlr, Ctrl, ctrl);
1527                 rw = igbemiirw;
1528                 break;
1529         default:
1530                 return -1;
1531         }
1532
1533         if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1534                 return -1;
1535         // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1536
1537         /*
1538          * 8254X-specific PHY registers not in 802.3:
1539          *      0x10    PHY specific control
1540          *      0x14    extended PHY specific control
1541          * Set appropriate values then reset the PHY to have
1542          * changes noted.
1543          */
1544         switch(ctlr->id){
1545         case i82547gi:
1546         case i82541gi:
1547         case i82541gi2:
1548         case i82541pi:
1549         case i82545em:
1550         case i82545gmc:
1551         case i82546gb:
1552         case i82546eb:
1553                 break;
1554         default:
1555                 r = miimir(ctlr->mii, 16);
1556                 r |= 0x0800;                    /* assert CRS on Tx */
1557                 r |= 0x0060;                    /* auto-crossover all speeds */
1558                 r |= 0x0002;                    /* polarity reversal enabled */
1559                 miimiw(ctlr->mii, 16, r);
1560
1561                 r = miimir(ctlr->mii, 20);
1562                 r |= 0x0070;                    /* +25MHz clock */
1563                 r &= ~0x0F00;
1564                 r |= 0x0100;                    /* 1x downshift */
1565                 miimiw(ctlr->mii, 20, r);
1566
1567                 miireset(ctlr->mii);
1568                 p = 0;
1569                 if(ctlr->txcw & TxcwPs)
1570                         p |= AnaP;
1571                 if(ctlr->txcw & TxcwAs)
1572                         p |= AnaAP;
1573                 miiane(ctlr->mii, ~0, p, ~0);
1574                 break;
1575         }
1576         return 0;
1577 }
1578
1579 static int
1580 at93c46io(struct ctlr* ctlr, char* op, int data)
1581 {
1582         char *lp, *p;
1583         int i, loop, eecd, r;
1584
1585         eecd = csr32r(ctlr, Eecd);
1586
1587         r = 0;
1588         loop = -1;
1589         lp = NULL;
1590         for(p = op; *p != '\0'; p++){
1591                 switch(*p){
1592                 default:
1593                         return -1;
1594                 case ' ':
1595                         continue;
1596                 case ':':                       /* start of loop */
1597                         loop = strtol(p+1, &lp, 0)-1;
1598                         lp--;
1599                         if(p == lp)
1600                                 loop = 7;
1601                         p = lp;
1602                         continue;
1603                 case ';':                       /* end of loop */
1604                         if(lp == NULL)
1605                                 return -1;
1606                         loop--;
1607                         if(loop >= 0)
1608                                 p = lp;
1609                         else
1610                                 lp = NULL;
1611                         continue;
1612                 case 'C':                       /* assert clock */
1613                         eecd |= Sk;
1614                         break;
1615                 case 'c':                       /* deassert clock */
1616                         eecd &= ~Sk;
1617                         break;
1618                 case 'D':                       /* next bit in 'data' byte */
1619                         if(loop < 0)
1620                                 return -1;
1621                         if(data & (1<<loop))
1622                                 eecd |= Di;
1623                         else
1624                                 eecd &= ~Di;
1625                         break;
1626                 case 'O':                       /* collect data output */
1627                         i = (csr32r(ctlr, Eecd) & Do) != 0;
1628                         if(loop >= 0)
1629                                 r |= (i<<loop);
1630                         else
1631                                 r = i;
1632                         continue;
1633                 case 'I':                       /* assert data input */
1634                         eecd |= Di;
1635                         break;
1636                 case 'i':                       /* deassert data input */
1637                         eecd &= ~Di;
1638                         break;
1639                 case 'S':                       /* enable chip select */
1640                         eecd |= Cs;
1641                         break;
1642                 case 's':                       /* disable chip select */
1643                         eecd &= ~Cs;
1644                         break;
1645                 }
1646                 csr32w(ctlr, Eecd, eecd);
1647                 udelay(50);
1648         }
1649         if(loop >= 0)
1650                 return -1;
1651         return r;
1652 }
1653
1654 static int
1655 at93c46r(struct ctlr* ctlr)
1656 {
1657         uint16_t sum;
1658         char rop[20];
1659         int addr, areq, bits, data, eecd, i;
1660
1661         eecd = csr32r(ctlr, Eecd);
1662         if(eecd & Spi){
1663                 printd("igbe: SPI EEPROM access not implemented\n");
1664                 return 0;
1665         }
1666         if(eecd & (Eeszaddr|Eesz256))
1667                 bits = 8;
1668         else
1669                 bits = 6;
1670
1671         sum = 0;
1672
1673         switch(ctlr->id){
1674         default:
1675                 areq = 0;
1676                 break;
1677         case i82540em:
1678         case i82540eplp:
1679         case i82541ei:
1680         case i82541gi:
1681         case i82541gi2:
1682         case i82541pi:
1683         case i82545em:
1684         case i82545gmc:
1685         case i82546gb:
1686         case i82546eb:
1687         case i82547ei:
1688         case i82547gi:
1689                 areq = 1;
1690                 csr32w(ctlr, Eecd, eecd|Areq);
1691                 for(i = 0; i < 1000; i++){
1692                         if((eecd = csr32r(ctlr, Eecd)) & Agnt)
1693                                 break;
1694                         udelay(5);
1695                 }
1696                 if(!(eecd & Agnt)){
1697                         printd("igbe: not granted EEPROM access\n");
1698                         goto release;
1699                 }
1700                 break;
1701         }
1702         snprintf(rop, sizeof(rop), "S :%dDCc;", bits+3);
1703
1704         for(addr = 0; addr < 0x40; addr++){
1705                 /*
1706                  * Read a word at address 'addr' from the Atmel AT93C46
1707                  * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1708                  * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1709                  * for protocol details.
1710                  */
1711                 if(at93c46io(ctlr, rop, (0x06<<bits)|addr) != 0){
1712                         printd("igbe: can't set EEPROM address 0x%2.2X\n", addr);
1713                         goto release;
1714                 }
1715                 data = at93c46io(ctlr, ":16COc;", 0);
1716                 at93c46io(ctlr, "sic", 0);
1717                 ctlr->eeprom[addr] = data;
1718                 sum += data;
1719         }
1720
1721 release:
1722         if(areq)
1723                 csr32w(ctlr, Eecd, eecd & ~Areq);
1724         return sum;
1725 }
1726
1727 static int
1728 igbedetach(struct ctlr* ctlr)
1729 {
1730         int r, timeo;
1731
1732         /*
1733          * Perform a device reset to get the chip back to the
1734          * power-on state, followed by an EEPROM reset to read
1735          * the defaults for some internal registers.
1736          */
1737         csr32w(ctlr, Imc, ~0);
1738         csr32w(ctlr, Rctl, 0);
1739         csr32w(ctlr, Tctl, 0);
1740
1741         udelay(10*1000);
1742
1743         csr32w(ctlr, Ctrl, Devrst);
1744         udelay(1*1000);
1745         for(timeo = 0; timeo < 1000; timeo++){
1746                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1747                         break;
1748                 udelay(1*1000);
1749         }
1750         if(csr32r(ctlr, Ctrl) & Devrst)
1751                 return -1;
1752         r = csr32r(ctlr, Ctrlext);
1753         csr32w(ctlr, Ctrlext, r|Eerst);
1754         udelay(1*1000);
1755
1756         for(timeo = 0; timeo < 1000; timeo++){
1757                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1758                         break;
1759                 udelay(1*1000);
1760         }
1761         if(csr32r(ctlr, Ctrlext) & Eerst)
1762                 return -1;
1763
1764         switch(ctlr->id){
1765         default:
1766                 break;
1767         case i82540em:
1768         case i82540eplp:
1769         case i82541gi:
1770         case i82541gi2:
1771         case i82541pi:
1772         case i82545em:
1773         case i82545gmc:
1774         case i82547gi:
1775         case i82546gb:
1776         case i82546eb:
1777                 r = csr32r(ctlr, Manc);
1778                 r &= ~Arpen;
1779                 csr32w(ctlr, Manc, r);
1780                 break;
1781         }
1782
1783         csr32w(ctlr, Imc, ~0);
1784         udelay(1*1000);
1785         for(timeo = 0; timeo < 1000; timeo++){
1786                 if(!csr32r(ctlr, Icr))
1787                         break;
1788                 udelay(1*1000);
1789         }
1790         if(csr32r(ctlr, Icr))
1791                 return -1;
1792
1793         return 0;
1794 }
1795
1796 static void
1797 igbeshutdown(struct ether* ether)
1798 {
1799         igbedetach(ether->ctlr);
1800 }
1801
1802 static int
1803 igbereset(struct ctlr* ctlr)
1804 {
1805         int ctrl, i, pause, r, swdpio, txcw;
1806
1807         if(igbedetach(ctlr))
1808                 return -1;
1809
1810         /*
1811          * Read the EEPROM, validate the checksum
1812          * then get the device back to a power-on state.
1813          */
1814         if((r = at93c46r(ctlr)) != 0xBABA){
1815                 printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1816                 return -1;
1817         }
1818
1819         /*
1820          * Snarf and set up the receive addresses.
1821          * There are 16 addresses. The first should be the MAC address.
1822          * The others are cleared and not marked valid (MS bit of Rah).
1823          */
1824         if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1825             (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1826                  pci_config_addr(0, 1, 0, 0)))
1827                 ctlr->eeprom[Ea+2] += 0x100;            /* second interface */
1828         if(ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1829                 ctlr->eeprom[Ea] = 0xD000;
1830         for(i = Ea; i < Eaddrlen/2; i++){
1831                 ctlr->ra[2*i] = ctlr->eeprom[i];
1832                 ctlr->ra[2*i+1] = ctlr->eeprom[i]>>8;
1833         }
1834         /* lan id seems to vary on 82543gc; don't use it */
1835         if (ctlr->id != i82543gc) {
1836                 r = (csr32r(ctlr, Status) & Lanid) >> 2;
1837                 ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1838         }
1839
1840         r = (ctlr->ra[3]<<24)|(ctlr->ra[2]<<16)|(ctlr->ra[1]<<8)|ctlr->ra[0];
1841         csr32w(ctlr, Ral, r);
1842         r = 0x80000000|(ctlr->ra[5]<<8)|ctlr->ra[4];
1843         csr32w(ctlr, Rah, r);
1844         for(i = 1; i < 16; i++){
1845                 csr32w(ctlr, Ral+i*8, 0);
1846                 csr32w(ctlr, Rah+i*8, 0);
1847         }
1848
1849         /*
1850          * Clear the Multicast Table Array.
1851          * It's a 4096 bit vector accessed as 128 32-bit registers.
1852          */
1853         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1854         for(i = 0; i < 128; i++)
1855                 csr32w(ctlr, Mta+i*4, 0);
1856
1857         /*
1858          * Just in case the Eerst didn't load the defaults
1859          * (doesn't appear to fully on the 82543GC), do it manually.
1860          */
1861         if (ctlr->id == i82543gc) {
1862                 txcw = csr32r(ctlr, Txcw);
1863                 txcw &= ~(TxcwAne|TxcwPauseMASK|TxcwFd);
1864                 ctrl = csr32r(ctlr, Ctrl);
1865                 ctrl &= ~(SwdpioloMASK|Frcspd|Ilos|Lrst|Fd);
1866
1867                 if(ctlr->eeprom[Icw1] & 0x0400){
1868                         ctrl |= Fd;
1869                         txcw |= TxcwFd;
1870                 }
1871                 if(ctlr->eeprom[Icw1] & 0x0200)
1872                         ctrl |= Lrst;
1873                 if(ctlr->eeprom[Icw1] & 0x0010)
1874                         ctrl |= Ilos;
1875                 if(ctlr->eeprom[Icw1] & 0x0800)
1876                         ctrl |= Frcspd;
1877                 swdpio = (ctlr->eeprom[Icw1] & 0x01E0)>>5;
1878                 ctrl |= swdpio<<SwdpioloSHIFT;
1879                 csr32w(ctlr, Ctrl, ctrl);
1880
1881                 ctrl = csr32r(ctlr, Ctrlext);
1882                 ctrl &= ~(Ips|SwdpiohiMASK);
1883                 swdpio = (ctlr->eeprom[Icw2] & 0x00F0)>>4;
1884                 if(ctlr->eeprom[Icw1] & 0x1000)
1885                         ctrl |= Ips;
1886                 ctrl |= swdpio<<SwdpiohiSHIFT;
1887                 csr32w(ctlr, Ctrlext, ctrl);
1888
1889                 if(ctlr->eeprom[Icw2] & 0x0800)
1890                         txcw |= TxcwAne;
1891                 pause = (ctlr->eeprom[Icw2] & 0x3000)>>12;
1892                 txcw |= pause<<TxcwPauseSHIFT;
1893                 switch(pause){
1894                 default:
1895                         ctlr->fcrtl = 0x00002000;
1896                         ctlr->fcrth = 0x00004000;
1897                         txcw |= TxcwAs|TxcwPs;
1898                         break;
1899                 case 0:
1900                         ctlr->fcrtl = 0x00002000;
1901                         ctlr->fcrth = 0x00004000;
1902                         break;
1903                 case 2:
1904                         ctlr->fcrtl = 0;
1905                         ctlr->fcrth = 0;
1906                         txcw |= TxcwAs;
1907                         break;
1908                 }
1909                 ctlr->txcw = txcw;
1910                 csr32w(ctlr, Txcw, txcw);
1911         }
1912
1913
1914         /*
1915          * Flow control - values from the datasheet.
1916          */
1917         csr32w(ctlr, Fcal, 0x00C28001);
1918         csr32w(ctlr, Fcah, 0x00000100);
1919         csr32w(ctlr, Fct, 0x00008808);
1920         csr32w(ctlr, Fcttv, 0x00000100);
1921
1922         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1923         csr32w(ctlr, Fcrth, ctlr->fcrth);
1924
1925         /* FYI, igbemii checks status right away too. */
1926         if(!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1927                 printk("igbemii failed!  igbe failing to reset!\n");
1928                 return -1;
1929         }
1930
1931         return 0;
1932 }
1933
1934 static void
1935 igbepci(void)
1936 {
1937         int id;
1938         struct pci_device *pcidev;
1939         struct ctlr *ctlr;
1940         void *mem;
1941         uintptr_t mmio_paddr;
1942
1943         STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
1944                 /* This checks that pcidev is a Network Controller for Ethernet */
1945                 if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1946                         continue;
1947                 id = pcidev->dev_id << 16 | pcidev->ven_id;
1948                 switch (id) {
1949                 default:
1950                         continue;
1951                 case i82543gc:
1952                 case i82544ei:
1953                 case i82544eif:
1954                 case i82544gc:
1955                 case i82547ei:
1956                 case i82547gi:
1957                 case i82540em:
1958                 case i82540eplp:
1959                 case i82541ei:
1960                 case i82541gi:
1961                 case i82541gi2:
1962                 case i82541pi:
1963                 case i82545em:
1964                 case i82545gmc:
1965                 case i82546gb:
1966                 case i82546eb:
1967                         break;
1968                 }
1969                 printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
1970                        pcidev->ven_id, pcidev->dev_id,
1971                        pcidev->bus, pcidev->dev, pcidev->func);
1972
1973                 mmio_paddr = pcidev->bar[0].mmio_base32 ? pcidev->bar[0].mmio_base32 :
1974                                                           pcidev->bar[0].mmio_base64;
1975                 mem = (void*)vmap_pmem_nocache(mmio_paddr, pcidev->bar[0].mmio_sz);
1976                 if(mem == NULL){
1977                         printd("igbe: can't map %p\n", pcidev->bar[0].mmio_base32);
1978                         continue;
1979                 }
1980                 pci_set_cacheline_size(pcidev);
1981                 ctlr = kzmalloc(sizeof(struct ctlr), 0);
1982                 if(ctlr == NULL) {
1983                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
1984                         error(ENOMEM, ERROR_FIXME);
1985                 }
1986                 spinlock_init_irqsave(&ctlr->imlock);
1987                 spinlock_init_irqsave(&ctlr->tlock);
1988                 qlock_init(&ctlr->alock);
1989                 qlock_init(&ctlr->slock);
1990                 rendez_init(&ctlr->lrendez);
1991                 rendez_init(&ctlr->rrendez);
1992                 /* port seems to be unused, and only used for some comparison with edev.
1993                  * plan9 just used the top of the raw bar, regardless of the type. */
1994                 ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
1995                 ctlr->pci = pcidev;
1996                 ctlr->id = id;
1997                 ctlr->cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
1998                 ctlr->nic = mem;
1999
2000                 if(igbereset(ctlr)){
2001                         kfree(ctlr);
2002                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2003                         continue;
2004                 }
2005                 pci_set_bus_master(pcidev);
2006
2007                 if(igbectlrhead != NULL)
2008                         igbectlrtail->next = ctlr;
2009                 else
2010                         igbectlrhead = ctlr;
2011                 igbectlrtail = ctlr;
2012         }
2013 }
2014
2015 static int
2016 igbepnp(struct ether* edev)
2017 {
2018         struct ctlr *ctlr;
2019
2020         run_once(igbepci());
2021
2022         /*
2023          * Any adapter matches if no edev->port is supplied,
2024          * otherwise the ports must match.
2025          */
2026         for(ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next){
2027                 if(ctlr->active)
2028                         continue;
2029                 if(edev->port == 0 || edev->port == ctlr->port){
2030                         ctlr->active = 1;
2031                         break;
2032                 }
2033         }
2034         if(ctlr == NULL)
2035                 return -1;
2036
2037         edev->ctlr = ctlr;
2038         strlcpy(edev->drv_name, "igbe", KNAMELEN);
2039         edev->port = ctlr->port;
2040         edev->irq = ctlr->pci->irqline;
2041         edev->tbdf = pci_to_tbdf(ctlr->pci);
2042         edev->mbps = 1000;
2043         memmove(edev->ea, ctlr->ra, Eaddrlen);
2044         /* Jim or whoever have this turned on already.  We might be capable of other
2045          * features. */
2046         edev->feat = NETF_RXCSUM;
2047
2048         /*
2049          * Linkage to the generic ethernet driver.
2050          */
2051         edev->attach = igbeattach;
2052         edev->transmit = igbetransmit;
2053         edev->ifstat = igbeifstat;
2054         edev->ctl = igbectl;
2055         edev->shutdown = igbeshutdown;
2056
2057         edev->arg = edev;
2058         edev->promiscuous = igbepromiscuous;
2059         edev->multicast = igbemulticast;
2060
2061         register_irq(edev->irq, igbeinterrupt, edev, edev->tbdf);
2062         return 0;
2063 }
2064
2065 linker_func_3(etherigbelink)
2066 {
2067         addethercard("i82543", igbepnp);
2068         addethercard("igbe", igbepnp);
2069 }