Adds the e1000, rtl8169, and ethermii drivers
[akaros.git] / kern / drivers / net / etherigbe.c
1 /* This file is part of the UCB release of Plan 9. It is subject to the license
2  * terms in the LICENSE file found in the top-level directory of this
3  * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
4  * part of the UCB release of Plan 9, including this file, may be copied,
5  * modified, propagated, or distributed except according to the terms contained
6  * in the LICENSE file. */
7
8 /*
9  * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
10  * as found on the Intel PRO/1000 series of adapters:
11  *      82543GC Intel PRO/1000 T
12  *      82544EI Intel PRO/1000 XT
13  *      82540EM Intel PRO/1000 MT
14  *      82541[GP]I
15  *      82547GI
16  *      82546GB
17  *      82546EB
18  * To Do:
19  *      finish autonegotiation code;
20  *      integrate fiber stuff back in (this ONLY handles
21  *      the CAT5 cards at the moment);
22  *      add checksum-offload;
23  *      add tuning control via ctl file;
24  *      this driver is little-endian specific.
25  *
26  * Modified by brho:
27  *      ported to Akaros
28  *      fixed mii bugs (allocation, startup, miirw, etc)
29  *      fixed CLS bug (continue -> break)
30  *      made sure igbepci only runs once, even if it fails */
31
32 #include <vfs.h>
33 #include <kfs.h>
34 #include <slab.h>
35 #include <kmalloc.h>
36 #include <kref.h>
37 #include <string.h>
38 #include <stdio.h>
39 #include <assert.h>
40 #include <error.h>
41 #include <cpio.h>
42 #include <pmap.h>
43 #include <smp.h>
44 #include <arch/pci.h>
45 #include <ip.h>
46 #include <ns.h>
47 #include "ethermii.h"
48
49 #define ilock(x) spin_lock_irqsave(x)
50 #define iunlock(x) spin_unlock_irqsave(x)
51
52 enum {
53         i82542          = (0x1000<<16)|0x8086,
54         i82543gc        = (0x1004<<16)|0x8086,
55         i82544ei        = (0x1008<<16)|0x8086,
56         i82544eif       = (0x1009<<16)|0x8086,
57         i82544gc        = (0x100d<<16)|0x8086,
58         i82540em        = (0x100E<<16)|0x8086,
59         i82540eplp      = (0x101E<<16)|0x8086,
60         i82545em        = (0x100F<<16)|0x8086,
61         i82545gmc       = (0x1026<<16)|0x8086,
62         i82547ei        = (0x1019<<16)|0x8086,
63         i82547gi        = (0x1075<<16)|0x8086,
64         i82541ei        = (0x1013<<16)|0x8086,
65         i82541gi        = (0x1076<<16)|0x8086,
66         i82541gi2       = (0x1077<<16)|0x8086,
67         i82541pi        = (0x107c<<16)|0x8086,
68         i82546gb        = (0x1079<<16)|0x8086,
69         i82546eb        = (0x1010<<16)|0x8086,
70 };
71
72 enum {
73         Ctrl            = 0x00000000,   /* Device Control */
74         Ctrldup         = 0x00000004,   /* Device Control Duplicate */
75         Status          = 0x00000008,   /* Device Status */
76         Eecd            = 0x00000010,   /* EEPROM/Flash Control/Data */
77         Ctrlext         = 0x00000018,   /* Extended Device Control */
78         Mdic            = 0x00000020,   /* MDI Control */
79         Fcal            = 0x00000028,   /* Flow Control Address Low */
80         Fcah            = 0x0000002C,   /* Flow Control Address High */
81         Fct             = 0x00000030,   /* Flow Control Type */
82         Icr             = 0x000000C0,   /* Interrupt Cause Read */
83         Ics             = 0x000000C8,   /* Interrupt Cause Set */
84         Ims             = 0x000000D0,   /* Interrupt Mask Set/Read */
85         Imc             = 0x000000D8,   /* Interrupt mask Clear */
86         Rctl            = 0x00000100,   /* Receive Control */
87         Fcttv           = 0x00000170,   /* Flow Control Transmit Timer Value */
88         Txcw            = 0x00000178,   /* Transmit Configuration Word */
89         Rxcw            = 0x00000180,   /* Receive Configuration Word */
90         /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
91         Tctl            = 0x00000400,   /* Transmit Control */
92         Tipg            = 0x00000410,   /* Transmit IPG */
93         Tbt             = 0x00000448,   /* Transmit Burst Timer */
94         Ait             = 0x00000458,   /* Adaptive IFS Throttle */
95         Fcrtl           = 0x00002160,   /* Flow Control RX Threshold Low */
96         Fcrth           = 0x00002168,   /* Flow Control Rx Threshold High */
97         Rdfh            = 0x00002410,   /* Receive data fifo head */
98         Rdft            = 0x00002418,   /* Receive data fifo tail */
99         Rdfhs           = 0x00002420,   /* Receive data fifo head saved */
100         Rdfts           = 0x00002428,   /* Receive data fifo tail saved */
101         Rdfpc           = 0x00002430,   /* Receive data fifo packet count */
102         Rdbal           = 0x00002800,   /* Rd Base Address Low */
103         Rdbah           = 0x00002804,   /* Rd Base Address High */
104         Rdlen           = 0x00002808,   /* Receive Descriptor Length */
105         Rdh             = 0x00002810,   /* Receive Descriptor Head */
106         Rdt             = 0x00002818,   /* Receive Descriptor Tail */
107         Rdtr            = 0x00002820,   /* Receive Descriptor Timer Ring */
108         Rxdctl          = 0x00002828,   /* Receive Descriptor Control */
109         Radv            = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
110         Txdmac          = 0x00003000,   /* Transfer DMA Control */
111         Ett             = 0x00003008,   /* Early Transmit Control */
112         Tdfh            = 0x00003410,   /* Transmit data fifo head */
113         Tdft            = 0x00003418,   /* Transmit data fifo tail */
114         Tdfhs           = 0x00003420,   /* Transmit data Fifo Head saved */
115         Tdfts           = 0x00003428,   /* Transmit data fifo tail saved */
116         Tdfpc           = 0x00003430,   /* Trasnmit data Fifo packet count */
117         Tdbal           = 0x00003800,   /* Td Base Address Low */
118         Tdbah           = 0x00003804,   /* Td Base Address High */
119         Tdlen           = 0x00003808,   /* Transmit Descriptor Length */
120         Tdh             = 0x00003810,   /* Transmit Descriptor Head */
121         Tdt             = 0x00003818,   /* Transmit Descriptor Tail */
122         Tidv            = 0x00003820,   /* Transmit Interrupt Delay Value */
123         Txdctl          = 0x00003828,   /* Transmit Descriptor Control */
124         Tadv            = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
125
126         Statistics      = 0x00004000,   /* Start of Statistics Area */
127         Gorcl           = 0x88/4,       /* Good Octets Received Count */
128         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
129         Torl            = 0xC0/4,       /* Total Octets Received */
130         Totl            = 0xC8/4,       /* Total Octets Transmitted */
131         Nstatistics     = 64,
132
133         Rxcsum          = 0x00005000,   /* Receive Checksum Control */
134         Mta             = 0x00005200,   /* Multicast Table Array */
135         Ral             = 0x00005400,   /* Receive Address Low */
136         Rah             = 0x00005404,   /* Receive Address High */
137         Manc            = 0x00005820,   /* Management Control */
138 };
139
140 enum {                                  /* Ctrl */
141         Bem             = 0x00000002,   /* Big Endian Mode */
142         Prior           = 0x00000004,   /* Priority on the PCI bus */
143         Lrst            = 0x00000008,   /* Link Reset */
144         Asde            = 0x00000020,   /* Auto-Speed Detection Enable */
145         Slu             = 0x00000040,   /* Set Link Up */
146         Ilos            = 0x00000080,   /* Invert Loss of Signal (LOS) */
147         SspeedMASK      = 0x00000300,   /* Speed Selection */
148         SspeedSHIFT     = 8,
149         Sspeed10        = 0x00000000,   /* 10Mb/s */
150         Sspeed100       = 0x00000100,   /* 100Mb/s */
151         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
152         Frcspd          = 0x00000800,   /* Force Speed */
153         Frcdplx         = 0x00001000,   /* Force Duplex */
154         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
155         SwdpinsloSHIFT  = 18,
156         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
157         SwdpioloSHIFT   = 22,
158         Devrst          = 0x04000000,   /* Device Reset */
159         Rfce            = 0x08000000,   /* Receive Flow Control Enable */
160         Tfce            = 0x10000000,   /* Transmit Flow Control Enable */
161         Vme             = 0x40000000,   /* VLAN Mode Enable */
162 };
163
164 /*
165  * can't find Tckok nor Rbcok in any Intel docs,
166  * but even 82543gc docs define Lanid.
167  */
168 enum {                                  /* Status */
169         Lu              = 0x00000002,   /* Link Up */
170         Lanid           = 0x0000000C,   /* mask for Lan ID. (function id) */
171 //      Tckok           = 0x00000004,   /* Transmit clock is running */
172 //      Rbcok           = 0x00000008,   /* Receive clock is running */
173         Txoff           = 0x00000010,   /* Transmission Paused */
174         Tbimode         = 0x00000020,   /* TBI Mode Indication */
175         LspeedMASK      = 0x000000C0,   /* Link Speed Setting */
176         LspeedSHIFT     = 6,
177         Lspeed10        = 0x00000000,   /* 10Mb/s */
178         Lspeed100       = 0x00000040,   /* 100Mb/s */
179         Lspeed1000      = 0x00000080,   /* 1000Mb/s */
180         Mtxckok         = 0x00000400,   /* MTX clock is running */
181         Pci66           = 0x00000800,   /* PCI Bus speed indication */
182         Bus64           = 0x00001000,   /* PCI Bus width indication */
183         Pcixmode        = 0x00002000,   /* PCI-X mode */
184         PcixspeedMASK   = 0x0000C000,   /* PCI-X bus speed */
185         PcixspeedSHIFT  = 14,
186         Pcix66          = 0x00000000,   /* 50-66MHz */
187         Pcix100         = 0x00004000,   /* 66-100MHz */
188         Pcix133         = 0x00008000,   /* 100-133MHz */
189 };
190
191 enum {                                  /* Ctrl and Status */
192         Fd              = 0x00000001,   /* Full-Duplex */
193         AsdvMASK        = 0x00000300,
194         AsdvSHIFT       = 8,
195         Asdv10          = 0x00000000,   /* 10Mb/s */
196         Asdv100         = 0x00000100,   /* 100Mb/s */
197         Asdv1000        = 0x00000200,   /* 1000Mb/s */
198 };
199
200 enum {                                  /* Eecd */
201         Sk              = 0x00000001,   /* Clock input to the EEPROM */
202         Cs              = 0x00000002,   /* Chip Select */
203         Di              = 0x00000004,   /* Data Input to the EEPROM */
204         Do              = 0x00000008,   /* Data Output from the EEPROM */
205         Areq            = 0x00000040,   /* EEPROM Access Request */
206         Agnt            = 0x00000080,   /* EEPROM Access Grant */
207         Eepresent       = 0x00000100,   /* EEPROM Present */
208         Eesz256         = 0x00000200,   /* EEPROM is 256 words not 64 */
209         Eeszaddr        = 0x00000400,   /* EEPROM size for 8254[17] */
210         Spi             = 0x00002000,   /* EEPROM is SPI not Microwire */
211 };
212
213 enum {                                  /* Ctrlext */
214         Gpien           = 0x0000000F,   /* General Purpose Interrupt Enables */
215         SwdpinshiMASK   = 0x000000F0,   /* Software Defined Pins - hi nibble */
216         SwdpinshiSHIFT  = 4,
217         SwdpiohiMASK    = 0x00000F00,   /* Software Defined Pins - I or O */
218         SwdpiohiSHIFT   = 8,
219         Asdchk          = 0x00001000,   /* ASD Check */
220         Eerst           = 0x00002000,   /* EEPROM Reset */
221         Ips             = 0x00004000,   /* Invert Power State */
222         Spdbyps         = 0x00008000,   /* Speed Select Bypass */
223 };
224
225 enum {                                  /* EEPROM content offsets */
226         Ea              = 0x00,         /* Ethernet Address */
227         Cf              = 0x03,         /* Compatibility Field */
228         Pba             = 0x08,         /* Printed Board Assembly number */
229         Icw1            = 0x0A,         /* Initialization Control Word 1 */
230         Sid             = 0x0B,         /* Subsystem ID */
231         Svid            = 0x0C,         /* Subsystem Vendor ID */
232         Did             = 0x0D,         /* Device ID */
233         Vid             = 0x0E,         /* Vendor ID */
234         Icw2            = 0x0F,         /* Initialization Control Word 2 */
235 };
236
237 enum {                                  /* Mdic */
238         MDIdMASK        = 0x0000FFFF,   /* Data */
239         MDIdSHIFT       = 0,
240         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
241         MDIrSHIFT       = 16,
242         MDIpMASK        = 0x03E00000,   /* PHY Address */
243         MDIpSHIFT       = 21,
244         MDIwop          = 0x04000000,   /* Write Operation */
245         MDIrop          = 0x08000000,   /* Read Operation */
246         MDIready        = 0x10000000,   /* End of Transaction */
247         MDIie           = 0x20000000,   /* Interrupt Enable */
248         MDIe            = 0x40000000,   /* Error */
249 };
250
251 enum {                                  /* Icr, Ics, Ims, Imc */
252         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
253         Txqe            = 0x00000002,   /* Transmit Queue Empty */
254         Lsc             = 0x00000004,   /* Link Status Change */
255         Rxseq           = 0x00000008,   /* Receive Sequence Error */
256         Rxdmt0          = 0x00000010,   /* Rd Minimum Threshold Reached */
257         Rxo             = 0x00000040,   /* Receiver Overrun */
258         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
259         Mdac            = 0x00000200,   /* MDIO Access Completed */
260         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
261         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
262         Gpi1            = 0x00001000,
263         Gpi2            = 0x00002000,
264         Gpi3            = 0x00004000,
265 };
266
267 /*
268  * The Mdic register isn't implemented on the 82543GC,
269  * the software defined pins are used instead.
270  * These definitions work for the Intel PRO/1000 T Server Adapter.
271  * The direction pin bits are read from the EEPROM.
272  */
273 enum {
274         Mdd             = ((1<<2)<<SwdpinsloSHIFT),     /* data */
275         Mddo            = ((1<<2)<<SwdpioloSHIFT),      /* pin direction */
276         Mdc             = ((1<<3)<<SwdpinsloSHIFT),     /* clock */
277         Mdco            = ((1<<3)<<SwdpioloSHIFT),      /* pin direction */
278         Mdr             = ((1<<0)<<SwdpinshiSHIFT),     /* reset */
279         Mdro            = ((1<<0)<<SwdpiohiSHIFT),      /* pin direction */
280 };
281
282 enum {                                  /* Txcw */
283         TxcwFd          = 0x00000020,   /* Full Duplex */
284         TxcwHd          = 0x00000040,   /* Half Duplex */
285         TxcwPauseMASK   = 0x00000180,   /* Pause */
286         TxcwPauseSHIFT  = 7,
287         TxcwPs          = (1<<TxcwPauseSHIFT),  /* Pause Supported */
288         TxcwAs          = (2<<TxcwPauseSHIFT),  /* Asymmetric FC desired */
289         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
290         TxcwRfiSHIFT    = 12,
291         TxcwNpr         = 0x00008000,   /* Next Page Request */
292         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
293         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
294 };
295
296 enum {                                  /* Rxcw */
297         Rxword          = 0x0000FFFF,   /* Data from auto-negotiation process */
298         Rxnocarrier     = 0x04000000,   /* Carrier Sense indication */
299         Rxinvalid       = 0x08000000,   /* Invalid Symbol during configuration */
300         Rxchange        = 0x10000000,   /* Change to the Rxword indication */
301         Rxconfig        = 0x20000000,   /* /C/ order set reception indication */
302         Rxsync          = 0x40000000,   /* Lost bit synchronization indication */
303         Anc             = 0x80000000,   /* Auto Negotiation Complete */
304 };
305
306 enum {                                  /* Rctl */
307         Rrst            = 0x00000001,   /* Receiver Software Reset */
308         Ren             = 0x00000002,   /* Receiver Enable */
309         Sbp             = 0x00000004,   /* Store Bad Packets */
310         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
311         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
312         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
313         LbmMASK         = 0x000000C0,   /* Loopback Mode */
314         LbmOFF          = 0x00000000,   /* No Loopback */
315         LbmTBI          = 0x00000040,   /* TBI Loopback */
316         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
317         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
318         RdtmsMASK       = 0x00000300,   /* Rd Minimum Threshold Size */
319         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
320         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
321         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
322         MoMASK          = 0x00003000,   /* Multicast Offset */
323         Mo47b36         = 0x00000000,   /* bits [47:36] of received address */
324         Mo46b35         = 0x00001000,   /* bits [46:35] of received address */
325         Mo45b34         = 0x00002000,   /* bits [45:34] of received address */
326         Mo43b32         = 0x00003000,   /* bits [43:32] of received address */
327         Bam             = 0x00008000,   /* Broadcast Accept Mode */
328         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
329         Bsize2048       = 0x00000000,   /* Bsex = 0 */
330         Bsize1024       = 0x00010000,   /* Bsex = 0 */
331         Bsize512        = 0x00020000,   /* Bsex = 0 */
332         Bsize256        = 0x00030000,   /* Bsex = 0 */
333         Bsize16384      = 0x00010000,   /* Bsex = 1 */
334         Vfe             = 0x00040000,   /* VLAN Filter Enable */
335         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
336         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
337         Dpf             = 0x00400000,   /* Discard Pause Frames */
338         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
339         Bsex            = 0x02000000,   /* Buffer Size Extension */
340         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
341 };
342
343 enum {                                  /* Tctl */
344         Trst            = 0x00000001,   /* Transmitter Software Reset */
345         Ten             = 0x00000002,   /* Transmit Enable */
346         Psp             = 0x00000008,   /* Pad Short Packets */
347         CtMASK          = 0x00000FF0,   /* Collision Threshold */
348         CtSHIFT         = 4,
349         ColdMASK        = 0x003FF000,   /* Collision Distance */
350         ColdSHIFT       = 12,
351         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
352         Pbe             = 0x00800000,   /* Packet Burst Enable */
353         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
354         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
355 };
356
357 enum {                                  /* [RT]xdctl */
358         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
359         PthreshSHIFT    = 0,
360         HthreshMASK     = 0x00003F00,   /* Host Threshold */
361         HthreshSHIFT    = 8,
362         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
363         WthreshSHIFT    = 16,
364         Gran            = 0x01000000,   /* Granularity */
365         LthreshMASK     = 0xFE000000,   /* Low Threshold */
366         LthreshSHIFT    = 25,
367 };
368
369 enum {                                  /* Rxcsum */
370         PcssMASK        = 0x000000FF,   /* Packet Checksum Start */
371         PcssSHIFT       = 0,
372         Ipofl           = 0x00000100,   /* IP Checksum Off-load Enable */
373         Tuofl           = 0x00000200,   /* TCP/UDP Checksum Off-load Enable */
374 };
375
376 enum {                                  /* Manc */
377         Arpen           = 0x00002000,   /* Enable ARP Request Filtering */
378 };
379
380 enum {                                  /* Receive Delay Timer Ring */
381         DelayMASK       = 0x0000FFFF,   /* delay timer in 1.024nS increments */
382         DelaySHIFT      = 0,
383         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
384 };
385
386 typedef struct Rd {                     /* Receive Descriptor */
387         unsigned int    addr[2];
388         uint16_t        length;
389         uint16_t        checksum;
390         uint8_t status;
391         uint8_t errors;
392         uint16_t        special;
393 } Rd;
394
395 enum {                                  /* Rd status */
396         Rdd             = 0x01,         /* Descriptor Done */
397         Reop            = 0x02,         /* End of Packet */
398         Ixsm            = 0x04,         /* Ignore Checksum Indication */
399         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
400         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
401         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
402         Pif             = 0x80,         /* Passed in-exact filter */
403 };
404
405 enum {                                  /* Rd errors */
406         Ce              = 0x01,         /* CRC Error or Alignment Error */
407         Se              = 0x02,         /* Symbol Error */
408         Seq             = 0x04,         /* Sequence Error */
409         Cxe             = 0x10,         /* Carrier Extension Error */
410         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
411         Ipe             = 0x40,         /* IP Checksum Error */
412         Rxe             = 0x80,         /* RX Data Error */
413 };
414
415 typedef struct Td Td;
416 struct Td {                             /* Transmit Descriptor */
417         union {
418                 unsigned int    addr[2];        /* Data */
419                 struct {                /* Context */
420                         uint8_t ipcss;
421                         uint8_t ipcso;
422                         uint16_t        ipcse;
423                         uint8_t tucss;
424                         uint8_t tucso;
425                         uint16_t        tucse;
426                 };
427         };
428         unsigned int    control;
429         unsigned int    status;
430 };
431
432 enum {                                  /* Td control */
433         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
434         LenSHIFT        = 0,
435         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
436         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
437         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
438         Teop            = 0x01000000,   /* End of Packet (DD) */
439         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
440         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
441         Tse             = 0x04000000,   /* TCP Segmentation Enable */
442         Rs              = 0x08000000,   /* Report Status */
443         Rps             = 0x10000000,   /* Report Status Sent */
444         Dext            = 0x20000000,   /* Descriptor Extension */
445         Vle             = 0x40000000,   /* VLAN Packet Enable */
446         Ide             = 0x80000000,   /* Interrupt Delay Enable */
447 };
448
449 enum {                                  /* Td status */
450         Tdd             = 0x00000001,   /* Descriptor Done */
451         Ec              = 0x00000002,   /* Excess Collisions */
452         Lc              = 0x00000004,   /* Late Collision */
453         Tu              = 0x00000008,   /* Transmit Underrun */
454         Iixsm           = 0x00000100,   /* Insert IP Checksum */
455         Itxsm           = 0x00000200,   /* Insert TCP/UDP Checksum */
456         HdrlenMASK      = 0x0000FF00,   /* Header Length (Tse) */
457         HdrlenSHIFT     = 8,
458         VlanMASK        = 0x0FFF0000,   /* VLAN Identifier */
459         VlanSHIFT       = 16,
460         Tcfi            = 0x10000000,   /* Canonical Form Indicator */
461         PriMASK         = 0xE0000000,   /* User Priority */
462         PriSHIFT        = 29,
463         MssMASK         = 0xFFFF0000,   /* Maximum Segment Size (Tse) */
464         MssSHIFT        = 16,
465 };
466
467 enum {
468         Nrd             = 256,          /* multiple of 8 */
469         Ntd             = 64,           /* multiple of 8 */
470         Nrb             = 1024,         /* private receive buffers per Ctlr */
471         Rbsz            = 2048,
472 };
473
474 struct ctlr {
475         int     port;
476         struct pci_device *pci;
477         struct ctlr*    next;
478         struct ether*   edev;
479         int     active;
480         int     started;
481         int     id;
482         int     cls;
483         uint16_t        eeprom[0x40];
484
485         qlock_t alock;                  /* attach */
486         void*   alloc;                  /* receive/transmit descriptors */
487         int     nrd;
488         int     ntd;
489         int     nrb;                    /* how many this Ctlr has in the pool */
490
491         int*    nic;
492         spinlock_t      imlock;
493         int     im;                     /* interrupt mask */
494
495         struct mii*     mii;
496         struct rendez   lrendez;
497         int     lim;
498
499         int     link;
500
501         qlock_t slock;
502         unsigned int    statistics[Nstatistics];
503         unsigned int    lsleep;
504         unsigned int    lintr;
505         unsigned int    rsleep;
506         unsigned int    rintr;
507         unsigned int    txdw;
508         unsigned int    tintr;
509         unsigned int    ixsm;
510         unsigned int    ipcs;
511         unsigned int    tcpcs;
512
513         uint8_t ra[Eaddrlen];           /* receive address */
514         uint32_t        mta[128];               /* multicast table array */
515
516         struct rendez   rrendez;
517         int     rim;
518         int     rdfree;
519         Rd*     rdba;                   /* receive descriptor base address */
520         struct block**  rb;                     /* receive buffers */
521         int     rdh;                    /* receive descriptor head */
522         int     rdt;                    /* receive descriptor tail */
523         int     rdtr;                   /* receive delay timer ring value */
524
525         spinlock_t      tlock;
526         int     tbusy;
527         int     tdfree;
528         Td*     tdba;                   /* transmit descriptor base address */
529         struct block**  tb;                     /* transmit buffers */
530         int     tdh;                    /* transmit descriptor head */
531         int     tdt;                    /* transmit descriptor tail */
532
533         int     txcw;
534         int     fcrtl;
535         int     fcrth;
536 };
537
538 #define csr32r(c, r)    (*((c)->nic+((r)/4)))
539 #define csr32w(c, r, v) (*((c)->nic+((r)/4)) = (v))
540
541 static struct ctlr* igbectlrhead;
542 static struct ctlr* igbectlrtail;
543
544 /* lock for igberpool (free receive Blocks) */
545 static spinlock_t igberblock = SPINLOCK_INITIALIZER_IRQSAVE;
546 static struct block* igberbpool;        /* receive Blocks for all igbe controllers */
547
548 static char* statistics[Nstatistics] = {
549         "CRC Error",
550         "Alignment Error",
551         "Symbol Error",
552         "RX Error",
553         "Missed Packets",
554         "Single Collision",
555         "Excessive Collisions",
556         "Multiple Collision",
557         "Late Collisions",
558         NULL,
559         "Collision",
560         "Transmit Underrun",
561         "Defer",
562         "Transmit - No CRS",
563         "Sequence Error",
564         "Carrier Extension Error",
565         "Receive Error Length",
566         NULL,
567         "XON Received",
568         "XON Transmitted",
569         "XOFF Received",
570         "XOFF Transmitted",
571         "FC Received Unsupported",
572         "Packets Received (64 Bytes)",
573         "Packets Received (65-127 Bytes)",
574         "Packets Received (128-255 Bytes)",
575         "Packets Received (256-511 Bytes)",
576         "Packets Received (512-1023 Bytes)",
577         "Packets Received (1024-1522 Bytes)",
578         "Good Packets Received",
579         "Broadcast Packets Received",
580         "Multicast Packets Received",
581         "Good Packets Transmitted",
582         NULL,
583         "Good Octets Received",
584         NULL,
585         "Good Octets Transmitted",
586         NULL,
587         NULL,
588         NULL,
589         "Receive No Buffers",
590         "Receive Undersize",
591         "Receive Fragment",
592         "Receive Oversize",
593         "Receive Jabber",
594         NULL,
595         NULL,
596         NULL,
597         "Total Octets Received",
598         NULL,
599         "Total Octets Transmitted",
600         NULL,
601         "Total Packets Received",
602         "Total Packets Transmitted",
603         "Packets Transmitted (64 Bytes)",
604         "Packets Transmitted (65-127 Bytes)",
605         "Packets Transmitted (128-255 Bytes)",
606         "Packets Transmitted (256-511 Bytes)",
607         "Packets Transmitted (512-1023 Bytes)",
608         "Packets Transmitted (1024-1522 Bytes)",
609         "Multicast Packets Transmitted",
610         "Broadcast Packets Transmitted",
611         "TCP Segmentation Context Transmitted",
612         "TCP Segmentation Context Fail",
613 };
614
615 static long
616 igbeifstat(struct ether* edev, void* a, long n, uint32_t offset)
617 {
618         struct ctlr *ctlr;
619         char *p, *s;
620         int i, l, r;
621         uint64_t tuvl, ruvl;
622
623         ctlr = edev->ctlr;
624         qlock(&ctlr->slock);
625         p = kzmalloc(READSTR, 0);
626         if(p == NULL) {
627                 qunlock(&ctlr->slock);
628                 error(Enomem);
629         }
630         l = 0;
631         for(i = 0; i < Nstatistics; i++){
632                 r = csr32r(ctlr, Statistics+i*4);
633                 if((s = statistics[i]) == NULL)
634                         continue;
635                 switch(i){
636                 case Gorcl:
637                 case Gotcl:
638                 case Torl:
639                 case Totl:
640                         ruvl = r;
641                         ruvl += ((uint64_t)csr32r(ctlr, Statistics+(i+1)*4))<<32;
642                         tuvl = ruvl;
643                         tuvl += ctlr->statistics[i];
644                         tuvl += ((uint64_t)ctlr->statistics[i+1])<<32;
645                         if(tuvl == 0)
646                                 continue;
647                         ctlr->statistics[i] = tuvl;
648                         ctlr->statistics[i+1] = tuvl>>32;
649                         l += snprintf(p+l, READSTR-l, "%s: %llud %llud\n",
650                                 s, tuvl, ruvl);
651                         i++;
652                         break;
653
654                 default:
655                         ctlr->statistics[i] += r;
656                         if(ctlr->statistics[i] == 0)
657                                 continue;
658                         l += snprintf(p+l, READSTR-l, "%s: %ud %ud\n",
659                                 s, ctlr->statistics[i], r);
660                         break;
661                 }
662         }
663
664         l += snprintf(p+l, READSTR-l, "lintr: %ud %ud\n",
665                 ctlr->lintr, ctlr->lsleep);
666         l += snprintf(p+l, READSTR-l, "rintr: %ud %ud\n",
667                 ctlr->rintr, ctlr->rsleep);
668         l += snprintf(p+l, READSTR-l, "tintr: %ud %ud\n",
669                 ctlr->tintr, ctlr->txdw);
670         l += snprintf(p+l, READSTR-l, "ixcs: %ud %ud %ud\n",
671                 ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
672         l += snprintf(p+l, READSTR-l, "rdtr: %ud\n", ctlr->rdtr);
673         l += snprintf(p+l, READSTR-l, "Ctrlext: %08x\n", csr32r(ctlr, Ctrlext));
674
675         l += snprintf(p+l, READSTR-l, "eeprom:");
676         for(i = 0; i < 0x40; i++){
677                 if(i && ((i & 0x07) == 0))
678                         l += snprintf(p+l, READSTR-l, "\n       ");
679                 l += snprintf(p+l, READSTR-l, " %4.4uX", ctlr->eeprom[i]);
680         }
681         l += snprintf(p+l, READSTR-l, "\n");
682
683         if(ctlr->mii != NULL && ctlr->mii->curphy != NULL){
684                 l += snprintf(p+l, READSTR-l, "phy:   ");
685                 for(i = 0; i < NMiiPhyr; i++){
686                         if(i && ((i & 0x07) == 0))
687                                 l += snprintf(p+l, READSTR-l, "\n       ");
688                         r = miimir(ctlr->mii, i);
689                         l += snprintf(p+l, READSTR-l, " %4.4uX", r);
690                 }
691                 snprintf(p+l, READSTR-l, "\n");
692         }
693         n = readstr(offset, a, n, p);
694         kfree(p);
695         qunlock(&ctlr->slock);
696
697         return n;
698 }
699
700 enum {
701         CMrdtr,
702 };
703
704 static struct cmdtab igbectlmsg[] = {
705         {CMrdtr,        "rdtr", 2},
706 };
707
708 static long
709 igbectl(struct ether* edev, void* buf, long n)
710 {
711         ERRSTACK(2);
712         int v;
713         char *p;
714         struct ctlr *ctlr;
715         struct cmdbuf *cb;
716         struct cmdtab *ct;
717
718         if((ctlr = edev->ctlr) == NULL)
719                 error(Enonexist);
720
721         cb = parsecmd(buf, n);
722         if(waserror()){
723                 kfree(cb);
724                 nexterror();
725         }
726
727         ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
728         switch(ct->index){
729         case CMrdtr:
730                 v = strtol(cb->f[1], &p, 0);
731                 if(v < 0 || p == cb->f[1] || v > 0xFFFF)
732                         error(Ebadarg);
733                 ctlr->rdtr = v;
734                 csr32w(ctlr, Rdtr, Fpd|v);
735                 break;
736         }
737         kfree(cb);
738         poperror();
739
740         return n;
741 }
742
743 static void
744 igbepromiscuous(void* arg, int on)
745 {
746         int rctl;
747         struct ctlr *ctlr;
748         struct ether *edev;
749
750         edev = arg;
751         ctlr = edev->ctlr;
752
753         rctl = csr32r(ctlr, Rctl);
754         rctl &= ~MoMASK;
755         rctl |= Mo47b36;
756         if(on)
757                 rctl |= Upe|Mpe;
758         else
759                 rctl &= ~(Upe|Mpe);
760         csr32w(ctlr, Rctl, rctl|Mpe);   /* temporarily keep Mpe on */
761 }
762
763 static void
764 igbemulticast(void* arg, uint8_t* addr, int add)
765 {
766         int bit, x;
767         struct ctlr *ctlr;
768         struct ether *edev;
769
770         edev = arg;
771         ctlr = edev->ctlr;
772
773         x = addr[5]>>1;
774         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
775         /*
776          * multiple ether addresses can hash to the same filter bit,
777          * so it's never safe to clear a filter bit.
778          * if we want to clear filter bits, we need to keep track of
779          * all the multicast addresses in use, clear all the filter bits,
780          * then set the ones corresponding to in-use addresses.
781          */
782         if(add)
783                 ctlr->mta[x] |= 1<<bit;
784 //      else
785 //              ctlr->mta[x] &= ~(1<<bit);
786
787         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
788 }
789
790 static struct block*
791 igberballoc(void)
792 {
793         struct block *bp;
794
795         ilock(&igberblock);
796         if((bp = igberbpool) != NULL){
797                 igberbpool = bp->next;
798                 bp->next = NULL;
799                 /* _xinc(&bp->ref);     prevent bp from being freed */
800         }
801         iunlock(&igberblock);
802
803         return bp;
804 }
805
806 static void
807 igberbfree(struct block* bp)
808 {
809         bp->rp = bp->lim - Rbsz;
810         bp->wp = bp->rp;
811         bp->flag &= ~(Bipck | Budpck | Btcpck | Bpktck);
812
813         ilock(&igberblock);
814         bp->next = igberbpool;
815         igberbpool = bp;
816         iunlock(&igberblock);
817 }
818
819 static void
820 igbeim(struct ctlr* ctlr, int im)
821 {
822         ilock(&ctlr->imlock);
823         ctlr->im |= im;
824         csr32w(ctlr, Ims, ctlr->im);
825         iunlock(&ctlr->imlock);
826 }
827
828 static int
829 igbelim(void* ctlr)
830 {
831         return ((struct ctlr*)ctlr)->lim != 0;
832 }
833
834 static void
835 igbelproc(void* arg)
836 {
837         struct ctlr *ctlr;
838         struct ether *edev;
839         struct miiphy *phy;
840         int ctrl, r;
841
842         edev = arg;
843         ctlr = edev->ctlr;
844         for(;;){
845                 /* plan9 originally had a busy loop here (just called continue).  though
846                  * either you have the mii or you don't.  i don't think it'll magically
847                  * show up later (it should have been initialized during pnp/pci, which
848                  * is before attach, which is before lproc).  -brho */
849                 if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
850                         printk("[kernel] igbelproc can't find a mii/curphy, aborting!\n");
851                         /* name alloc'd in attach */
852                         kfree(per_cpu_info[core_id()].cur_kthread->name);
853                         return;
854                 }
855                 /*
856                  * To do:
857                  *      logic to manage status change,
858                  *      this is incomplete but should work
859                  *      one time to set up the hardware.
860                  *
861                  *      MiiPhy.speed, etc. should be in Mii.
862                  */
863                 if(miistatus(ctlr->mii) < 0)
864                         //continue;     /* this comment out was plan9, not brho */
865                         goto enable;
866
867                 phy = ctlr->mii->curphy;
868                 ctrl = csr32r(ctlr, Ctrl);
869
870                 switch(ctlr->id){
871                 case i82543gc:
872                 case i82544ei:
873                 case i82544eif:
874                 default:
875                         if(!(ctrl & Asde)){
876                                 ctrl &= ~(SspeedMASK|Ilos|Fd);
877                                 ctrl |= Frcdplx|Frcspd;
878                                 if(phy->speed == 1000)
879                                         ctrl |= Sspeed1000;
880                                 else if(phy->speed == 100)
881                                         ctrl |= Sspeed100;
882                                 if(phy->fd)
883                                         ctrl |= Fd;
884                         }
885                         break;
886
887                 case i82540em:
888                 case i82540eplp:
889                 case i82547gi:
890                 case i82541gi:
891                 case i82541gi2:
892                 case i82541pi:
893                         break;
894                 }
895
896                 /*
897                  * Collision Distance.
898                  */
899                 r = csr32r(ctlr, Tctl);
900                 r &= ~ColdMASK;
901                 if(phy->fd)
902                         r |= 64<<ColdSHIFT;
903                 else
904                         r |= 512<<ColdSHIFT;
905                 csr32w(ctlr, Tctl, r);
906
907                 /*
908                  * Flow control.
909                  */
910                 if(phy->rfc)
911                         ctrl |= Rfce;
912                 if(phy->tfc)
913                         ctrl |= Tfce;
914                 csr32w(ctlr, Ctrl, ctrl);
915
916 enable:
917                 ctlr->lim = 0;
918                 igbeim(ctlr, Lsc);
919
920                 ctlr->lsleep++;
921                 rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
922         }
923 }
924
925 static void
926 igbetxinit(struct ctlr* ctlr)
927 {
928         int i, r;
929         struct block *bp;
930
931         csr32w(ctlr, Tctl, (0x0F<<CtSHIFT)|Psp|(66<<ColdSHIFT));
932         switch(ctlr->id){
933         default:
934                 r = 6;
935                 break;
936         case i82543gc:
937         case i82544ei:
938         case i82544eif:
939         case i82544gc:
940         case i82540em:
941         case i82540eplp:
942         case i82541ei:
943         case i82541gi:
944         case i82541gi2:
945         case i82541pi:
946         case i82545em:
947         case i82545gmc:
948         case i82546gb:
949         case i82546eb:
950         case i82547ei:
951         case i82547gi:
952                 r = 8;
953                 break;
954         }
955         csr32w(ctlr, Tipg, (6<<20)|(8<<10)|r);
956         csr32w(ctlr, Ait, 0);
957         csr32w(ctlr, Txdmac, 0);
958         csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
959         csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
960         csr32w(ctlr, Tdlen, ctlr->ntd*sizeof(Td));
961         ctlr->tdh = PREV_RING(0, ctlr->ntd);
962         csr32w(ctlr, Tdh, 0);
963         ctlr->tdt = 0;
964         csr32w(ctlr, Tdt, 0);
965
966         for(i = 0; i < ctlr->ntd; i++){
967                 if((bp = ctlr->tb[i]) != NULL){
968                         ctlr->tb[i] = NULL;
969                         freeb(bp);
970                 }
971                 memset(&ctlr->tdba[i], 0, sizeof(Td));
972         }
973         ctlr->tdfree = ctlr->ntd;
974
975         csr32w(ctlr, Tidv, 128);
976         r = (4<<WthreshSHIFT)|(4<<HthreshSHIFT)|(8<<PthreshSHIFT);
977
978         switch(ctlr->id){
979         default:
980                 break;
981         case i82540em:
982         case i82540eplp:
983         case i82547gi:
984         case i82545em:
985         case i82545gmc:
986         case i82546gb:
987         case i82546eb:
988         case i82541gi:
989         case i82541gi2:
990         case i82541pi:
991                 r = csr32r(ctlr, Txdctl);
992                 r &= ~WthreshMASK;
993                 r |= Gran|(4<<WthreshSHIFT);
994
995                 csr32w(ctlr, Tadv, 64);
996                 break;
997         }
998
999         csr32w(ctlr, Txdctl, r);
1000
1001         r = csr32r(ctlr, Tctl);
1002         r |= Ten;
1003         csr32w(ctlr, Tctl, r);
1004 }
1005
1006 static void
1007 igbetransmit(struct ether* edev)
1008 {
1009         Td *td;
1010         struct block *bp;
1011         struct ctlr *ctlr;
1012         int tdh, tdt;
1013
1014         ctlr = edev->ctlr;
1015
1016         ilock(&ctlr->tlock);
1017
1018         /*
1019          * Free any completed packets
1020          */
1021         tdh = ctlr->tdh;
1022         while(NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)){
1023                 if((bp = ctlr->tb[tdh]) != NULL){
1024                         ctlr->tb[tdh] = NULL;
1025                         freeb(bp);
1026                 }
1027                 memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1028                 tdh = NEXT_RING(tdh, ctlr->ntd);
1029         }
1030         ctlr->tdh = tdh;
1031
1032         /*
1033          * Try to fill the ring back up.
1034          */
1035         tdt = ctlr->tdt;
1036         while(NEXT_RING(tdt, ctlr->ntd) != tdh){
1037                 if((bp = qget(edev->oq)) == NULL)
1038                         break;
1039                 td = &ctlr->tdba[tdt];
1040                 td->addr[0] = paddr_low32(bp->rp);
1041                 td->addr[1] = paddr_high32(bp->rp);
1042                 td->control = ((BLEN(bp) & LenMASK)<<LenSHIFT);
1043                 td->control |= Dext|Ifcs|Teop|DtypeDD;
1044                 ctlr->tb[tdt] = bp;
1045                 tdt = NEXT_RING(tdt, ctlr->ntd);
1046                 if(NEXT_RING(tdt, ctlr->ntd) == tdh){
1047                         td->control |= Rs;
1048                         ctlr->txdw++;
1049                         ctlr->tdt = tdt;
1050                         csr32w(ctlr, Tdt, tdt);
1051                         igbeim(ctlr, Txdw);
1052                         break;
1053                 }
1054                 ctlr->tdt = tdt;
1055                 csr32w(ctlr, Tdt, tdt);
1056         }
1057
1058         iunlock(&ctlr->tlock);
1059 }
1060
1061 static void
1062 igbereplenish(struct ctlr* ctlr)
1063 {
1064         Rd *rd;
1065         int rdt;
1066         struct block *bp;
1067
1068         rdt = ctlr->rdt;
1069         while(NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh){
1070                 rd = &ctlr->rdba[rdt];
1071                 if(ctlr->rb[rdt] == NULL){
1072                         bp = igberballoc();
1073                         if(bp == NULL){
1074                                 /* needs to be a safe print for interrupt level */
1075                                 printk("#l%d: igbereplenish: no available buffers\n",
1076                                         ctlr->edev->ctlrno);
1077                                 break;
1078                         }
1079                         ctlr->rb[rdt] = bp;
1080                         rd->addr[0] = paddr_low32(bp->rp);
1081                         rd->addr[1] = paddr_high32(bp->rp);
1082                 }
1083                 wmb();  /* ensure prev rd writes come before status = 0. */
1084                 rd->status = 0;
1085                 rdt = NEXT_RING(rdt, ctlr->nrd);
1086                 ctlr->rdfree++;
1087         }
1088         ctlr->rdt = rdt;
1089         csr32w(ctlr, Rdt, rdt);
1090 }
1091
1092 static void
1093 igberxinit(struct ctlr* ctlr)
1094 {
1095         int i;
1096         struct block *bp;
1097
1098         /* temporarily keep Mpe on */
1099         csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF|Mpe);
1100         csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1101         csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1102         csr32w(ctlr, Rdlen, ctlr->nrd*sizeof(Rd));
1103         ctlr->rdh = 0;
1104         csr32w(ctlr, Rdh, 0);
1105         ctlr->rdt = 0;
1106         csr32w(ctlr, Rdt, 0);
1107         ctlr->rdtr = 0;
1108         csr32w(ctlr, Rdtr, Fpd|0);
1109
1110         for(i = 0; i < ctlr->nrd; i++){
1111                 if((bp = ctlr->rb[i]) != NULL){
1112                         ctlr->rb[i] = NULL;
1113                         freeb(bp);
1114                 }
1115         }
1116         igbereplenish(ctlr);
1117
1118         switch(ctlr->id){
1119         case i82540em:
1120         case i82540eplp:
1121         case i82541gi:
1122         case i82541gi2:
1123         case i82541pi:
1124         case i82545em:
1125         case i82545gmc:
1126         case i82546gb:
1127         case i82546eb:
1128         case i82547gi:
1129                 csr32w(ctlr, Radv, 64);
1130                 break;
1131         }
1132         csr32w(ctlr, Rxdctl, (8<<WthreshSHIFT)|(8<<HthreshSHIFT)|4);
1133
1134         /*
1135          * Enable checksum offload.
1136          */
1137         csr32w(ctlr, Rxcsum, Tuofl|Ipofl|(ETHERHDRSIZE<<PcssSHIFT));
1138 }
1139
1140 static int
1141 igberim(void* ctlr)
1142 {
1143         return ((struct ctlr*)ctlr)->rim != 0;
1144 }
1145
1146 static void
1147 igberproc(void* arg)
1148 {
1149         Rd *rd;
1150         struct block *bp;
1151         struct ctlr *ctlr;
1152         int r, rdh;
1153         struct ether *edev;
1154
1155         edev = arg;
1156         ctlr = edev->ctlr;
1157
1158         igberxinit(ctlr);
1159         r = csr32r(ctlr, Rctl);
1160         r |= Ren;
1161         csr32w(ctlr, Rctl, r);
1162
1163         for(;;){
1164                 ctlr->rim = 0;
1165                 igbeim(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq);
1166                 ctlr->rsleep++;
1167                 rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1168
1169                 rdh = ctlr->rdh;
1170                 for(;;){
1171                         rd = &ctlr->rdba[rdh];
1172
1173                         if(!(rd->status & Rdd))
1174                                 break;
1175
1176                         /*
1177                          * Accept eop packets with no errors.
1178                          * With no errors and the Ixsm bit set,
1179                          * the descriptor status Tpcs and Ipcs bits give
1180                          * an indication of whether the checksums were
1181                          * calculated and valid.
1182                          */
1183                         if((rd->status & Reop) && rd->errors == 0){
1184                                 bp = ctlr->rb[rdh];
1185                                 ctlr->rb[rdh] = NULL;
1186                                 bp->wp += rd->length;
1187                                 bp->next = NULL;
1188                                 if(!(rd->status & Ixsm)){
1189                                         ctlr->ixsm++;
1190                                         if(rd->status & Ipcs){
1191                                                 /*
1192                                                  * IP checksum calculated
1193                                                  * (and valid as errors == 0).
1194                                                  */
1195                                                 ctlr->ipcs++;
1196                                                 bp->flag |= Bipck;
1197                                         }
1198                                         if(rd->status & Tcpcs){
1199                                                 /*
1200                                                  * TCP/UDP checksum calculated
1201                                                  * (and valid as errors == 0).
1202                                                  */
1203                                                 ctlr->tcpcs++;
1204                                                 bp->flag |= Btcpck|Budpck;
1205                                         }
1206                                         bp->checksum = rd->checksum;
1207                                         bp->flag |= Bpktck;
1208                                 }
1209                                 etheriq(edev, bp, 1);
1210                         }
1211                         else if(ctlr->rb[rdh] != NULL){
1212                                 freeb(ctlr->rb[rdh]);
1213                                 ctlr->rb[rdh] = NULL;
1214                         }
1215
1216                         memset(rd, 0, sizeof(Rd));
1217                         wmb();  /* make sure the zeroing happens before free (i think) */
1218                         ctlr->rdfree--;
1219                         rdh = NEXT_RING(rdh, ctlr->nrd);
1220                 }
1221                 ctlr->rdh = rdh;
1222
1223                 if(ctlr->rdfree < ctlr->nrd/2 || (ctlr->rim & Rxdmt0))
1224                         igbereplenish(ctlr);
1225         }
1226 }
1227
1228 static void
1229 igbeattach(struct ether* edev)
1230 {
1231         ERRSTACK(1);
1232         struct block *bp;
1233         struct ctlr *ctlr;
1234         char *name;
1235
1236         ctlr = edev->ctlr;
1237         ctlr->edev = edev;                      /* point back to Ether* */
1238         qlock(&ctlr->alock);
1239         if(ctlr->alloc != NULL){                        /* already allocated? */
1240                 qunlock(&ctlr->alock);
1241                 return;
1242         }
1243
1244         ctlr->tb = NULL;
1245         ctlr->rb = NULL;
1246         ctlr->alloc = NULL;
1247         ctlr->nrb = 0;
1248         if(waserror()){
1249                 while(ctlr->nrb > 0){
1250                         bp = igberballoc();
1251                         bp->free = NULL;
1252                         freeb(bp);
1253                         ctlr->nrb--;
1254                 }
1255                 kfree(ctlr->tb);
1256                 ctlr->tb = NULL;
1257                 kfree(ctlr->rb);
1258                 ctlr->rb = NULL;
1259                 kfree(ctlr->alloc);
1260                 ctlr->alloc = NULL;
1261                 qunlock(&ctlr->alock);
1262                 nexterror();
1263         }
1264
1265         ctlr->nrd = Nrd;
1266         ctlr->ntd = Ntd;
1267         ctlr->alloc = kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1268         if(ctlr->alloc == NULL) {
1269                 printd("igbe: can't allocate ctlr->alloc\n");
1270                 error(Enomem);
1271         }
1272         ctlr->rdba = (Rd*)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1273         ctlr->tdba = (Td*)(ctlr->rdba+ctlr->nrd);
1274
1275         ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1276         ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1277         if (ctlr->rb == NULL || ctlr->tb == NULL) {
1278                 printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1279                 error(Enomem);
1280         }
1281
1282         for(ctlr->nrb = 0; ctlr->nrb < Nrb; ctlr->nrb++){
1283                 if((bp = allocb(Rbsz)) == NULL)
1284                         break;
1285                 bp->free = igberbfree;
1286                 freeb(bp);
1287         }
1288
1289         /* the ktasks should free these names, if they ever exit */
1290         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1291         snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1292         ktask(name, igbelproc, edev);
1293
1294         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1295         snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1296         ktask(name, igberproc, edev);
1297
1298         igbetxinit(ctlr);
1299
1300         qunlock(&ctlr->alock);
1301         poperror();
1302 }
1303
1304 static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1305 {
1306         struct ctlr *ctlr;
1307         struct ether *edev;
1308         int icr, im, txdw;
1309
1310         edev = arg;
1311         ctlr = edev->ctlr;
1312
1313         ilock(&ctlr->imlock);
1314         csr32w(ctlr, Imc, ~0);
1315         im = ctlr->im;
1316         txdw = 0;
1317
1318         while((icr = csr32r(ctlr, Icr) & ctlr->im) != 0){
1319                 if(icr & Lsc){
1320                         im &= ~Lsc;
1321                         ctlr->lim = icr & Lsc;
1322                         rendez_wakeup(&ctlr->lrendez);
1323                         ctlr->lintr++;
1324                 }
1325                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq)){
1326                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq);
1327                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq);
1328                         rendez_wakeup(&ctlr->rrendez);
1329                         ctlr->rintr++;
1330                 }
1331                 if(icr & Txdw){
1332                         im &= ~Txdw;
1333                         txdw++;
1334                         ctlr->tintr++;
1335                 }
1336         }
1337
1338         ctlr->im = im;
1339         csr32w(ctlr, Ims, im);
1340         iunlock(&ctlr->imlock);
1341
1342         if(txdw)
1343                 igbetransmit(edev);
1344 }
1345
1346 static int
1347 i82543mdior(struct ctlr* ctlr, int n)
1348 {
1349         int ctrl, data, i, r;
1350
1351         /*
1352          * Read n bits from the Management Data I/O Interface.
1353          */
1354         ctrl = csr32r(ctlr, Ctrl);
1355         r = (ctrl & ~Mddo)|Mdco;
1356         data = 0;
1357         for(i = n-1; i >= 0; i--){
1358                 if(csr32r(ctlr, Ctrl) & Mdd)
1359                         data |= (1<<i);
1360                 csr32w(ctlr, Ctrl, Mdc|r);
1361                 csr32w(ctlr, Ctrl, r);
1362         }
1363         csr32w(ctlr, Ctrl, ctrl);
1364
1365         return data;
1366 }
1367
1368 static int
1369 i82543mdiow(struct ctlr* ctlr, int bits, int n)
1370 {
1371         int ctrl, i, r;
1372
1373         /*
1374          * Write n bits to the Management Data I/O Interface.
1375          */
1376         ctrl = csr32r(ctlr, Ctrl);
1377         r = Mdco|Mddo|ctrl;
1378         for(i = n-1; i >= 0; i--){
1379                 if(bits & (1<<i))
1380                         r |= Mdd;
1381                 else
1382                         r &= ~Mdd;
1383                 csr32w(ctlr, Ctrl, Mdc|r);
1384                 csr32w(ctlr, Ctrl, r);
1385         }
1386         csr32w(ctlr, Ctrl, ctrl);
1387
1388         return 0;
1389 }
1390
1391 static int
1392 i82543miimir(struct mii* mii, int pa, int ra)
1393 {
1394         int data;
1395         struct ctlr *ctlr;
1396
1397         ctlr = mii->ctlr;
1398
1399         /*
1400          * MII Management Interface Read.
1401          *
1402          * Preamble;
1403          * ST+OP+PHYAD+REGAD;
1404          * TA + 16 data bits.
1405          */
1406         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1407         i82543mdiow(ctlr, 0x1800|(pa<<5)|ra, 14);
1408         data = i82543mdior(ctlr, 18);
1409
1410         if(data & 0x10000)
1411                 return -1;
1412
1413         return data & 0xFFFF;
1414 }
1415
1416 static int
1417 i82543miimiw(struct mii* mii, int pa, int ra, int data)
1418 {
1419         struct ctlr *ctlr;
1420
1421         ctlr = mii->ctlr;
1422
1423         /*
1424          * MII Management Interface Write.
1425          *
1426          * Preamble;
1427          * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1428          * Z.
1429          */
1430         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1431         data &= 0xFFFF;
1432         data |= (0x05<<(5+5+2+16))|(pa<<(5+2+16))|(ra<<(2+16))|(0x02<<16);
1433         i82543mdiow(ctlr, data, 32);
1434
1435         return 0;
1436 }
1437
1438 static int
1439 igbemiimir(struct mii* mii, int pa, int ra)
1440 {
1441         struct ctlr *ctlr;
1442         int mdic, timo;
1443
1444         ctlr = mii->ctlr;
1445
1446         csr32w(ctlr, Mdic, MDIrop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT));
1447         mdic = 0;
1448         for(timo = 64; timo; timo--){
1449                 mdic = csr32r(ctlr, Mdic);
1450                 if(mdic & (MDIe|MDIready))
1451                         break;
1452                 udelay(1);
1453         }
1454
1455         if((mdic & (MDIe|MDIready)) == MDIready)
1456                 return mdic & 0xFFFF;
1457         return -1;
1458 }
1459
1460 static int
1461 igbemiimiw(struct mii* mii, int pa, int ra, int data)
1462 {
1463         struct ctlr *ctlr;
1464         int mdic, timo;
1465
1466         ctlr = mii->ctlr;
1467
1468         data &= MDIdMASK;
1469         csr32w(ctlr, Mdic, MDIwop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT)|data);
1470         mdic = 0;
1471         for(timo = 64; timo; timo--){
1472                 mdic = csr32r(ctlr, Mdic);
1473                 if(mdic & (MDIe|MDIready))
1474                         break;
1475                 udelay(1);
1476         }
1477         if((mdic & (MDIe|MDIready)) == MDIready)
1478                 return 0;
1479         return -1;
1480 }
1481
1482 static int
1483 i82543miirw(struct mii* mii, int write, int pa, int ra, int data)
1484 {
1485         if(write)
1486                 return i82543miimiw(mii, pa, ra, data);
1487
1488         return i82543miimir(mii, pa, ra);
1489 }
1490
1491 static int
1492 igbemiirw(struct mii* mii, int write, int pa, int ra, int data)
1493 {
1494         if(write)
1495                 return igbemiimiw(mii, pa, ra, data);
1496
1497         return igbemiimir(mii, pa, ra);
1498 }
1499
1500 static int
1501 igbemii(struct ctlr* ctlr)
1502 {
1503         int ctrl, p, r;
1504         int (*rw)(struct mii*, int unused_int, int, int, int);
1505
1506         r = csr32r(ctlr, Status);
1507         if(r & Tbimode)
1508                 return -1;
1509
1510         ctrl = csr32r(ctlr, Ctrl);
1511         ctrl |= Slu;
1512
1513         switch(ctlr->id){
1514         case i82543gc:
1515                 ctrl |= Frcdplx|Frcspd;
1516                 csr32w(ctlr, Ctrl, ctrl);
1517
1518                 /*
1519                  * The reset pin direction (Mdro) should already
1520                  * be set from the EEPROM load.
1521                  * If it's not set this configuration is unexpected
1522                  * so bail.
1523                  */
1524                 r = csr32r(ctlr, Ctrlext);
1525                 if(!(r & Mdro))
1526                         return -1;
1527                 csr32w(ctlr, Ctrlext, r);
1528                 udelay(20*1000);
1529                 r = csr32r(ctlr, Ctrlext);
1530                 r &= ~Mdr;
1531                 csr32w(ctlr, Ctrlext, r);
1532                 udelay(20*1000);
1533                 r = csr32r(ctlr, Ctrlext);
1534                 r |= Mdr;
1535                 csr32w(ctlr, Ctrlext, r);
1536                 udelay(20*1000);
1537
1538                 rw = i82543miirw;
1539                 break;
1540         case i82544ei:
1541         case i82544eif:
1542         case i82544gc:
1543         case i82540em:
1544         case i82540eplp:
1545         case i82547ei:
1546         case i82547gi:
1547         case i82541ei:
1548         case i82541gi:
1549         case i82541gi2:
1550         case i82541pi:
1551         case i82545em:
1552         case i82545gmc:
1553         case i82546gb:
1554         case i82546eb:
1555                 ctrl &= ~(Frcdplx|Frcspd);
1556                 csr32w(ctlr, Ctrl, ctrl);
1557                 rw = igbemiirw;
1558                 break;
1559         default:
1560                 return -1;
1561         }
1562
1563         if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1564                 return -1;
1565         // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1566
1567         /*
1568          * 8254X-specific PHY registers not in 802.3:
1569          *      0x10    PHY specific control
1570          *      0x14    extended PHY specific control
1571          * Set appropriate values then reset the PHY to have
1572          * changes noted.
1573          */
1574         switch(ctlr->id){
1575         case i82547gi:
1576         case i82541gi:
1577         case i82541gi2:
1578         case i82541pi:
1579         case i82545em:
1580         case i82545gmc:
1581         case i82546gb:
1582         case i82546eb:
1583                 break;
1584         default:
1585                 r = miimir(ctlr->mii, 16);
1586                 r |= 0x0800;                    /* assert CRS on Tx */
1587                 r |= 0x0060;                    /* auto-crossover all speeds */
1588                 r |= 0x0002;                    /* polarity reversal enabled */
1589                 miimiw(ctlr->mii, 16, r);
1590
1591                 r = miimir(ctlr->mii, 20);
1592                 r |= 0x0070;                    /* +25MHz clock */
1593                 r &= ~0x0F00;
1594                 r |= 0x0100;                    /* 1x downshift */
1595                 miimiw(ctlr->mii, 20, r);
1596
1597                 miireset(ctlr->mii);
1598                 p = 0;
1599                 if(ctlr->txcw & TxcwPs)
1600                         p |= AnaP;
1601                 if(ctlr->txcw & TxcwAs)
1602                         p |= AnaAP;
1603                 miiane(ctlr->mii, ~0, p, ~0);
1604                 break;
1605         }
1606         return 0;
1607 }
1608
1609 static int
1610 at93c46io(struct ctlr* ctlr, char* op, int data)
1611 {
1612         char *lp, *p;
1613         int i, loop, eecd, r;
1614
1615         eecd = csr32r(ctlr, Eecd);
1616
1617         r = 0;
1618         loop = -1;
1619         lp = NULL;
1620         for(p = op; *p != '\0'; p++){
1621                 switch(*p){
1622                 default:
1623                         return -1;
1624                 case ' ':
1625                         continue;
1626                 case ':':                       /* start of loop */
1627                         loop = strtol(p+1, &lp, 0)-1;
1628                         lp--;
1629                         if(p == lp)
1630                                 loop = 7;
1631                         p = lp;
1632                         continue;
1633                 case ';':                       /* end of loop */
1634                         if(lp == NULL)
1635                                 return -1;
1636                         loop--;
1637                         if(loop >= 0)
1638                                 p = lp;
1639                         else
1640                                 lp = NULL;
1641                         continue;
1642                 case 'C':                       /* assert clock */
1643                         eecd |= Sk;
1644                         break;
1645                 case 'c':                       /* deassert clock */
1646                         eecd &= ~Sk;
1647                         break;
1648                 case 'D':                       /* next bit in 'data' byte */
1649                         if(loop < 0)
1650                                 return -1;
1651                         if(data & (1<<loop))
1652                                 eecd |= Di;
1653                         else
1654                                 eecd &= ~Di;
1655                         break;
1656                 case 'O':                       /* collect data output */
1657                         i = (csr32r(ctlr, Eecd) & Do) != 0;
1658                         if(loop >= 0)
1659                                 r |= (i<<loop);
1660                         else
1661                                 r = i;
1662                         continue;
1663                 case 'I':                       /* assert data input */
1664                         eecd |= Di;
1665                         break;
1666                 case 'i':                       /* deassert data input */
1667                         eecd &= ~Di;
1668                         break;
1669                 case 'S':                       /* enable chip select */
1670                         eecd |= Cs;
1671                         break;
1672                 case 's':                       /* disable chip select */
1673                         eecd &= ~Cs;
1674                         break;
1675                 }
1676                 csr32w(ctlr, Eecd, eecd);
1677                 udelay(50);
1678         }
1679         if(loop >= 0)
1680                 return -1;
1681         return r;
1682 }
1683
1684 static int
1685 at93c46r(struct ctlr* ctlr)
1686 {
1687         uint16_t sum;
1688         char rop[20];
1689         int addr, areq, bits, data, eecd, i;
1690
1691         eecd = csr32r(ctlr, Eecd);
1692         if(eecd & Spi){
1693                 printd("igbe: SPI EEPROM access not implemented\n");
1694                 return 0;
1695         }
1696         if(eecd & (Eeszaddr|Eesz256))
1697                 bits = 8;
1698         else
1699                 bits = 6;
1700
1701         sum = 0;
1702
1703         switch(ctlr->id){
1704         default:
1705                 areq = 0;
1706                 break;
1707         case i82540em:
1708         case i82540eplp:
1709         case i82541ei:
1710         case i82541gi:
1711         case i82541gi2:
1712         case i82541pi:
1713         case i82545em:
1714         case i82545gmc:
1715         case i82546gb:
1716         case i82546eb:
1717         case i82547ei:
1718         case i82547gi:
1719                 areq = 1;
1720                 csr32w(ctlr, Eecd, eecd|Areq);
1721                 for(i = 0; i < 1000; i++){
1722                         if((eecd = csr32r(ctlr, Eecd)) & Agnt)
1723                                 break;
1724                         udelay(5);
1725                 }
1726                 if(!(eecd & Agnt)){
1727                         printd("igbe: not granted EEPROM access\n");
1728                         goto release;
1729                 }
1730                 break;
1731         }
1732         snprintf(rop, sizeof(rop), "S :%dDCc;", bits+3);
1733
1734         for(addr = 0; addr < 0x40; addr++){
1735                 /*
1736                  * Read a word at address 'addr' from the Atmel AT93C46
1737                  * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1738                  * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1739                  * for protocol details.
1740                  */
1741                 if(at93c46io(ctlr, rop, (0x06<<bits)|addr) != 0){
1742                         printd("igbe: can't set EEPROM address 0x%2.2X\n", addr);
1743                         goto release;
1744                 }
1745                 data = at93c46io(ctlr, ":16COc;", 0);
1746                 at93c46io(ctlr, "sic", 0);
1747                 ctlr->eeprom[addr] = data;
1748                 sum += data;
1749         }
1750
1751 release:
1752         if(areq)
1753                 csr32w(ctlr, Eecd, eecd & ~Areq);
1754         return sum;
1755 }
1756
1757 static int
1758 igbedetach(struct ctlr* ctlr)
1759 {
1760         int r, timeo;
1761
1762         /*
1763          * Perform a device reset to get the chip back to the
1764          * power-on state, followed by an EEPROM reset to read
1765          * the defaults for some internal registers.
1766          */
1767         csr32w(ctlr, Imc, ~0);
1768         csr32w(ctlr, Rctl, 0);
1769         csr32w(ctlr, Tctl, 0);
1770
1771         udelay(10*1000);
1772
1773         csr32w(ctlr, Ctrl, Devrst);
1774         udelay(1*1000);
1775         for(timeo = 0; timeo < 1000; timeo++){
1776                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1777                         break;
1778                 udelay(1*1000);
1779         }
1780         if(csr32r(ctlr, Ctrl) & Devrst)
1781                 return -1;
1782         r = csr32r(ctlr, Ctrlext);
1783         csr32w(ctlr, Ctrlext, r|Eerst);
1784         udelay(1*1000);
1785
1786         for(timeo = 0; timeo < 1000; timeo++){
1787                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1788                         break;
1789                 udelay(1*1000);
1790         }
1791         if(csr32r(ctlr, Ctrlext) & Eerst)
1792                 return -1;
1793
1794         switch(ctlr->id){
1795         default:
1796                 break;
1797         case i82540em:
1798         case i82540eplp:
1799         case i82541gi:
1800         case i82541gi2:
1801         case i82541pi:
1802         case i82545em:
1803         case i82545gmc:
1804         case i82547gi:
1805         case i82546gb:
1806         case i82546eb:
1807                 r = csr32r(ctlr, Manc);
1808                 r &= ~Arpen;
1809                 csr32w(ctlr, Manc, r);
1810                 break;
1811         }
1812
1813         csr32w(ctlr, Imc, ~0);
1814         udelay(1*1000);
1815         for(timeo = 0; timeo < 1000; timeo++){
1816                 if(!csr32r(ctlr, Icr))
1817                         break;
1818                 udelay(1*1000);
1819         }
1820         if(csr32r(ctlr, Icr))
1821                 return -1;
1822
1823         return 0;
1824 }
1825
1826 static void
1827 igbeshutdown(struct ether* ether)
1828 {
1829         igbedetach(ether->ctlr);
1830 }
1831
1832 static int
1833 igbereset(struct ctlr* ctlr)
1834 {
1835         int ctrl, i, pause, r, swdpio, txcw;
1836
1837         if(igbedetach(ctlr))
1838                 return -1;
1839
1840         /*
1841          * Read the EEPROM, validate the checksum
1842          * then get the device back to a power-on state.
1843          */
1844         if((r = at93c46r(ctlr)) != 0xBABA){
1845                 printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1846                 return -1;
1847         }
1848
1849         /*
1850          * Snarf and set up the receive addresses.
1851          * There are 16 addresses. The first should be the MAC address.
1852          * The others are cleared and not marked valid (MS bit of Rah).
1853          */
1854         if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1855             (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1856                  pci_config_addr(0, 1, 0, 0)))
1857                 ctlr->eeprom[Ea+2] += 0x100;            /* second interface */
1858         if(ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1859                 ctlr->eeprom[Ea] = 0xD000;
1860         for(i = Ea; i < Eaddrlen/2; i++){
1861                 ctlr->ra[2*i] = ctlr->eeprom[i];
1862                 ctlr->ra[2*i+1] = ctlr->eeprom[i]>>8;
1863         }
1864         /* lan id seems to vary on 82543gc; don't use it */
1865         if (ctlr->id != i82543gc) {
1866                 r = (csr32r(ctlr, Status) & Lanid) >> 2;
1867                 ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1868         }
1869
1870         r = (ctlr->ra[3]<<24)|(ctlr->ra[2]<<16)|(ctlr->ra[1]<<8)|ctlr->ra[0];
1871         csr32w(ctlr, Ral, r);
1872         r = 0x80000000|(ctlr->ra[5]<<8)|ctlr->ra[4];
1873         csr32w(ctlr, Rah, r);
1874         for(i = 1; i < 16; i++){
1875                 csr32w(ctlr, Ral+i*8, 0);
1876                 csr32w(ctlr, Rah+i*8, 0);
1877         }
1878
1879         /*
1880          * Clear the Multicast Table Array.
1881          * It's a 4096 bit vector accessed as 128 32-bit registers.
1882          */
1883         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1884         for(i = 0; i < 128; i++)
1885                 csr32w(ctlr, Mta+i*4, 0);
1886
1887         /*
1888          * Just in case the Eerst didn't load the defaults
1889          * (doesn't appear to fully on the 82543GC), do it manually.
1890          */
1891         if (ctlr->id == i82543gc) {
1892                 txcw = csr32r(ctlr, Txcw);
1893                 txcw &= ~(TxcwAne|TxcwPauseMASK|TxcwFd);
1894                 ctrl = csr32r(ctlr, Ctrl);
1895                 ctrl &= ~(SwdpioloMASK|Frcspd|Ilos|Lrst|Fd);
1896
1897                 if(ctlr->eeprom[Icw1] & 0x0400){
1898                         ctrl |= Fd;
1899                         txcw |= TxcwFd;
1900                 }
1901                 if(ctlr->eeprom[Icw1] & 0x0200)
1902                         ctrl |= Lrst;
1903                 if(ctlr->eeprom[Icw1] & 0x0010)
1904                         ctrl |= Ilos;
1905                 if(ctlr->eeprom[Icw1] & 0x0800)
1906                         ctrl |= Frcspd;
1907                 swdpio = (ctlr->eeprom[Icw1] & 0x01E0)>>5;
1908                 ctrl |= swdpio<<SwdpioloSHIFT;
1909                 csr32w(ctlr, Ctrl, ctrl);
1910
1911                 ctrl = csr32r(ctlr, Ctrlext);
1912                 ctrl &= ~(Ips|SwdpiohiMASK);
1913                 swdpio = (ctlr->eeprom[Icw2] & 0x00F0)>>4;
1914                 if(ctlr->eeprom[Icw1] & 0x1000)
1915                         ctrl |= Ips;
1916                 ctrl |= swdpio<<SwdpiohiSHIFT;
1917                 csr32w(ctlr, Ctrlext, ctrl);
1918
1919                 if(ctlr->eeprom[Icw2] & 0x0800)
1920                         txcw |= TxcwAne;
1921                 pause = (ctlr->eeprom[Icw2] & 0x3000)>>12;
1922                 txcw |= pause<<TxcwPauseSHIFT;
1923                 switch(pause){
1924                 default:
1925                         ctlr->fcrtl = 0x00002000;
1926                         ctlr->fcrth = 0x00004000;
1927                         txcw |= TxcwAs|TxcwPs;
1928                         break;
1929                 case 0:
1930                         ctlr->fcrtl = 0x00002000;
1931                         ctlr->fcrth = 0x00004000;
1932                         break;
1933                 case 2:
1934                         ctlr->fcrtl = 0;
1935                         ctlr->fcrth = 0;
1936                         txcw |= TxcwAs;
1937                         break;
1938                 }
1939                 ctlr->txcw = txcw;
1940                 csr32w(ctlr, Txcw, txcw);
1941         }
1942
1943
1944         /*
1945          * Flow control - values from the datasheet.
1946          */
1947         csr32w(ctlr, Fcal, 0x00C28001);
1948         csr32w(ctlr, Fcah, 0x00000100);
1949         csr32w(ctlr, Fct, 0x00008808);
1950         csr32w(ctlr, Fcttv, 0x00000100);
1951
1952         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1953         csr32w(ctlr, Fcrth, ctlr->fcrth);
1954
1955         /* FYI, igbemii checks status right away too. */
1956         if(!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1957                 printk("igbemii failed!  igbe failing to reset!\n");
1958                 return -1;
1959         }
1960
1961         return 0;
1962 }
1963
1964 static void
1965 igbepci(void)
1966 {
1967         int cls, id;
1968         struct pci_device *pcidev;
1969         struct ctlr *ctlr;
1970         void *mem;
1971         uintptr_t mmio_paddr;
1972
1973         STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
1974                 /* This checks that pcidev is a Network Controller for Ethernet */
1975                 if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1976                         continue;
1977                 id = pcidev->dev_id << 16 | pcidev->ven_id;
1978                 switch (id) {
1979                 default:
1980                         continue;
1981                 case i82543gc:
1982                 case i82544ei:
1983                 case i82544eif:
1984                 case i82544gc:
1985                 case i82547ei:
1986                 case i82547gi:
1987                 case i82540em:
1988                 case i82540eplp:
1989                 case i82541ei:
1990                 case i82541gi:
1991                 case i82541gi2:
1992                 case i82541pi:
1993                 case i82545em:
1994                 case i82545gmc:
1995                 case i82546gb:
1996                 case i82546eb:
1997                         break;
1998                 }
1999                 printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
2000                        pcidev->ven_id, pcidev->dev_id,
2001                        pcidev->bus, pcidev->dev, pcidev->func);
2002
2003                 mmio_paddr = pcidev->bar[0].mmio_base32 ? pcidev->bar[0].mmio_base32 : 
2004                                                           pcidev->bar[0].mmio_base64;
2005                 mem = (void*)vmap_pmem(mmio_paddr, pcidev->bar[0].mmio_sz);
2006                 if(mem == NULL){
2007                         printd("igbe: can't map %p\n", pcidev->bar[0].mmio_base32);
2008                         continue;
2009                 }
2010                 cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
2011                 switch(cls){
2012                         default:
2013                                 printd("igbe: unexpected CLS - %d\n", cls*4);
2014                                 break;
2015                         case 0x00:
2016                         case 0xFF:
2017                                 /* bogus value; use a sane default.  cls is set in DWORD (u32)
2018                                  * units. */
2019                                 cls = ARCH_CL_SIZE / sizeof(long);
2020                                 pcidev_write8(pcidev, PCI_CLSZ_REG, cls);
2021                                 break;
2022                         case 0x08:
2023                         case 0x10:
2024                                 break;
2025                 }
2026                 ctlr = kzmalloc(sizeof(struct ctlr), 0);
2027                 if(ctlr == NULL) {
2028                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2029                         error(Enomem);
2030                 }
2031                 spinlock_init_irqsave(&ctlr->imlock);
2032                 spinlock_init_irqsave(&ctlr->tlock);
2033                 qlock_init(&ctlr->alock);
2034                 qlock_init(&ctlr->slock);
2035                 rendez_init(&ctlr->lrendez);
2036                 rendez_init(&ctlr->rrendez);
2037                 /* port seems to be unused, and only used for some comparison with edev.
2038                  * plan9 just used the top of the raw bar, regardless of the type. */
2039                 ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
2040                 ctlr->pci = pcidev;
2041                 ctlr->id = id;
2042                 ctlr->cls = cls * sizeof(long);
2043                 ctlr->nic = mem;
2044
2045                 if(igbereset(ctlr)){
2046                         kfree(ctlr);
2047                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2048                         continue;
2049                 }
2050                 pci_set_bus_master(pcidev);
2051
2052                 if(igbectlrhead != NULL)
2053                         igbectlrtail->next = ctlr;
2054                 else
2055                         igbectlrhead = ctlr;
2056                 igbectlrtail = ctlr;
2057         }
2058 }
2059
2060 static int
2061 igbepnp(struct ether* edev)
2062 {
2063         struct ctlr *ctlr;
2064
2065         run_once(igbepci());
2066
2067         /*
2068          * Any adapter matches if no edev->port is supplied,
2069          * otherwise the ports must match.
2070          */
2071         for(ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next){
2072                 if(ctlr->active)
2073                         continue;
2074                 if(edev->port == 0 || edev->port == ctlr->port){
2075                         ctlr->active = 1;
2076                         break;
2077                 }
2078         }
2079         if(ctlr == NULL)
2080                 return -1;
2081
2082         edev->ctlr = ctlr;
2083         edev->port = ctlr->port;
2084         edev->irq = ctlr->pci->irqline;
2085         edev->netif.mbps = 1000;
2086         memmove(edev->ea, ctlr->ra, Eaddrlen);
2087
2088         /*
2089          * Linkage to the generic ethernet driver.
2090          */
2091         edev->tbdf = MKBUS(BusPCI, ctlr->pci->bus, ctlr->pci->dev,
2092                            ctlr->pci->func);
2093         edev->attach = igbeattach;
2094         edev->transmit = igbetransmit;
2095         edev->interrupt = igbeinterrupt;
2096         edev->ifstat = igbeifstat;
2097         edev->ctl = igbectl;
2098         edev->shutdown = igbeshutdown;
2099
2100         edev->netif.arg = edev;
2101         edev->netif.promiscuous = igbepromiscuous;
2102         edev->netif.multicast = igbemulticast;
2103
2104         return 0;
2105 }
2106
2107 linker_func_3(etherigbelink)
2108 {
2109         addethercard("i82543", igbepnp);
2110         addethercard("igbe", igbepnp);
2111 }