Fixes unchecked results from parsecmd
[akaros.git] / kern / drivers / net / etherigbe.c
1 /* This file is part of the UCB release of Plan 9. It is subject to the license
2  * terms in the LICENSE file found in the top-level directory of this
3  * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
4  * part of the UCB release of Plan 9, including this file, may be copied,
5  * modified, propagated, or distributed except according to the terms contained
6  * in the LICENSE file. */
7
8 /*
9  * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
10  * as found on the Intel PRO/1000 series of adapters:
11  *      82543GC Intel PRO/1000 T
12  *      82544EI Intel PRO/1000 XT
13  *      82540EM Intel PRO/1000 MT
14  *      82541[GP]I
15  *      82547GI
16  *      82546GB
17  *      82546EB
18  * To Do:
19  *      finish autonegotiation code;
20  *      integrate fiber stuff back in (this ONLY handles
21  *      the CAT5 cards at the moment);
22  *      add checksum-offload;
23  *      add tuning control via ctl file;
24  *      this driver is little-endian specific.
25  *
26  * Modified by brho:
27  *      ported to Akaros
28  *      fixed mii bugs (allocation, startup, miirw, etc)
29  *      fixed CLS bug (continue -> break)
30  *      made sure igbepci only runs once, even if it fails */
31
32 #include <vfs.h>
33 #include <kfs.h>
34 #include <slab.h>
35 #include <kmalloc.h>
36 #include <kref.h>
37 #include <string.h>
38 #include <stdio.h>
39 #include <assert.h>
40 #include <error.h>
41 #include <cpio.h>
42 #include <pmap.h>
43 #include <smp.h>
44 #include <arch/pci.h>
45 #include <ip.h>
46 #include <ns.h>
47 #include "ethermii.h"
48
49 #define ilock(x) spin_lock_irqsave(x)
50 #define iunlock(x) spin_unlock_irqsave(x)
51
52 enum {
53         i82542          = (0x1000<<16)|0x8086,
54         i82543gc        = (0x1004<<16)|0x8086,
55         i82544ei        = (0x1008<<16)|0x8086,
56         i82544eif       = (0x1009<<16)|0x8086,
57         i82544gc        = (0x100d<<16)|0x8086,
58         i82540em        = (0x100E<<16)|0x8086,
59         i82540eplp      = (0x101E<<16)|0x8086,
60         i82545em        = (0x100F<<16)|0x8086,
61         i82545gmc       = (0x1026<<16)|0x8086,
62         i82547ei        = (0x1019<<16)|0x8086,
63         i82547gi        = (0x1075<<16)|0x8086,
64         i82541ei        = (0x1013<<16)|0x8086,
65         i82541gi        = (0x1076<<16)|0x8086,
66         i82541gi2       = (0x1077<<16)|0x8086,
67         i82541pi        = (0x107c<<16)|0x8086,
68         i82546gb        = (0x1079<<16)|0x8086,
69         i82546eb        = (0x1010<<16)|0x8086,
70 };
71
72 enum {
73         Ctrl            = 0x00000000,   /* Device Control */
74         Ctrldup         = 0x00000004,   /* Device Control Duplicate */
75         Status          = 0x00000008,   /* Device Status */
76         Eecd            = 0x00000010,   /* EEPROM/Flash Control/Data */
77         Ctrlext         = 0x00000018,   /* Extended Device Control */
78         Mdic            = 0x00000020,   /* MDI Control */
79         Fcal            = 0x00000028,   /* Flow Control Address Low */
80         Fcah            = 0x0000002C,   /* Flow Control Address High */
81         Fct             = 0x00000030,   /* Flow Control Type */
82         Icr             = 0x000000C0,   /* Interrupt Cause Read */
83         Ics             = 0x000000C8,   /* Interrupt Cause Set */
84         Ims             = 0x000000D0,   /* Interrupt Mask Set/Read */
85         Imc             = 0x000000D8,   /* Interrupt mask Clear */
86         Rctl            = 0x00000100,   /* Receive Control */
87         Fcttv           = 0x00000170,   /* Flow Control Transmit Timer Value */
88         Txcw            = 0x00000178,   /* Transmit Configuration Word */
89         Rxcw            = 0x00000180,   /* Receive Configuration Word */
90         /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
91         Tctl            = 0x00000400,   /* Transmit Control */
92         Tipg            = 0x00000410,   /* Transmit IPG */
93         Tbt             = 0x00000448,   /* Transmit Burst Timer */
94         Ait             = 0x00000458,   /* Adaptive IFS Throttle */
95         Fcrtl           = 0x00002160,   /* Flow Control RX Threshold Low */
96         Fcrth           = 0x00002168,   /* Flow Control Rx Threshold High */
97         Rdfh            = 0x00002410,   /* Receive data fifo head */
98         Rdft            = 0x00002418,   /* Receive data fifo tail */
99         Rdfhs           = 0x00002420,   /* Receive data fifo head saved */
100         Rdfts           = 0x00002428,   /* Receive data fifo tail saved */
101         Rdfpc           = 0x00002430,   /* Receive data fifo packet count */
102         Rdbal           = 0x00002800,   /* Rd Base Address Low */
103         Rdbah           = 0x00002804,   /* Rd Base Address High */
104         Rdlen           = 0x00002808,   /* Receive Descriptor Length */
105         Rdh             = 0x00002810,   /* Receive Descriptor Head */
106         Rdt             = 0x00002818,   /* Receive Descriptor Tail */
107         Rdtr            = 0x00002820,   /* Receive Descriptor Timer Ring */
108         Rxdctl          = 0x00002828,   /* Receive Descriptor Control */
109         Radv            = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
110         Txdmac          = 0x00003000,   /* Transfer DMA Control */
111         Ett             = 0x00003008,   /* Early Transmit Control */
112         Tdfh            = 0x00003410,   /* Transmit data fifo head */
113         Tdft            = 0x00003418,   /* Transmit data fifo tail */
114         Tdfhs           = 0x00003420,   /* Transmit data Fifo Head saved */
115         Tdfts           = 0x00003428,   /* Transmit data fifo tail saved */
116         Tdfpc           = 0x00003430,   /* Trasnmit data Fifo packet count */
117         Tdbal           = 0x00003800,   /* Td Base Address Low */
118         Tdbah           = 0x00003804,   /* Td Base Address High */
119         Tdlen           = 0x00003808,   /* Transmit Descriptor Length */
120         Tdh             = 0x00003810,   /* Transmit Descriptor Head */
121         Tdt             = 0x00003818,   /* Transmit Descriptor Tail */
122         Tidv            = 0x00003820,   /* Transmit Interrupt Delay Value */
123         Txdctl          = 0x00003828,   /* Transmit Descriptor Control */
124         Tadv            = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
125
126         Statistics      = 0x00004000,   /* Start of Statistics Area */
127         Gorcl           = 0x88/4,       /* Good Octets Received Count */
128         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
129         Torl            = 0xC0/4,       /* Total Octets Received */
130         Totl            = 0xC8/4,       /* Total Octets Transmitted */
131         Nstatistics     = 64,
132
133         Rxcsum          = 0x00005000,   /* Receive Checksum Control */
134         Mta             = 0x00005200,   /* Multicast Table Array */
135         Ral             = 0x00005400,   /* Receive Address Low */
136         Rah             = 0x00005404,   /* Receive Address High */
137         Manc            = 0x00005820,   /* Management Control */
138 };
139
140 enum {                                  /* Ctrl */
141         Bem             = 0x00000002,   /* Big Endian Mode */
142         Prior           = 0x00000004,   /* Priority on the PCI bus */
143         Lrst            = 0x00000008,   /* Link Reset */
144         Asde            = 0x00000020,   /* Auto-Speed Detection Enable */
145         Slu             = 0x00000040,   /* Set Link Up */
146         Ilos            = 0x00000080,   /* Invert Loss of Signal (LOS) */
147         SspeedMASK      = 0x00000300,   /* Speed Selection */
148         SspeedSHIFT     = 8,
149         Sspeed10        = 0x00000000,   /* 10Mb/s */
150         Sspeed100       = 0x00000100,   /* 100Mb/s */
151         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
152         Frcspd          = 0x00000800,   /* Force Speed */
153         Frcdplx         = 0x00001000,   /* Force Duplex */
154         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
155         SwdpinsloSHIFT  = 18,
156         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
157         SwdpioloSHIFT   = 22,
158         Devrst          = 0x04000000,   /* Device Reset */
159         Rfce            = 0x08000000,   /* Receive Flow Control Enable */
160         Tfce            = 0x10000000,   /* Transmit Flow Control Enable */
161         Vme             = 0x40000000,   /* VLAN Mode Enable */
162 };
163
164 /*
165  * can't find Tckok nor Rbcok in any Intel docs,
166  * but even 82543gc docs define Lanid.
167  */
168 enum {                                  /* Status */
169         Lu              = 0x00000002,   /* Link Up */
170         Lanid           = 0x0000000C,   /* mask for Lan ID. (function id) */
171 //      Tckok           = 0x00000004,   /* Transmit clock is running */
172 //      Rbcok           = 0x00000008,   /* Receive clock is running */
173         Txoff           = 0x00000010,   /* Transmission Paused */
174         Tbimode         = 0x00000020,   /* TBI Mode Indication */
175         LspeedMASK      = 0x000000C0,   /* Link Speed Setting */
176         LspeedSHIFT     = 6,
177         Lspeed10        = 0x00000000,   /* 10Mb/s */
178         Lspeed100       = 0x00000040,   /* 100Mb/s */
179         Lspeed1000      = 0x00000080,   /* 1000Mb/s */
180         Mtxckok         = 0x00000400,   /* MTX clock is running */
181         Pci66           = 0x00000800,   /* PCI Bus speed indication */
182         Bus64           = 0x00001000,   /* PCI Bus width indication */
183         Pcixmode        = 0x00002000,   /* PCI-X mode */
184         PcixspeedMASK   = 0x0000C000,   /* PCI-X bus speed */
185         PcixspeedSHIFT  = 14,
186         Pcix66          = 0x00000000,   /* 50-66MHz */
187         Pcix100         = 0x00004000,   /* 66-100MHz */
188         Pcix133         = 0x00008000,   /* 100-133MHz */
189 };
190
191 enum {                                  /* Ctrl and Status */
192         Fd              = 0x00000001,   /* Full-Duplex */
193         AsdvMASK        = 0x00000300,
194         AsdvSHIFT       = 8,
195         Asdv10          = 0x00000000,   /* 10Mb/s */
196         Asdv100         = 0x00000100,   /* 100Mb/s */
197         Asdv1000        = 0x00000200,   /* 1000Mb/s */
198 };
199
200 enum {                                  /* Eecd */
201         Sk              = 0x00000001,   /* Clock input to the EEPROM */
202         Cs              = 0x00000002,   /* Chip Select */
203         Di              = 0x00000004,   /* Data Input to the EEPROM */
204         Do              = 0x00000008,   /* Data Output from the EEPROM */
205         Areq            = 0x00000040,   /* EEPROM Access Request */
206         Agnt            = 0x00000080,   /* EEPROM Access Grant */
207         Eepresent       = 0x00000100,   /* EEPROM Present */
208         Eesz256         = 0x00000200,   /* EEPROM is 256 words not 64 */
209         Eeszaddr        = 0x00000400,   /* EEPROM size for 8254[17] */
210         Spi             = 0x00002000,   /* EEPROM is SPI not Microwire */
211 };
212
213 enum {                                  /* Ctrlext */
214         Gpien           = 0x0000000F,   /* General Purpose Interrupt Enables */
215         SwdpinshiMASK   = 0x000000F0,   /* Software Defined Pins - hi nibble */
216         SwdpinshiSHIFT  = 4,
217         SwdpiohiMASK    = 0x00000F00,   /* Software Defined Pins - I or O */
218         SwdpiohiSHIFT   = 8,
219         Asdchk          = 0x00001000,   /* ASD Check */
220         Eerst           = 0x00002000,   /* EEPROM Reset */
221         Ips             = 0x00004000,   /* Invert Power State */
222         Spdbyps         = 0x00008000,   /* Speed Select Bypass */
223 };
224
225 enum {                                  /* EEPROM content offsets */
226         Ea              = 0x00,         /* Ethernet Address */
227         Cf              = 0x03,         /* Compatibility Field */
228         Pba             = 0x08,         /* Printed Board Assembly number */
229         Icw1            = 0x0A,         /* Initialization Control Word 1 */
230         Sid             = 0x0B,         /* Subsystem ID */
231         Svid            = 0x0C,         /* Subsystem Vendor ID */
232         Did             = 0x0D,         /* Device ID */
233         Vid             = 0x0E,         /* Vendor ID */
234         Icw2            = 0x0F,         /* Initialization Control Word 2 */
235 };
236
237 enum {                                  /* Mdic */
238         MDIdMASK        = 0x0000FFFF,   /* Data */
239         MDIdSHIFT       = 0,
240         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
241         MDIrSHIFT       = 16,
242         MDIpMASK        = 0x03E00000,   /* PHY Address */
243         MDIpSHIFT       = 21,
244         MDIwop          = 0x04000000,   /* Write Operation */
245         MDIrop          = 0x08000000,   /* Read Operation */
246         MDIready        = 0x10000000,   /* End of Transaction */
247         MDIie           = 0x20000000,   /* Interrupt Enable */
248         MDIe            = 0x40000000,   /* Error */
249 };
250
251 enum {                                  /* Icr, Ics, Ims, Imc */
252         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
253         Txqe            = 0x00000002,   /* Transmit Queue Empty */
254         Lsc             = 0x00000004,   /* Link Status Change */
255         Rxseq           = 0x00000008,   /* Receive Sequence Error */
256         Rxdmt0          = 0x00000010,   /* Rd Minimum Threshold Reached */
257         Rxo             = 0x00000040,   /* Receiver Overrun */
258         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
259         Mdac            = 0x00000200,   /* MDIO Access Completed */
260         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
261         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
262         Gpi1            = 0x00001000,
263         Gpi2            = 0x00002000,
264         Gpi3            = 0x00004000,
265 };
266
267 /*
268  * The Mdic register isn't implemented on the 82543GC,
269  * the software defined pins are used instead.
270  * These definitions work for the Intel PRO/1000 T Server Adapter.
271  * The direction pin bits are read from the EEPROM.
272  */
273 enum {
274         Mdd             = ((1<<2)<<SwdpinsloSHIFT),     /* data */
275         Mddo            = ((1<<2)<<SwdpioloSHIFT),      /* pin direction */
276         Mdc             = ((1<<3)<<SwdpinsloSHIFT),     /* clock */
277         Mdco            = ((1<<3)<<SwdpioloSHIFT),      /* pin direction */
278         Mdr             = ((1<<0)<<SwdpinshiSHIFT),     /* reset */
279         Mdro            = ((1<<0)<<SwdpiohiSHIFT),      /* pin direction */
280 };
281
282 enum {                                  /* Txcw */
283         TxcwFd          = 0x00000020,   /* Full Duplex */
284         TxcwHd          = 0x00000040,   /* Half Duplex */
285         TxcwPauseMASK   = 0x00000180,   /* Pause */
286         TxcwPauseSHIFT  = 7,
287         TxcwPs          = (1<<TxcwPauseSHIFT),  /* Pause Supported */
288         TxcwAs          = (2<<TxcwPauseSHIFT),  /* Asymmetric FC desired */
289         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
290         TxcwRfiSHIFT    = 12,
291         TxcwNpr         = 0x00008000,   /* Next Page Request */
292         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
293         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
294 };
295
296 enum {                                  /* Rxcw */
297         Rxword          = 0x0000FFFF,   /* Data from auto-negotiation process */
298         Rxnocarrier     = 0x04000000,   /* Carrier Sense indication */
299         Rxinvalid       = 0x08000000,   /* Invalid Symbol during configuration */
300         Rxchange        = 0x10000000,   /* Change to the Rxword indication */
301         Rxconfig        = 0x20000000,   /* /C/ order set reception indication */
302         Rxsync          = 0x40000000,   /* Lost bit synchronization indication */
303         Anc             = 0x80000000,   /* Auto Negotiation Complete */
304 };
305
306 enum {                                  /* Rctl */
307         Rrst            = 0x00000001,   /* Receiver Software Reset */
308         Ren             = 0x00000002,   /* Receiver Enable */
309         Sbp             = 0x00000004,   /* Store Bad Packets */
310         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
311         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
312         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
313         LbmMASK         = 0x000000C0,   /* Loopback Mode */
314         LbmOFF          = 0x00000000,   /* No Loopback */
315         LbmTBI          = 0x00000040,   /* TBI Loopback */
316         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
317         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
318         RdtmsMASK       = 0x00000300,   /* Rd Minimum Threshold Size */
319         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
320         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
321         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
322         MoMASK          = 0x00003000,   /* Multicast Offset */
323         Mo47b36         = 0x00000000,   /* bits [47:36] of received address */
324         Mo46b35         = 0x00001000,   /* bits [46:35] of received address */
325         Mo45b34         = 0x00002000,   /* bits [45:34] of received address */
326         Mo43b32         = 0x00003000,   /* bits [43:32] of received address */
327         Bam             = 0x00008000,   /* Broadcast Accept Mode */
328         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
329         Bsize2048       = 0x00000000,   /* Bsex = 0 */
330         Bsize1024       = 0x00010000,   /* Bsex = 0 */
331         Bsize512        = 0x00020000,   /* Bsex = 0 */
332         Bsize256        = 0x00030000,   /* Bsex = 0 */
333         Bsize16384      = 0x00010000,   /* Bsex = 1 */
334         Vfe             = 0x00040000,   /* VLAN Filter Enable */
335         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
336         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
337         Dpf             = 0x00400000,   /* Discard Pause Frames */
338         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
339         Bsex            = 0x02000000,   /* Buffer Size Extension */
340         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
341 };
342
343 enum {                                  /* Tctl */
344         Trst            = 0x00000001,   /* Transmitter Software Reset */
345         Ten             = 0x00000002,   /* Transmit Enable */
346         Psp             = 0x00000008,   /* Pad Short Packets */
347         CtMASK          = 0x00000FF0,   /* Collision Threshold */
348         CtSHIFT         = 4,
349         ColdMASK        = 0x003FF000,   /* Collision Distance */
350         ColdSHIFT       = 12,
351         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
352         Pbe             = 0x00800000,   /* Packet Burst Enable */
353         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
354         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
355 };
356
357 enum {                                  /* [RT]xdctl */
358         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
359         PthreshSHIFT    = 0,
360         HthreshMASK     = 0x00003F00,   /* Host Threshold */
361         HthreshSHIFT    = 8,
362         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
363         WthreshSHIFT    = 16,
364         Gran            = 0x01000000,   /* Granularity */
365         LthreshMASK     = 0xFE000000,   /* Low Threshold */
366         LthreshSHIFT    = 25,
367 };
368
369 enum {                                  /* Rxcsum */
370         PcssMASK        = 0x000000FF,   /* Packet Checksum Start */
371         PcssSHIFT       = 0,
372         Ipofl           = 0x00000100,   /* IP Checksum Off-load Enable */
373         Tuofl           = 0x00000200,   /* TCP/UDP Checksum Off-load Enable */
374 };
375
376 enum {                                  /* Manc */
377         Arpen           = 0x00002000,   /* Enable ARP Request Filtering */
378 };
379
380 enum {                                  /* Receive Delay Timer Ring */
381         DelayMASK       = 0x0000FFFF,   /* delay timer in 1.024nS increments */
382         DelaySHIFT      = 0,
383         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
384 };
385
386 typedef struct Rd {                     /* Receive Descriptor */
387         unsigned int    addr[2];
388         uint16_t        length;
389         uint16_t        checksum;
390         uint8_t status;
391         uint8_t errors;
392         uint16_t        special;
393 } Rd;
394
395 enum {                                  /* Rd status */
396         Rdd             = 0x01,         /* Descriptor Done */
397         Reop            = 0x02,         /* End of Packet */
398         Ixsm            = 0x04,         /* Ignore Checksum Indication */
399         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
400         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
401         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
402         Pif             = 0x80,         /* Passed in-exact filter */
403 };
404
405 enum {                                  /* Rd errors */
406         Ce              = 0x01,         /* CRC Error or Alignment Error */
407         Se              = 0x02,         /* Symbol Error */
408         Seq             = 0x04,         /* Sequence Error */
409         Cxe             = 0x10,         /* Carrier Extension Error */
410         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
411         Ipe             = 0x40,         /* IP Checksum Error */
412         Rxe             = 0x80,         /* RX Data Error */
413 };
414
415 typedef struct Td Td;
416 struct Td {                             /* Transmit Descriptor */
417         union {
418                 unsigned int    addr[2];        /* Data */
419                 struct {                /* Context */
420                         uint8_t ipcss;
421                         uint8_t ipcso;
422                         uint16_t        ipcse;
423                         uint8_t tucss;
424                         uint8_t tucso;
425                         uint16_t        tucse;
426                 };
427         };
428         unsigned int    control;
429         unsigned int    status;
430 };
431
432 enum {                                  /* Td control */
433         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
434         LenSHIFT        = 0,
435         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
436         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
437         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
438         Teop            = 0x01000000,   /* End of Packet (DD) */
439         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
440         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
441         Tse             = 0x04000000,   /* TCP Segmentation Enable */
442         Rs              = 0x08000000,   /* Report Status */
443         Rps             = 0x10000000,   /* Report Status Sent */
444         Dext            = 0x20000000,   /* Descriptor Extension */
445         Vle             = 0x40000000,   /* VLAN Packet Enable */
446         Ide             = 0x80000000,   /* Interrupt Delay Enable */
447 };
448
449 enum {                                  /* Td status */
450         Tdd             = 0x00000001,   /* Descriptor Done */
451         Ec              = 0x00000002,   /* Excess Collisions */
452         Lc              = 0x00000004,   /* Late Collision */
453         Tu              = 0x00000008,   /* Transmit Underrun */
454         Iixsm           = 0x00000100,   /* Insert IP Checksum */
455         Itxsm           = 0x00000200,   /* Insert TCP/UDP Checksum */
456         HdrlenMASK      = 0x0000FF00,   /* Header Length (Tse) */
457         HdrlenSHIFT     = 8,
458         VlanMASK        = 0x0FFF0000,   /* VLAN Identifier */
459         VlanSHIFT       = 16,
460         Tcfi            = 0x10000000,   /* Canonical Form Indicator */
461         PriMASK         = 0xE0000000,   /* User Priority */
462         PriSHIFT        = 29,
463         MssMASK         = 0xFFFF0000,   /* Maximum Segment Size (Tse) */
464         MssSHIFT        = 16,
465 };
466
467 enum {
468         Nrd             = 256,          /* multiple of 8 */
469         Ntd             = 64,           /* multiple of 8 */
470         Rbsz            = 2048,
471 };
472
473 struct ctlr {
474         int     port;
475         struct pci_device *pci;
476         struct ctlr*    next;
477         struct ether*   edev;
478         int     active;
479         int     started;
480         int     id;
481         int     cls;
482         uint16_t        eeprom[0x40];
483
484         qlock_t alock;                  /* attach */
485         void*   alloc;                  /* receive/transmit descriptors */
486         int     nrd;
487         int     ntd;
488
489         int*    nic;
490         spinlock_t      imlock;
491         int     im;                     /* interrupt mask */
492
493         struct mii*     mii;
494         struct rendez   lrendez;
495         int     lim;
496
497         int     link;
498
499         qlock_t slock;
500         unsigned int    statistics[Nstatistics];
501         unsigned int    lsleep;
502         unsigned int    lintr;
503         unsigned int    rsleep;
504         unsigned int    rintr;
505         unsigned int    txdw;
506         unsigned int    tintr;
507         unsigned int    ixsm;
508         unsigned int    ipcs;
509         unsigned int    tcpcs;
510
511         uint8_t ra[Eaddrlen];           /* receive address */
512         uint32_t        mta[128];               /* multicast table array */
513
514         struct rendez   rrendez;
515         int     rim;
516         int     rdfree;
517         Rd*     rdba;                   /* receive descriptor base address */
518         struct block**  rb;                     /* receive buffers */
519         int     rdh;                    /* receive descriptor head */
520         int     rdt;                    /* receive descriptor tail */
521         int     rdtr;                   /* receive delay timer ring value */
522
523         spinlock_t      tlock;
524         int     tbusy;
525         int     tdfree;
526         Td*     tdba;                   /* transmit descriptor base address */
527         struct block**  tb;                     /* transmit buffers */
528         int     tdh;                    /* transmit descriptor head */
529         int     tdt;                    /* transmit descriptor tail */
530
531         int     txcw;
532         int     fcrtl;
533         int     fcrth;
534 };
535
536 static inline uint32_t csr32r(struct ctlr *c, uintptr_t reg)
537 {
538         return read_mmreg32((uintptr_t)(c->nic + (reg / 4)));
539 }
540
541 static inline void csr32w(struct ctlr *c, uintptr_t reg, uint32_t val)
542 {
543         write_mmreg32((uintptr_t)(c->nic + (reg / 4)), val);
544 }
545
546 static struct ctlr* igbectlrhead;
547 static struct ctlr* igbectlrtail;
548
549 static char* statistics[Nstatistics] = {
550         "CRC Error",
551         "Alignment Error",
552         "Symbol Error",
553         "RX Error",
554         "Missed Packets",
555         "Single Collision",
556         "Excessive Collisions",
557         "Multiple Collision",
558         "Late Collisions",
559         NULL,
560         "Collision",
561         "Transmit Underrun",
562         "Defer",
563         "Transmit - No CRS",
564         "Sequence Error",
565         "Carrier Extension Error",
566         "Receive Error Length",
567         NULL,
568         "XON Received",
569         "XON Transmitted",
570         "XOFF Received",
571         "XOFF Transmitted",
572         "FC Received Unsupported",
573         "Packets Received (64 Bytes)",
574         "Packets Received (65-127 Bytes)",
575         "Packets Received (128-255 Bytes)",
576         "Packets Received (256-511 Bytes)",
577         "Packets Received (512-1023 Bytes)",
578         "Packets Received (1024-1522 Bytes)",
579         "Good Packets Received",
580         "Broadcast Packets Received",
581         "Multicast Packets Received",
582         "Good Packets Transmitted",
583         NULL,
584         "Good Octets Received",
585         NULL,
586         "Good Octets Transmitted",
587         NULL,
588         NULL,
589         NULL,
590         "Receive No Buffers",
591         "Receive Undersize",
592         "Receive Fragment",
593         "Receive Oversize",
594         "Receive Jabber",
595         NULL,
596         NULL,
597         NULL,
598         "Total Octets Received",
599         NULL,
600         "Total Octets Transmitted",
601         NULL,
602         "Total Packets Received",
603         "Total Packets Transmitted",
604         "Packets Transmitted (64 Bytes)",
605         "Packets Transmitted (65-127 Bytes)",
606         "Packets Transmitted (128-255 Bytes)",
607         "Packets Transmitted (256-511 Bytes)",
608         "Packets Transmitted (512-1023 Bytes)",
609         "Packets Transmitted (1024-1522 Bytes)",
610         "Multicast Packets Transmitted",
611         "Broadcast Packets Transmitted",
612         "TCP Segmentation Context Transmitted",
613         "TCP Segmentation Context Fail",
614 };
615
616 static void igbe_print_rd(struct Rd *rd)
617 {
618         printk("Rd %p: stat 0x%02x, err 0x%02x, len 0x%04x, check 0x%04x, "
619                "spec 0x%04x, addr[1] 0x%08x, addr[0] 0x%08x\n", rd,
620                    rd->status,
621                    rd->errors,
622                    rd->length,
623                    rd->checksum,
624                    rd->special,
625                    rd->addr[1],
626                    rd->addr[0]);
627 }
628
629 static long
630 igbeifstat(struct ether* edev, void* a, long n, uint32_t offset)
631 {
632         struct ctlr *ctlr;
633         char *p, *s;
634         int i, l, r;
635         uint64_t tuvl, ruvl;
636
637         ctlr = edev->ctlr;
638         qlock(&ctlr->slock);
639         p = kzmalloc(READSTR, 0);
640         if(p == NULL) {
641                 qunlock(&ctlr->slock);
642                 error(Enomem);
643         }
644         l = 0;
645         for(i = 0; i < Nstatistics; i++){
646                 r = csr32r(ctlr, Statistics+i*4);
647                 if((s = statistics[i]) == NULL)
648                         continue;
649                 switch(i){
650                 case Gorcl:
651                 case Gotcl:
652                 case Torl:
653                 case Totl:
654                         ruvl = r;
655                         ruvl += ((uint64_t)csr32r(ctlr, Statistics+(i+1)*4))<<32;
656                         tuvl = ruvl;
657                         tuvl += ctlr->statistics[i];
658                         tuvl += ((uint64_t)ctlr->statistics[i+1])<<32;
659                         if(tuvl == 0)
660                                 continue;
661                         ctlr->statistics[i] = tuvl;
662                         ctlr->statistics[i+1] = tuvl>>32;
663                         l += snprintf(p+l, READSTR-l, "%s: %llud %llud\n",
664                                 s, tuvl, ruvl);
665                         i++;
666                         break;
667
668                 default:
669                         ctlr->statistics[i] += r;
670                         if(ctlr->statistics[i] == 0)
671                                 continue;
672                         l += snprintf(p+l, READSTR-l, "%s: %ud %ud\n",
673                                 s, ctlr->statistics[i], r);
674                         break;
675                 }
676         }
677
678         l += snprintf(p+l, READSTR-l, "lintr: %ud %ud\n",
679                 ctlr->lintr, ctlr->lsleep);
680         l += snprintf(p+l, READSTR-l, "rintr: %ud %ud\n",
681                 ctlr->rintr, ctlr->rsleep);
682         l += snprintf(p+l, READSTR-l, "tintr: %ud %ud\n",
683                 ctlr->tintr, ctlr->txdw);
684         l += snprintf(p+l, READSTR-l, "ixcs: %ud %ud %ud\n",
685                 ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
686         l += snprintf(p+l, READSTR-l, "rdtr: %ud\n", ctlr->rdtr);
687         l += snprintf(p+l, READSTR-l, "Ctrlext: %08x\n", csr32r(ctlr, Ctrlext));
688
689         l += snprintf(p+l, READSTR-l, "eeprom:");
690         for(i = 0; i < 0x40; i++){
691                 if(i && ((i & 0x07) == 0))
692                         l += snprintf(p+l, READSTR-l, "\n       ");
693                 l += snprintf(p+l, READSTR-l, " %4.4uX", ctlr->eeprom[i]);
694         }
695         l += snprintf(p+l, READSTR-l, "\n");
696
697         if(ctlr->mii != NULL && ctlr->mii->curphy != NULL){
698                 l += snprintf(p+l, READSTR-l, "phy:   ");
699                 for(i = 0; i < NMiiPhyr; i++){
700                         if(i && ((i & 0x07) == 0))
701                                 l += snprintf(p+l, READSTR-l, "\n       ");
702                         r = miimir(ctlr->mii, i);
703                         l += snprintf(p+l, READSTR-l, " %4.4uX", r);
704                 }
705                 snprintf(p+l, READSTR-l, "\n");
706         }
707         n = readstr(offset, a, n, p);
708         kfree(p);
709         qunlock(&ctlr->slock);
710
711         return n;
712 }
713
714 enum {
715         CMrdtr,
716 };
717
718 static struct cmdtab igbectlmsg[] = {
719         {CMrdtr,        "rdtr", 2},
720 };
721
722 static long
723 igbectl(struct ether* edev, void* buf, long n)
724 {
725         ERRSTACK(2);
726         int v;
727         char *p;
728         struct ctlr *ctlr;
729         struct cmdbuf *cb;
730         struct cmdtab *ct;
731
732         if((ctlr = edev->ctlr) == NULL)
733                 error(Enonexist);
734
735         cb = parsecmd(buf, n);
736         if(waserror()){
737                 kfree(cb);
738                 nexterror();
739         }
740
741         ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
742         switch(ct->index){
743         case CMrdtr:
744                 v = strtol(cb->f[1], &p, 0);
745                 if(v < 0 || p == cb->f[1] || v > 0xFFFF)
746                         error(Ebadarg);
747                 ctlr->rdtr = v;
748                 csr32w(ctlr, Rdtr, Fpd|v);
749                 break;
750         }
751         kfree(cb);
752         poperror();
753
754         return n;
755 }
756
757 static void
758 igbepromiscuous(void* arg, int on)
759 {
760         int rctl;
761         struct ctlr *ctlr;
762         struct ether *edev;
763
764         edev = arg;
765         ctlr = edev->ctlr;
766
767         rctl = csr32r(ctlr, Rctl);
768         rctl &= ~MoMASK;
769         rctl |= Mo47b36;
770         if(on)
771                 rctl |= Upe|Mpe;
772         else
773                 rctl &= ~(Upe|Mpe);
774         csr32w(ctlr, Rctl, rctl|Mpe);   /* temporarily keep Mpe on */
775 }
776
777 static void
778 igbemulticast(void* arg, uint8_t* addr, int add)
779 {
780         int bit, x;
781         struct ctlr *ctlr;
782         struct ether *edev;
783
784         edev = arg;
785         ctlr = edev->ctlr;
786
787         x = addr[5]>>1;
788         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
789         /*
790          * multiple ether addresses can hash to the same filter bit,
791          * so it's never safe to clear a filter bit.
792          * if we want to clear filter bits, we need to keep track of
793          * all the multicast addresses in use, clear all the filter bits,
794          * then set the ones corresponding to in-use addresses.
795          */
796         if(add)
797                 ctlr->mta[x] |= 1<<bit;
798 //      else
799 //              ctlr->mta[x] &= ~(1<<bit);
800
801         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
802 }
803
804 static void
805 igbeim(struct ctlr* ctlr, int im)
806 {
807         ilock(&ctlr->imlock);
808         ctlr->im |= im;
809         csr32w(ctlr, Ims, ctlr->im);
810         iunlock(&ctlr->imlock);
811 }
812
813 static int
814 igbelim(void* ctlr)
815 {
816         return ((struct ctlr*)ctlr)->lim != 0;
817 }
818
819 static void
820 igbelproc(void* arg)
821 {
822         struct ctlr *ctlr;
823         struct ether *edev;
824         struct miiphy *phy;
825         int ctrl, r;
826
827         edev = arg;
828         ctlr = edev->ctlr;
829         for(;;){
830                 /* plan9 originally had a busy loop here (just called continue).  though
831                  * either you have the mii or you don't.  i don't think it'll magically
832                  * show up later (it should have been initialized during pnp/pci, which
833                  * is before attach, which is before lproc).  -brho */
834                 if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
835                         printk("[kernel] igbelproc can't find a mii/curphy, aborting!\n");
836                         /* name alloc'd in attach */
837                         kfree(per_cpu_info[core_id()].cur_kthread->name);
838                         return;
839                 }
840                 /*
841                  * To do:
842                  *      logic to manage status change,
843                  *      this is incomplete but should work
844                  *      one time to set up the hardware.
845                  *
846                  *      MiiPhy.speed, etc. should be in Mii.
847                  */
848                 if(miistatus(ctlr->mii) < 0)
849                         //continue;     /* this comment out was plan9, not brho */
850                         goto enable;
851
852                 phy = ctlr->mii->curphy;
853                 ctrl = csr32r(ctlr, Ctrl);
854
855                 switch(ctlr->id){
856                 case i82543gc:
857                 case i82544ei:
858                 case i82544eif:
859                 default:
860                         if(!(ctrl & Asde)){
861                                 ctrl &= ~(SspeedMASK|Ilos|Fd);
862                                 ctrl |= Frcdplx|Frcspd;
863                                 if(phy->speed == 1000)
864                                         ctrl |= Sspeed1000;
865                                 else if(phy->speed == 100)
866                                         ctrl |= Sspeed100;
867                                 if(phy->fd)
868                                         ctrl |= Fd;
869                         }
870                         break;
871
872                 case i82540em:
873                 case i82540eplp:
874                 case i82547gi:
875                 case i82541gi:
876                 case i82541gi2:
877                 case i82541pi:
878                         break;
879                 }
880
881                 /*
882                  * Collision Distance.
883                  */
884                 r = csr32r(ctlr, Tctl);
885                 r &= ~ColdMASK;
886                 if(phy->fd)
887                         r |= 64<<ColdSHIFT;
888                 else
889                         r |= 512<<ColdSHIFT;
890                 csr32w(ctlr, Tctl, r);
891
892                 /*
893                  * Flow control.
894                  */
895                 if(phy->rfc)
896                         ctrl |= Rfce;
897                 if(phy->tfc)
898                         ctrl |= Tfce;
899                 csr32w(ctlr, Ctrl, ctrl);
900
901 enable:
902                 ctlr->lim = 0;
903                 igbeim(ctlr, Lsc);
904
905                 ctlr->lsleep++;
906                 rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
907         }
908 }
909
910 static void
911 igbetxinit(struct ctlr* ctlr)
912 {
913         int i, r;
914         struct block *bp;
915
916         csr32w(ctlr, Tctl, (0x0F<<CtSHIFT)|Psp|(66<<ColdSHIFT));
917         switch(ctlr->id){
918         default:
919                 r = 6;
920                 break;
921         case i82543gc:
922         case i82544ei:
923         case i82544eif:
924         case i82544gc:
925         case i82540em:
926         case i82540eplp:
927         case i82541ei:
928         case i82541gi:
929         case i82541gi2:
930         case i82541pi:
931         case i82545em:
932         case i82545gmc:
933         case i82546gb:
934         case i82546eb:
935         case i82547ei:
936         case i82547gi:
937                 r = 8;
938                 break;
939         }
940         csr32w(ctlr, Tipg, (6<<20)|(8<<10)|r);
941         csr32w(ctlr, Ait, 0);
942         csr32w(ctlr, Txdmac, 0);
943         csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
944         csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
945         csr32w(ctlr, Tdlen, ctlr->ntd*sizeof(Td));
946         ctlr->tdh = PREV_RING(0, ctlr->ntd);
947         csr32w(ctlr, Tdh, 0);
948         ctlr->tdt = 0;
949         csr32w(ctlr, Tdt, 0);
950
951         for(i = 0; i < ctlr->ntd; i++){
952                 if((bp = ctlr->tb[i]) != NULL){
953                         ctlr->tb[i] = NULL;
954                         freeb(bp);
955                 }
956                 memset(&ctlr->tdba[i], 0, sizeof(Td));
957         }
958         ctlr->tdfree = ctlr->ntd;
959
960         csr32w(ctlr, Tidv, 128);
961         r = (4<<WthreshSHIFT)|(4<<HthreshSHIFT)|(8<<PthreshSHIFT);
962
963         switch(ctlr->id){
964         default:
965                 break;
966         case i82540em:
967         case i82540eplp:
968         case i82547gi:
969         case i82545em:
970         case i82545gmc:
971         case i82546gb:
972         case i82546eb:
973         case i82541gi:
974         case i82541gi2:
975         case i82541pi:
976                 r = csr32r(ctlr, Txdctl);
977                 r &= ~WthreshMASK;
978                 r |= Gran|(4<<WthreshSHIFT);
979
980                 csr32w(ctlr, Tadv, 64);
981                 break;
982         }
983
984         csr32w(ctlr, Txdctl, r);
985
986         r = csr32r(ctlr, Tctl);
987         r |= Ten;
988         csr32w(ctlr, Tctl, r);
989 }
990
991 static void
992 igbetransmit(struct ether* edev)
993 {
994         Td *td;
995         struct block *bp;
996         struct ctlr *ctlr;
997         int tdh, tdt;
998
999         ctlr = edev->ctlr;
1000
1001         ilock(&ctlr->tlock);
1002
1003         /*
1004          * Free any completed packets
1005          */
1006         tdh = ctlr->tdh;
1007         while(NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)){
1008                 if((bp = ctlr->tb[tdh]) != NULL){
1009                         ctlr->tb[tdh] = NULL;
1010                         freeb(bp);
1011                 }
1012                 memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1013                 tdh = NEXT_RING(tdh, ctlr->ntd);
1014         }
1015         ctlr->tdh = tdh;
1016
1017         /*
1018          * Try to fill the ring back up.
1019          */
1020         tdt = ctlr->tdt;
1021         while(NEXT_RING(tdt, ctlr->ntd) != tdh){
1022                 if((bp = qget(edev->oq)) == NULL)
1023                         break;
1024                 td = &ctlr->tdba[tdt];
1025                 td->addr[0] = paddr_low32(bp->rp);
1026                 td->addr[1] = paddr_high32(bp->rp);
1027                 td->control = ((BLEN(bp) & LenMASK)<<LenSHIFT);
1028                 td->control |= Dext|Ifcs|Teop|DtypeDD;
1029                 ctlr->tb[tdt] = bp;
1030                 tdt = NEXT_RING(tdt, ctlr->ntd);
1031                 if(NEXT_RING(tdt, ctlr->ntd) == tdh){
1032                         td->control |= Rs;
1033                         ctlr->txdw++;
1034                         ctlr->tdt = tdt;
1035                         csr32w(ctlr, Tdt, tdt);
1036                         igbeim(ctlr, Txdw);
1037                         break;
1038                 }
1039                 ctlr->tdt = tdt;
1040                 csr32w(ctlr, Tdt, tdt);
1041         }
1042
1043         iunlock(&ctlr->tlock);
1044 }
1045
1046 static void
1047 igbereplenish(struct ctlr* ctlr)
1048 {
1049         Rd *rd;
1050         int rdt;
1051         struct block *bp;
1052
1053         rdt = ctlr->rdt;
1054         while(NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh){
1055                 rd = &ctlr->rdba[rdt];
1056                 if(ctlr->rb[rdt] == NULL){
1057                         bp = iallocb(Rbsz);
1058                         if(bp == NULL){
1059                                 /* needs to be a safe print for interrupt level */
1060                                 printk("#l%d: igbereplenish: no available buffers\n",
1061                                         ctlr->edev->ctlrno);
1062                                 break;
1063                         }
1064                         ctlr->rb[rdt] = bp;
1065                         rd->addr[0] = paddr_low32(bp->rp);
1066                         rd->addr[1] = paddr_high32(bp->rp);
1067                 }
1068                 wmb();  /* ensure prev rd writes come before status = 0. */
1069                 rd->status = 0;
1070                 rdt = NEXT_RING(rdt, ctlr->nrd);
1071                 ctlr->rdfree++;
1072         }
1073         ctlr->rdt = rdt;
1074         csr32w(ctlr, Rdt, rdt);
1075 }
1076
1077 static void
1078 igberxinit(struct ctlr* ctlr)
1079 {
1080         int i;
1081         struct block *bp;
1082
1083         /* temporarily keep Mpe on */
1084         csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF|Mpe);
1085         csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1086         csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1087         csr32w(ctlr, Rdlen, ctlr->nrd*sizeof(Rd));
1088         ctlr->rdh = 0;
1089         csr32w(ctlr, Rdh, 0);
1090         ctlr->rdt = 0;
1091         csr32w(ctlr, Rdt, 0);
1092         ctlr->rdtr = 0;
1093         csr32w(ctlr, Rdtr, Fpd|0);
1094
1095         for(i = 0; i < ctlr->nrd; i++){
1096                 if((bp = ctlr->rb[i]) != NULL){
1097                         ctlr->rb[i] = NULL;
1098                         freeb(bp);
1099                 }
1100         }
1101         igbereplenish(ctlr);
1102
1103         switch(ctlr->id){
1104         case i82540em:
1105         case i82540eplp:
1106         case i82541gi:
1107         case i82541gi2:
1108         case i82541pi:
1109         case i82545em:
1110         case i82545gmc:
1111         case i82546gb:
1112         case i82546eb:
1113         case i82547gi:
1114                 csr32w(ctlr, Radv, 64);
1115                 break;
1116         }
1117         csr32w(ctlr, Rxdctl, (8<<WthreshSHIFT)|(8<<HthreshSHIFT)|4);
1118
1119         /*
1120          * Enable checksum offload.
1121          */
1122         csr32w(ctlr, Rxcsum, Tuofl|Ipofl|(ETHERHDRSIZE<<PcssSHIFT));
1123 }
1124
1125 static int
1126 igberim(void* ctlr)
1127 {
1128         return ((struct ctlr*)ctlr)->rim != 0;
1129 }
1130
1131 static void
1132 igberproc(void* arg)
1133 {
1134         Rd *rd;
1135         struct block *bp;
1136         struct ctlr *ctlr;
1137         int r, rdh;
1138         struct ether *edev;
1139
1140         edev = arg;
1141         ctlr = edev->ctlr;
1142
1143         igberxinit(ctlr);
1144         r = csr32r(ctlr, Rctl);
1145         r |= Ren;
1146         csr32w(ctlr, Rctl, r);
1147
1148         for(;;){
1149                 ctlr->rim = 0;
1150                 igbeim(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq);
1151                 ctlr->rsleep++;
1152                 rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1153
1154                 rdh = ctlr->rdh;
1155                 for(;;){
1156                         rd = &ctlr->rdba[rdh];
1157
1158                         if(!(rd->status & Rdd))
1159                                 break;
1160
1161                         /*
1162                          * Accept eop packets with no errors.
1163                          * With no errors and the Ixsm bit set,
1164                          * the descriptor status Tpcs and Ipcs bits give
1165                          * an indication of whether the checksums were
1166                          * calculated and valid.
1167                          */
1168                         if((rd->status & Reop) && rd->errors == 0){
1169                                 bp = ctlr->rb[rdh];
1170                                 ctlr->rb[rdh] = NULL;
1171                                 bp->wp += rd->length;
1172                                 bp->next = NULL;
1173                                 if(!(rd->status & Ixsm)){
1174                                         ctlr->ixsm++;
1175                                         if(rd->status & Ipcs){
1176                                                 /*
1177                                                  * IP checksum calculated
1178                                                  * (and valid as errors == 0).
1179                                                  */
1180                                                 ctlr->ipcs++;
1181                                                 bp->flag |= Bipck;
1182                                         }
1183                                         if(rd->status & Tcpcs){
1184                                                 /*
1185                                                  * TCP/UDP checksum calculated
1186                                                  * (and valid as errors == 0).
1187                                                  */
1188                                                 ctlr->tcpcs++;
1189                                                 bp->flag |= Btcpck|Budpck;
1190                                         }
1191                                         bp->checksum = rd->checksum;
1192                                         bp->flag |= Bpktck;
1193                                 }
1194                                 etheriq(edev, bp, 1);
1195                         }
1196                         else if(ctlr->rb[rdh] != NULL){
1197                                 freeb(ctlr->rb[rdh]);
1198                                 ctlr->rb[rdh] = NULL;
1199                         }
1200
1201                         memset(rd, 0, sizeof(Rd));
1202                         wmb();  /* make sure the zeroing happens before free (i think) */
1203                         ctlr->rdfree--;
1204                         rdh = NEXT_RING(rdh, ctlr->nrd);
1205                 }
1206                 ctlr->rdh = rdh;
1207
1208                 if(ctlr->rdfree < ctlr->nrd/2 || (ctlr->rim & Rxdmt0))
1209                         igbereplenish(ctlr);
1210         }
1211 }
1212
1213 static void
1214 igbeattach(struct ether* edev)
1215 {
1216         ERRSTACK(1);
1217         struct block *bp;
1218         struct ctlr *ctlr;
1219         char *name;
1220
1221         ctlr = edev->ctlr;
1222         ctlr->edev = edev;                      /* point back to Ether* */
1223         qlock(&ctlr->alock);
1224         if(ctlr->alloc != NULL){                        /* already allocated? */
1225                 qunlock(&ctlr->alock);
1226                 return;
1227         }
1228
1229         ctlr->tb = NULL;
1230         ctlr->rb = NULL;
1231         ctlr->alloc = NULL;
1232         if(waserror()){
1233                 kfree(ctlr->tb);
1234                 ctlr->tb = NULL;
1235                 kfree(ctlr->rb);
1236                 ctlr->rb = NULL;
1237                 kfree(ctlr->alloc);
1238                 ctlr->alloc = NULL;
1239                 qunlock(&ctlr->alock);
1240                 nexterror();
1241         }
1242
1243         ctlr->nrd = Nrd;
1244         ctlr->ntd = Ntd;
1245         ctlr->alloc = kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1246         if(ctlr->alloc == NULL) {
1247                 printd("igbe: can't allocate ctlr->alloc\n");
1248                 error(Enomem);
1249         }
1250         ctlr->rdba = (Rd*)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1251         ctlr->tdba = (Td*)(ctlr->rdba+ctlr->nrd);
1252
1253         ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1254         ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1255         if (ctlr->rb == NULL || ctlr->tb == NULL) {
1256                 printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1257                 error(Enomem);
1258         }
1259
1260         /* the ktasks should free these names, if they ever exit */
1261         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1262         snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1263         ktask(name, igbelproc, edev);
1264
1265         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1266         snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1267         ktask(name, igberproc, edev);
1268
1269         igbetxinit(ctlr);
1270
1271         qunlock(&ctlr->alock);
1272         poperror();
1273 }
1274
1275 static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1276 {
1277         struct ctlr *ctlr;
1278         struct ether *edev;
1279         int icr, im, txdw;
1280
1281         edev = arg;
1282         ctlr = edev->ctlr;
1283
1284         ilock(&ctlr->imlock);
1285         csr32w(ctlr, Imc, ~0);
1286         im = ctlr->im;
1287         txdw = 0;
1288
1289         while((icr = csr32r(ctlr, Icr) & ctlr->im) != 0){
1290                 if(icr & Lsc){
1291                         im &= ~Lsc;
1292                         ctlr->lim = icr & Lsc;
1293                         rendez_wakeup(&ctlr->lrendez);
1294                         ctlr->lintr++;
1295                 }
1296                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq)){
1297                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq);
1298                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq);
1299                         rendez_wakeup(&ctlr->rrendez);
1300                         ctlr->rintr++;
1301                 }
1302                 if(icr & Txdw){
1303                         im &= ~Txdw;
1304                         txdw++;
1305                         ctlr->tintr++;
1306                 }
1307         }
1308
1309         ctlr->im = im;
1310         csr32w(ctlr, Ims, im);
1311         iunlock(&ctlr->imlock);
1312
1313         if(txdw)
1314                 igbetransmit(edev);
1315 }
1316
1317 static int
1318 i82543mdior(struct ctlr* ctlr, int n)
1319 {
1320         int ctrl, data, i, r;
1321
1322         /*
1323          * Read n bits from the Management Data I/O Interface.
1324          */
1325         ctrl = csr32r(ctlr, Ctrl);
1326         r = (ctrl & ~Mddo)|Mdco;
1327         data = 0;
1328         for(i = n-1; i >= 0; i--){
1329                 if(csr32r(ctlr, Ctrl) & Mdd)
1330                         data |= (1<<i);
1331                 csr32w(ctlr, Ctrl, Mdc|r);
1332                 csr32w(ctlr, Ctrl, r);
1333         }
1334         csr32w(ctlr, Ctrl, ctrl);
1335
1336         return data;
1337 }
1338
1339 static int
1340 i82543mdiow(struct ctlr* ctlr, int bits, int n)
1341 {
1342         int ctrl, i, r;
1343
1344         /*
1345          * Write n bits to the Management Data I/O Interface.
1346          */
1347         ctrl = csr32r(ctlr, Ctrl);
1348         r = Mdco|Mddo|ctrl;
1349         for(i = n-1; i >= 0; i--){
1350                 if(bits & (1<<i))
1351                         r |= Mdd;
1352                 else
1353                         r &= ~Mdd;
1354                 csr32w(ctlr, Ctrl, Mdc|r);
1355                 csr32w(ctlr, Ctrl, r);
1356         }
1357         csr32w(ctlr, Ctrl, ctrl);
1358
1359         return 0;
1360 }
1361
1362 static int
1363 i82543miimir(struct mii* mii, int pa, int ra)
1364 {
1365         int data;
1366         struct ctlr *ctlr;
1367
1368         ctlr = mii->ctlr;
1369
1370         /*
1371          * MII Management Interface Read.
1372          *
1373          * Preamble;
1374          * ST+OP+PHYAD+REGAD;
1375          * TA + 16 data bits.
1376          */
1377         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1378         i82543mdiow(ctlr, 0x1800|(pa<<5)|ra, 14);
1379         data = i82543mdior(ctlr, 18);
1380
1381         if(data & 0x10000)
1382                 return -1;
1383
1384         return data & 0xFFFF;
1385 }
1386
1387 static int
1388 i82543miimiw(struct mii* mii, int pa, int ra, int data)
1389 {
1390         struct ctlr *ctlr;
1391
1392         ctlr = mii->ctlr;
1393
1394         /*
1395          * MII Management Interface Write.
1396          *
1397          * Preamble;
1398          * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1399          * Z.
1400          */
1401         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1402         data &= 0xFFFF;
1403         data |= (0x05<<(5+5+2+16))|(pa<<(5+2+16))|(ra<<(2+16))|(0x02<<16);
1404         i82543mdiow(ctlr, data, 32);
1405
1406         return 0;
1407 }
1408
1409 static int
1410 igbemiimir(struct mii* mii, int pa, int ra)
1411 {
1412         struct ctlr *ctlr;
1413         int mdic, timo;
1414
1415         ctlr = mii->ctlr;
1416
1417         csr32w(ctlr, Mdic, MDIrop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT));
1418         mdic = 0;
1419         for(timo = 64; timo; timo--){
1420                 mdic = csr32r(ctlr, Mdic);
1421                 if(mdic & (MDIe|MDIready))
1422                         break;
1423                 udelay(1);
1424         }
1425
1426         if((mdic & (MDIe|MDIready)) == MDIready)
1427                 return mdic & 0xFFFF;
1428         return -1;
1429 }
1430
1431 static int
1432 igbemiimiw(struct mii* mii, int pa, int ra, int data)
1433 {
1434         struct ctlr *ctlr;
1435         int mdic, timo;
1436
1437         ctlr = mii->ctlr;
1438
1439         data &= MDIdMASK;
1440         csr32w(ctlr, Mdic, MDIwop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT)|data);
1441         mdic = 0;
1442         for(timo = 64; timo; timo--){
1443                 mdic = csr32r(ctlr, Mdic);
1444                 if(mdic & (MDIe|MDIready))
1445                         break;
1446                 udelay(1);
1447         }
1448         if((mdic & (MDIe|MDIready)) == MDIready)
1449                 return 0;
1450         return -1;
1451 }
1452
1453 static int
1454 i82543miirw(struct mii* mii, int write, int pa, int ra, int data)
1455 {
1456         if(write)
1457                 return i82543miimiw(mii, pa, ra, data);
1458
1459         return i82543miimir(mii, pa, ra);
1460 }
1461
1462 static int
1463 igbemiirw(struct mii* mii, int write, int pa, int ra, int data)
1464 {
1465         if(write)
1466                 return igbemiimiw(mii, pa, ra, data);
1467
1468         return igbemiimir(mii, pa, ra);
1469 }
1470
1471 static int
1472 igbemii(struct ctlr* ctlr)
1473 {
1474         int ctrl, p, r;
1475         int (*rw)(struct mii*, int unused_int, int, int, int);
1476
1477         r = csr32r(ctlr, Status);
1478         if(r & Tbimode)
1479                 return -1;
1480
1481         ctrl = csr32r(ctlr, Ctrl);
1482         ctrl |= Slu;
1483
1484         switch(ctlr->id){
1485         case i82543gc:
1486                 ctrl |= Frcdplx|Frcspd;
1487                 csr32w(ctlr, Ctrl, ctrl);
1488
1489                 /*
1490                  * The reset pin direction (Mdro) should already
1491                  * be set from the EEPROM load.
1492                  * If it's not set this configuration is unexpected
1493                  * so bail.
1494                  */
1495                 r = csr32r(ctlr, Ctrlext);
1496                 if(!(r & Mdro))
1497                         return -1;
1498                 csr32w(ctlr, Ctrlext, r);
1499                 udelay(20*1000);
1500                 r = csr32r(ctlr, Ctrlext);
1501                 r &= ~Mdr;
1502                 csr32w(ctlr, Ctrlext, r);
1503                 udelay(20*1000);
1504                 r = csr32r(ctlr, Ctrlext);
1505                 r |= Mdr;
1506                 csr32w(ctlr, Ctrlext, r);
1507                 udelay(20*1000);
1508
1509                 rw = i82543miirw;
1510                 break;
1511         case i82544ei:
1512         case i82544eif:
1513         case i82544gc:
1514         case i82540em:
1515         case i82540eplp:
1516         case i82547ei:
1517         case i82547gi:
1518         case i82541ei:
1519         case i82541gi:
1520         case i82541gi2:
1521         case i82541pi:
1522         case i82545em:
1523         case i82545gmc:
1524         case i82546gb:
1525         case i82546eb:
1526                 ctrl &= ~(Frcdplx|Frcspd);
1527                 csr32w(ctlr, Ctrl, ctrl);
1528                 rw = igbemiirw;
1529                 break;
1530         default:
1531                 return -1;
1532         }
1533
1534         if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1535                 return -1;
1536         // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1537
1538         /*
1539          * 8254X-specific PHY registers not in 802.3:
1540          *      0x10    PHY specific control
1541          *      0x14    extended PHY specific control
1542          * Set appropriate values then reset the PHY to have
1543          * changes noted.
1544          */
1545         switch(ctlr->id){
1546         case i82547gi:
1547         case i82541gi:
1548         case i82541gi2:
1549         case i82541pi:
1550         case i82545em:
1551         case i82545gmc:
1552         case i82546gb:
1553         case i82546eb:
1554                 break;
1555         default:
1556                 r = miimir(ctlr->mii, 16);
1557                 r |= 0x0800;                    /* assert CRS on Tx */
1558                 r |= 0x0060;                    /* auto-crossover all speeds */
1559                 r |= 0x0002;                    /* polarity reversal enabled */
1560                 miimiw(ctlr->mii, 16, r);
1561
1562                 r = miimir(ctlr->mii, 20);
1563                 r |= 0x0070;                    /* +25MHz clock */
1564                 r &= ~0x0F00;
1565                 r |= 0x0100;                    /* 1x downshift */
1566                 miimiw(ctlr->mii, 20, r);
1567
1568                 miireset(ctlr->mii);
1569                 p = 0;
1570                 if(ctlr->txcw & TxcwPs)
1571                         p |= AnaP;
1572                 if(ctlr->txcw & TxcwAs)
1573                         p |= AnaAP;
1574                 miiane(ctlr->mii, ~0, p, ~0);
1575                 break;
1576         }
1577         return 0;
1578 }
1579
1580 static int
1581 at93c46io(struct ctlr* ctlr, char* op, int data)
1582 {
1583         char *lp, *p;
1584         int i, loop, eecd, r;
1585
1586         eecd = csr32r(ctlr, Eecd);
1587
1588         r = 0;
1589         loop = -1;
1590         lp = NULL;
1591         for(p = op; *p != '\0'; p++){
1592                 switch(*p){
1593                 default:
1594                         return -1;
1595                 case ' ':
1596                         continue;
1597                 case ':':                       /* start of loop */
1598                         loop = strtol(p+1, &lp, 0)-1;
1599                         lp--;
1600                         if(p == lp)
1601                                 loop = 7;
1602                         p = lp;
1603                         continue;
1604                 case ';':                       /* end of loop */
1605                         if(lp == NULL)
1606                                 return -1;
1607                         loop--;
1608                         if(loop >= 0)
1609                                 p = lp;
1610                         else
1611                                 lp = NULL;
1612                         continue;
1613                 case 'C':                       /* assert clock */
1614                         eecd |= Sk;
1615                         break;
1616                 case 'c':                       /* deassert clock */
1617                         eecd &= ~Sk;
1618                         break;
1619                 case 'D':                       /* next bit in 'data' byte */
1620                         if(loop < 0)
1621                                 return -1;
1622                         if(data & (1<<loop))
1623                                 eecd |= Di;
1624                         else
1625                                 eecd &= ~Di;
1626                         break;
1627                 case 'O':                       /* collect data output */
1628                         i = (csr32r(ctlr, Eecd) & Do) != 0;
1629                         if(loop >= 0)
1630                                 r |= (i<<loop);
1631                         else
1632                                 r = i;
1633                         continue;
1634                 case 'I':                       /* assert data input */
1635                         eecd |= Di;
1636                         break;
1637                 case 'i':                       /* deassert data input */
1638                         eecd &= ~Di;
1639                         break;
1640                 case 'S':                       /* enable chip select */
1641                         eecd |= Cs;
1642                         break;
1643                 case 's':                       /* disable chip select */
1644                         eecd &= ~Cs;
1645                         break;
1646                 }
1647                 csr32w(ctlr, Eecd, eecd);
1648                 udelay(50);
1649         }
1650         if(loop >= 0)
1651                 return -1;
1652         return r;
1653 }
1654
1655 static int
1656 at93c46r(struct ctlr* ctlr)
1657 {
1658         uint16_t sum;
1659         char rop[20];
1660         int addr, areq, bits, data, eecd, i;
1661
1662         eecd = csr32r(ctlr, Eecd);
1663         if(eecd & Spi){
1664                 printd("igbe: SPI EEPROM access not implemented\n");
1665                 return 0;
1666         }
1667         if(eecd & (Eeszaddr|Eesz256))
1668                 bits = 8;
1669         else
1670                 bits = 6;
1671
1672         sum = 0;
1673
1674         switch(ctlr->id){
1675         default:
1676                 areq = 0;
1677                 break;
1678         case i82540em:
1679         case i82540eplp:
1680         case i82541ei:
1681         case i82541gi:
1682         case i82541gi2:
1683         case i82541pi:
1684         case i82545em:
1685         case i82545gmc:
1686         case i82546gb:
1687         case i82546eb:
1688         case i82547ei:
1689         case i82547gi:
1690                 areq = 1;
1691                 csr32w(ctlr, Eecd, eecd|Areq);
1692                 for(i = 0; i < 1000; i++){
1693                         if((eecd = csr32r(ctlr, Eecd)) & Agnt)
1694                                 break;
1695                         udelay(5);
1696                 }
1697                 if(!(eecd & Agnt)){
1698                         printd("igbe: not granted EEPROM access\n");
1699                         goto release;
1700                 }
1701                 break;
1702         }
1703         snprintf(rop, sizeof(rop), "S :%dDCc;", bits+3);
1704
1705         for(addr = 0; addr < 0x40; addr++){
1706                 /*
1707                  * Read a word at address 'addr' from the Atmel AT93C46
1708                  * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1709                  * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1710                  * for protocol details.
1711                  */
1712                 if(at93c46io(ctlr, rop, (0x06<<bits)|addr) != 0){
1713                         printd("igbe: can't set EEPROM address 0x%2.2X\n", addr);
1714                         goto release;
1715                 }
1716                 data = at93c46io(ctlr, ":16COc;", 0);
1717                 at93c46io(ctlr, "sic", 0);
1718                 ctlr->eeprom[addr] = data;
1719                 sum += data;
1720         }
1721
1722 release:
1723         if(areq)
1724                 csr32w(ctlr, Eecd, eecd & ~Areq);
1725         return sum;
1726 }
1727
1728 static int
1729 igbedetach(struct ctlr* ctlr)
1730 {
1731         int r, timeo;
1732
1733         /*
1734          * Perform a device reset to get the chip back to the
1735          * power-on state, followed by an EEPROM reset to read
1736          * the defaults for some internal registers.
1737          */
1738         csr32w(ctlr, Imc, ~0);
1739         csr32w(ctlr, Rctl, 0);
1740         csr32w(ctlr, Tctl, 0);
1741
1742         udelay(10*1000);
1743
1744         csr32w(ctlr, Ctrl, Devrst);
1745         udelay(1*1000);
1746         for(timeo = 0; timeo < 1000; timeo++){
1747                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1748                         break;
1749                 udelay(1*1000);
1750         }
1751         if(csr32r(ctlr, Ctrl) & Devrst)
1752                 return -1;
1753         r = csr32r(ctlr, Ctrlext);
1754         csr32w(ctlr, Ctrlext, r|Eerst);
1755         udelay(1*1000);
1756
1757         for(timeo = 0; timeo < 1000; timeo++){
1758                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1759                         break;
1760                 udelay(1*1000);
1761         }
1762         if(csr32r(ctlr, Ctrlext) & Eerst)
1763                 return -1;
1764
1765         switch(ctlr->id){
1766         default:
1767                 break;
1768         case i82540em:
1769         case i82540eplp:
1770         case i82541gi:
1771         case i82541gi2:
1772         case i82541pi:
1773         case i82545em:
1774         case i82545gmc:
1775         case i82547gi:
1776         case i82546gb:
1777         case i82546eb:
1778                 r = csr32r(ctlr, Manc);
1779                 r &= ~Arpen;
1780                 csr32w(ctlr, Manc, r);
1781                 break;
1782         }
1783
1784         csr32w(ctlr, Imc, ~0);
1785         udelay(1*1000);
1786         for(timeo = 0; timeo < 1000; timeo++){
1787                 if(!csr32r(ctlr, Icr))
1788                         break;
1789                 udelay(1*1000);
1790         }
1791         if(csr32r(ctlr, Icr))
1792                 return -1;
1793
1794         return 0;
1795 }
1796
1797 static void
1798 igbeshutdown(struct ether* ether)
1799 {
1800         igbedetach(ether->ctlr);
1801 }
1802
1803 static int
1804 igbereset(struct ctlr* ctlr)
1805 {
1806         int ctrl, i, pause, r, swdpio, txcw;
1807
1808         if(igbedetach(ctlr))
1809                 return -1;
1810
1811         /*
1812          * Read the EEPROM, validate the checksum
1813          * then get the device back to a power-on state.
1814          */
1815         if((r = at93c46r(ctlr)) != 0xBABA){
1816                 printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1817                 return -1;
1818         }
1819
1820         /*
1821          * Snarf and set up the receive addresses.
1822          * There are 16 addresses. The first should be the MAC address.
1823          * The others are cleared and not marked valid (MS bit of Rah).
1824          */
1825         if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1826             (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1827                  pci_config_addr(0, 1, 0, 0)))
1828                 ctlr->eeprom[Ea+2] += 0x100;            /* second interface */
1829         if(ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1830                 ctlr->eeprom[Ea] = 0xD000;
1831         for(i = Ea; i < Eaddrlen/2; i++){
1832                 ctlr->ra[2*i] = ctlr->eeprom[i];
1833                 ctlr->ra[2*i+1] = ctlr->eeprom[i]>>8;
1834         }
1835         /* lan id seems to vary on 82543gc; don't use it */
1836         if (ctlr->id != i82543gc) {
1837                 r = (csr32r(ctlr, Status) & Lanid) >> 2;
1838                 ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1839         }
1840
1841         r = (ctlr->ra[3]<<24)|(ctlr->ra[2]<<16)|(ctlr->ra[1]<<8)|ctlr->ra[0];
1842         csr32w(ctlr, Ral, r);
1843         r = 0x80000000|(ctlr->ra[5]<<8)|ctlr->ra[4];
1844         csr32w(ctlr, Rah, r);
1845         for(i = 1; i < 16; i++){
1846                 csr32w(ctlr, Ral+i*8, 0);
1847                 csr32w(ctlr, Rah+i*8, 0);
1848         }
1849
1850         /*
1851          * Clear the Multicast Table Array.
1852          * It's a 4096 bit vector accessed as 128 32-bit registers.
1853          */
1854         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1855         for(i = 0; i < 128; i++)
1856                 csr32w(ctlr, Mta+i*4, 0);
1857
1858         /*
1859          * Just in case the Eerst didn't load the defaults
1860          * (doesn't appear to fully on the 82543GC), do it manually.
1861          */
1862         if (ctlr->id == i82543gc) {
1863                 txcw = csr32r(ctlr, Txcw);
1864                 txcw &= ~(TxcwAne|TxcwPauseMASK|TxcwFd);
1865                 ctrl = csr32r(ctlr, Ctrl);
1866                 ctrl &= ~(SwdpioloMASK|Frcspd|Ilos|Lrst|Fd);
1867
1868                 if(ctlr->eeprom[Icw1] & 0x0400){
1869                         ctrl |= Fd;
1870                         txcw |= TxcwFd;
1871                 }
1872                 if(ctlr->eeprom[Icw1] & 0x0200)
1873                         ctrl |= Lrst;
1874                 if(ctlr->eeprom[Icw1] & 0x0010)
1875                         ctrl |= Ilos;
1876                 if(ctlr->eeprom[Icw1] & 0x0800)
1877                         ctrl |= Frcspd;
1878                 swdpio = (ctlr->eeprom[Icw1] & 0x01E0)>>5;
1879                 ctrl |= swdpio<<SwdpioloSHIFT;
1880                 csr32w(ctlr, Ctrl, ctrl);
1881
1882                 ctrl = csr32r(ctlr, Ctrlext);
1883                 ctrl &= ~(Ips|SwdpiohiMASK);
1884                 swdpio = (ctlr->eeprom[Icw2] & 0x00F0)>>4;
1885                 if(ctlr->eeprom[Icw1] & 0x1000)
1886                         ctrl |= Ips;
1887                 ctrl |= swdpio<<SwdpiohiSHIFT;
1888                 csr32w(ctlr, Ctrlext, ctrl);
1889
1890                 if(ctlr->eeprom[Icw2] & 0x0800)
1891                         txcw |= TxcwAne;
1892                 pause = (ctlr->eeprom[Icw2] & 0x3000)>>12;
1893                 txcw |= pause<<TxcwPauseSHIFT;
1894                 switch(pause){
1895                 default:
1896                         ctlr->fcrtl = 0x00002000;
1897                         ctlr->fcrth = 0x00004000;
1898                         txcw |= TxcwAs|TxcwPs;
1899                         break;
1900                 case 0:
1901                         ctlr->fcrtl = 0x00002000;
1902                         ctlr->fcrth = 0x00004000;
1903                         break;
1904                 case 2:
1905                         ctlr->fcrtl = 0;
1906                         ctlr->fcrth = 0;
1907                         txcw |= TxcwAs;
1908                         break;
1909                 }
1910                 ctlr->txcw = txcw;
1911                 csr32w(ctlr, Txcw, txcw);
1912         }
1913
1914
1915         /*
1916          * Flow control - values from the datasheet.
1917          */
1918         csr32w(ctlr, Fcal, 0x00C28001);
1919         csr32w(ctlr, Fcah, 0x00000100);
1920         csr32w(ctlr, Fct, 0x00008808);
1921         csr32w(ctlr, Fcttv, 0x00000100);
1922
1923         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1924         csr32w(ctlr, Fcrth, ctlr->fcrth);
1925
1926         /* FYI, igbemii checks status right away too. */
1927         if(!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1928                 printk("igbemii failed!  igbe failing to reset!\n");
1929                 return -1;
1930         }
1931
1932         return 0;
1933 }
1934
1935 static void
1936 igbepci(void)
1937 {
1938         int cls, id;
1939         struct pci_device *pcidev;
1940         struct ctlr *ctlr;
1941         void *mem;
1942         uintptr_t mmio_paddr;
1943
1944         STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
1945                 /* This checks that pcidev is a Network Controller for Ethernet */
1946                 if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1947                         continue;
1948                 id = pcidev->dev_id << 16 | pcidev->ven_id;
1949                 switch (id) {
1950                 default:
1951                         continue;
1952                 case i82543gc:
1953                 case i82544ei:
1954                 case i82544eif:
1955                 case i82544gc:
1956                 case i82547ei:
1957                 case i82547gi:
1958                 case i82540em:
1959                 case i82540eplp:
1960                 case i82541ei:
1961                 case i82541gi:
1962                 case i82541gi2:
1963                 case i82541pi:
1964                 case i82545em:
1965                 case i82545gmc:
1966                 case i82546gb:
1967                 case i82546eb:
1968                         break;
1969                 }
1970                 printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
1971                        pcidev->ven_id, pcidev->dev_id,
1972                        pcidev->bus, pcidev->dev, pcidev->func);
1973
1974                 mmio_paddr = pcidev->bar[0].mmio_base32 ? pcidev->bar[0].mmio_base32 : 
1975                                                           pcidev->bar[0].mmio_base64;
1976                 mem = (void*)vmap_pmem_nocache(mmio_paddr, pcidev->bar[0].mmio_sz);
1977                 if(mem == NULL){
1978                         printd("igbe: can't map %p\n", pcidev->bar[0].mmio_base32);
1979                         continue;
1980                 }
1981                 cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
1982                 switch(cls){
1983                         default:
1984                                 printd("igbe: unexpected CLS - %d\n", cls*4);
1985                                 break;
1986                         case 0x00:
1987                         case 0xFF:
1988                                 /* bogus value; use a sane default.  cls is set in DWORD (u32)
1989                                  * units. */
1990                                 cls = ARCH_CL_SIZE / sizeof(long);
1991                                 pcidev_write8(pcidev, PCI_CLSZ_REG, cls);
1992                                 break;
1993                         case 0x08:
1994                         case 0x10:
1995                                 break;
1996                 }
1997                 ctlr = kzmalloc(sizeof(struct ctlr), 0);
1998                 if(ctlr == NULL) {
1999                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2000                         error(Enomem);
2001                 }
2002                 spinlock_init_irqsave(&ctlr->imlock);
2003                 spinlock_init_irqsave(&ctlr->tlock);
2004                 qlock_init(&ctlr->alock);
2005                 qlock_init(&ctlr->slock);
2006                 rendez_init(&ctlr->lrendez);
2007                 rendez_init(&ctlr->rrendez);
2008                 /* port seems to be unused, and only used for some comparison with edev.
2009                  * plan9 just used the top of the raw bar, regardless of the type. */
2010                 ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
2011                 ctlr->pci = pcidev;
2012                 ctlr->id = id;
2013                 ctlr->cls = cls * sizeof(long);
2014                 ctlr->nic = mem;
2015
2016                 if(igbereset(ctlr)){
2017                         kfree(ctlr);
2018                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2019                         continue;
2020                 }
2021                 pci_set_bus_master(pcidev);
2022
2023                 if(igbectlrhead != NULL)
2024                         igbectlrtail->next = ctlr;
2025                 else
2026                         igbectlrhead = ctlr;
2027                 igbectlrtail = ctlr;
2028         }
2029 }
2030
2031 static int
2032 igbepnp(struct ether* edev)
2033 {
2034         struct ctlr *ctlr;
2035
2036         run_once(igbepci());
2037
2038         /*
2039          * Any adapter matches if no edev->port is supplied,
2040          * otherwise the ports must match.
2041          */
2042         for(ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next){
2043                 if(ctlr->active)
2044                         continue;
2045                 if(edev->port == 0 || edev->port == ctlr->port){
2046                         ctlr->active = 1;
2047                         break;
2048                 }
2049         }
2050         if(ctlr == NULL)
2051                 return -1;
2052
2053         edev->ctlr = ctlr;
2054         edev->port = ctlr->port;
2055         edev->irq = ctlr->pci->irqline;
2056         edev->tbdf = pci_to_tbdf(ctlr->pci);
2057         edev->mbps = 1000;
2058         memmove(edev->ea, ctlr->ra, Eaddrlen);
2059
2060         /*
2061          * Linkage to the generic ethernet driver.
2062          */
2063         edev->attach = igbeattach;
2064         edev->transmit = igbetransmit;
2065         edev->ifstat = igbeifstat;
2066         edev->ctl = igbectl;
2067         edev->shutdown = igbeshutdown;
2068
2069         edev->arg = edev;
2070         edev->promiscuous = igbepromiscuous;
2071         edev->multicast = igbemulticast;
2072
2073         register_irq(edev->irq, igbeinterrupt, edev, edev->tbdf);
2074         return 0;
2075 }
2076
2077 linker_func_3(etherigbelink)
2078 {
2079         addethercard("i82543", igbepnp);
2080         addethercard("igbe", igbepnp);
2081 }