9ns: Support rename
[akaros.git] / kern / drivers / net / etherigbe.c
1 /* This file is part of the UCB release of Plan 9. It is subject to the license
2  * terms in the LICENSE file found in the top-level directory of this
3  * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
4  * part of the UCB release of Plan 9, including this file, may be copied,
5  * modified, propagated, or distributed except according to the terms contained
6  * in the LICENSE file. */
7
8 /*
9  * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
10  * as found on the Intel PRO/1000 series of adapters:
11  *      82543GC Intel PRO/1000 T
12  *      82544EI Intel PRO/1000 XT
13  *      82540EM Intel PRO/1000 MT
14  *      82541[GP]I
15  *      82547GI
16  *      82546GB
17  *      82546EB
18  * To Do:
19  *      finish autonegotiation code;
20  *      integrate fiber stuff back in (this ONLY handles
21  *      the CAT5 cards at the moment);
22  *      add checksum-offload;
23  *      add tuning control via ctl file;
24  *      this driver is little-endian specific.
25  *
26  * Modified by brho:
27  *      ported to Akaros
28  *      fixed mii bugs (allocation, startup, miirw, etc)
29  *      fixed CLS bug (continue -> break)
30  *      made sure igbepci only runs once, even if it fails */
31
32 #include <slab.h>
33 #include <kmalloc.h>
34 #include <kref.h>
35 #include <string.h>
36 #include <stdio.h>
37 #include <assert.h>
38 #include <error.h>
39 #include <cpio.h>
40 #include <pmap.h>
41 #include <smp.h>
42 #include <arch/pci.h>
43 #include <net/ip.h>
44 #include <ns.h>
45 #include "ethermii.h"
46
47 #define ilock(x) spin_lock_irqsave(x)
48 #define iunlock(x) spin_unlock_irqsave(x)
49
50 enum {
51         i82542          = (0x1000<<16)|0x8086,
52         i82543gc        = (0x1004<<16)|0x8086,
53         i82544ei        = (0x1008<<16)|0x8086,
54         i82544eif       = (0x1009<<16)|0x8086,
55         i82544gc        = (0x100d<<16)|0x8086,
56         i82540em        = (0x100E<<16)|0x8086,
57         i82540eplp      = (0x101E<<16)|0x8086,
58         i82545em        = (0x100F<<16)|0x8086,
59         i82545gmc       = (0x1026<<16)|0x8086,
60         i82547ei        = (0x1019<<16)|0x8086,
61         i82547gi        = (0x1075<<16)|0x8086,
62         i82541ei        = (0x1013<<16)|0x8086,
63         i82541gi        = (0x1076<<16)|0x8086,
64         i82541gi2       = (0x1077<<16)|0x8086,
65         i82541pi        = (0x107c<<16)|0x8086,
66         i82546gb        = (0x1079<<16)|0x8086,
67         i82546eb        = (0x1010<<16)|0x8086,
68 };
69
70 enum {
71         Ctrl            = 0x00000000,   /* Device Control */
72         Ctrldup         = 0x00000004,   /* Device Control Duplicate */
73         Status          = 0x00000008,   /* Device Status */
74         Eecd            = 0x00000010,   /* EEPROM/Flash Control/Data */
75         Ctrlext         = 0x00000018,   /* Extended Device Control */
76         Mdic            = 0x00000020,   /* MDI Control */
77         Fcal            = 0x00000028,   /* Flow Control Address Low */
78         Fcah            = 0x0000002C,   /* Flow Control Address High */
79         Fct             = 0x00000030,   /* Flow Control Type */
80         Icr             = 0x000000C0,   /* Interrupt Cause Read */
81         Ics             = 0x000000C8,   /* Interrupt Cause Set */
82         Ims             = 0x000000D0,   /* Interrupt Mask Set/Read */
83         Imc             = 0x000000D8,   /* Interrupt mask Clear */
84         Rctl            = 0x00000100,   /* Receive Control */
85         Fcttv           = 0x00000170,   /* Flow Control Transmit Timer Value */
86         Txcw            = 0x00000178,   /* Transmit Configuration Word */
87         Rxcw            = 0x00000180,   /* Receive Configuration Word */
88         /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
89         Tctl            = 0x00000400,   /* Transmit Control */
90         Tipg            = 0x00000410,   /* Transmit IPG */
91         Tbt             = 0x00000448,   /* Transmit Burst Timer */
92         Ait             = 0x00000458,   /* Adaptive IFS Throttle */
93         Fcrtl           = 0x00002160,   /* Flow Control RX Threshold Low */
94         Fcrth           = 0x00002168,   /* Flow Control Rx Threshold High */
95         Rdfh            = 0x00002410,   /* Receive data fifo head */
96         Rdft            = 0x00002418,   /* Receive data fifo tail */
97         Rdfhs           = 0x00002420,   /* Receive data fifo head saved */
98         Rdfts           = 0x00002428,   /* Receive data fifo tail saved */
99         Rdfpc           = 0x00002430,   /* Receive data fifo packet count */
100         Rdbal           = 0x00002800,   /* Rd Base Address Low */
101         Rdbah           = 0x00002804,   /* Rd Base Address High */
102         Rdlen           = 0x00002808,   /* Receive Descriptor Length */
103         Rdh             = 0x00002810,   /* Receive Descriptor Head */
104         Rdt             = 0x00002818,   /* Receive Descriptor Tail */
105         Rdtr            = 0x00002820,   /* Receive Descriptor Timer Ring */
106         Rxdctl          = 0x00002828,   /* Receive Descriptor Control */
107         Radv            = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
108         Txdmac          = 0x00003000,   /* Transfer DMA Control */
109         Ett             = 0x00003008,   /* Early Transmit Control */
110         Tdfh            = 0x00003410,   /* Transmit data fifo head */
111         Tdft            = 0x00003418,   /* Transmit data fifo tail */
112         Tdfhs           = 0x00003420,   /* Transmit data Fifo Head saved */
113         Tdfts           = 0x00003428,   /* Transmit data fifo tail saved */
114         Tdfpc           = 0x00003430,   /* Trasnmit data Fifo packet count */
115         Tdbal           = 0x00003800,   /* Td Base Address Low */
116         Tdbah           = 0x00003804,   /* Td Base Address High */
117         Tdlen           = 0x00003808,   /* Transmit Descriptor Length */
118         Tdh             = 0x00003810,   /* Transmit Descriptor Head */
119         Tdt             = 0x00003818,   /* Transmit Descriptor Tail */
120         Tidv            = 0x00003820,   /* Transmit Interrupt Delay Value */
121         Txdctl          = 0x00003828,   /* Transmit Descriptor Control */
122         Tadv            = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
123
124         Statistics      = 0x00004000,   /* Start of Statistics Area */
125         Gorcl           = 0x88/4,       /* Good Octets Received Count */
126         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
127         Torl            = 0xC0/4,       /* Total Octets Received */
128         Totl            = 0xC8/4,       /* Total Octets Transmitted */
129         Nstatistics     = 64,
130
131         Rxcsum          = 0x00005000,   /* Receive Checksum Control */
132         Mta             = 0x00005200,   /* Multicast Table Array */
133         Ral             = 0x00005400,   /* Receive Address Low */
134         Rah             = 0x00005404,   /* Receive Address High */
135         Manc            = 0x00005820,   /* Management Control */
136 };
137
138 enum {                                  /* Ctrl */
139         Bem             = 0x00000002,   /* Big Endian Mode */
140         Prior           = 0x00000004,   /* Priority on the PCI bus */
141         Lrst            = 0x00000008,   /* Link Reset */
142         Asde            = 0x00000020,   /* Auto-Speed Detection Enable */
143         Slu             = 0x00000040,   /* Set Link Up */
144         Ilos            = 0x00000080,   /* Invert Loss of Signal (LOS) */
145         SspeedMASK      = 0x00000300,   /* Speed Selection */
146         SspeedSHIFT     = 8,
147         Sspeed10        = 0x00000000,   /* 10Mb/s */
148         Sspeed100       = 0x00000100,   /* 100Mb/s */
149         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
150         Frcspd          = 0x00000800,   /* Force Speed */
151         Frcdplx         = 0x00001000,   /* Force Duplex */
152         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
153         SwdpinsloSHIFT  = 18,
154         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
155         SwdpioloSHIFT   = 22,
156         Devrst          = 0x04000000,   /* Device Reset */
157         Rfce            = 0x08000000,   /* Receive Flow Control Enable */
158         Tfce            = 0x10000000,   /* Transmit Flow Control Enable */
159         Vme             = 0x40000000,   /* VLAN Mode Enable */
160 };
161
162 /*
163  * can't find Tckok nor Rbcok in any Intel docs,
164  * but even 82543gc docs define Lanid.
165  */
166 enum {                                  /* Status */
167         Lu              = 0x00000002,   /* Link Up */
168         Lanid           = 0x0000000C,   /* mask for Lan ID. (function id) */
169 //      Tckok           = 0x00000004,   /* Transmit clock is running */
170 //      Rbcok           = 0x00000008,   /* Receive clock is running */
171         Txoff           = 0x00000010,   /* Transmission Paused */
172         Tbimode         = 0x00000020,   /* TBI Mode Indication */
173         LspeedMASK      = 0x000000C0,   /* Link Speed Setting */
174         LspeedSHIFT     = 6,
175         Lspeed10        = 0x00000000,   /* 10Mb/s */
176         Lspeed100       = 0x00000040,   /* 100Mb/s */
177         Lspeed1000      = 0x00000080,   /* 1000Mb/s */
178         Mtxckok         = 0x00000400,   /* MTX clock is running */
179         Pci66           = 0x00000800,   /* PCI Bus speed indication */
180         Bus64           = 0x00001000,   /* PCI Bus width indication */
181         Pcixmode        = 0x00002000,   /* PCI-X mode */
182         PcixspeedMASK   = 0x0000C000,   /* PCI-X bus speed */
183         PcixspeedSHIFT  = 14,
184         Pcix66          = 0x00000000,   /* 50-66MHz */
185         Pcix100         = 0x00004000,   /* 66-100MHz */
186         Pcix133         = 0x00008000,   /* 100-133MHz */
187 };
188
189 enum {                                  /* Ctrl and Status */
190         Fd              = 0x00000001,   /* Full-Duplex */
191         AsdvMASK        = 0x00000300,
192         AsdvSHIFT       = 8,
193         Asdv10          = 0x00000000,   /* 10Mb/s */
194         Asdv100         = 0x00000100,   /* 100Mb/s */
195         Asdv1000        = 0x00000200,   /* 1000Mb/s */
196 };
197
198 enum {                                  /* Eecd */
199         Sk              = 0x00000001,   /* Clock input to the EEPROM */
200         Cs              = 0x00000002,   /* Chip Select */
201         Di              = 0x00000004,   /* Data Input to the EEPROM */
202         Do              = 0x00000008,   /* Data Output from the EEPROM */
203         Areq            = 0x00000040,   /* EEPROM Access Request */
204         Agnt            = 0x00000080,   /* EEPROM Access Grant */
205         Eepresent       = 0x00000100,   /* EEPROM Present */
206         Eesz256         = 0x00000200,   /* EEPROM is 256 words not 64 */
207         Eeszaddr        = 0x00000400,   /* EEPROM size for 8254[17] */
208         Spi             = 0x00002000,   /* EEPROM is SPI not Microwire */
209 };
210
211 enum {                                  /* Ctrlext */
212         Gpien           = 0x0000000F,   /* General Purpose Interrupt Enables */
213         SwdpinshiMASK   = 0x000000F0,   /* Software Defined Pins - hi nibble */
214         SwdpinshiSHIFT  = 4,
215         SwdpiohiMASK    = 0x00000F00,   /* Software Defined Pins - I or O */
216         SwdpiohiSHIFT   = 8,
217         Asdchk          = 0x00001000,   /* ASD Check */
218         Eerst           = 0x00002000,   /* EEPROM Reset */
219         Ips             = 0x00004000,   /* Invert Power State */
220         Spdbyps         = 0x00008000,   /* Speed Select Bypass */
221 };
222
223 enum {                                  /* EEPROM content offsets */
224         Ea              = 0x00,         /* Ethernet Address */
225         Cf              = 0x03,         /* Compatibility Field */
226         Pba             = 0x08,         /* Printed Board Assembly number */
227         Icw1            = 0x0A,         /* Initialization Control Word 1 */
228         Sid             = 0x0B,         /* Subsystem ID */
229         Svid            = 0x0C,         /* Subsystem Vendor ID */
230         Did             = 0x0D,         /* Device ID */
231         Vid             = 0x0E,         /* Vendor ID */
232         Icw2            = 0x0F,         /* Initialization Control Word 2 */
233 };
234
235 enum {                                  /* Mdic */
236         MDIdMASK        = 0x0000FFFF,   /* Data */
237         MDIdSHIFT       = 0,
238         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
239         MDIrSHIFT       = 16,
240         MDIpMASK        = 0x03E00000,   /* PHY Address */
241         MDIpSHIFT       = 21,
242         MDIwop          = 0x04000000,   /* Write Operation */
243         MDIrop          = 0x08000000,   /* Read Operation */
244         MDIready        = 0x10000000,   /* End of Transaction */
245         MDIie           = 0x20000000,   /* Interrupt Enable */
246         MDIe            = 0x40000000,   /* Error */
247 };
248
249 enum {                                  /* Icr, Ics, Ims, Imc */
250         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
251         Txqe            = 0x00000002,   /* Transmit Queue Empty */
252         Lsc             = 0x00000004,   /* Link Status Change */
253         Rxseq           = 0x00000008,   /* Receive Sequence Error */
254         Rxdmt0          = 0x00000010,   /* Rd Minimum Threshold Reached */
255         Rxo             = 0x00000040,   /* Receiver Overrun */
256         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
257         Mdac            = 0x00000200,   /* MDIO Access Completed */
258         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
259         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
260         Gpi1            = 0x00001000,
261         Gpi2            = 0x00002000,
262         Gpi3            = 0x00004000,
263 };
264
265 /*
266  * The Mdic register isn't implemented on the 82543GC,
267  * the software defined pins are used instead.
268  * These definitions work for the Intel PRO/1000 T Server Adapter.
269  * The direction pin bits are read from the EEPROM.
270  */
271 enum {
272         Mdd             = ((1<<2)<<SwdpinsloSHIFT),     /* data */
273         Mddo            = ((1<<2)<<SwdpioloSHIFT),      /* pin direction */
274         Mdc             = ((1<<3)<<SwdpinsloSHIFT),     /* clock */
275         Mdco            = ((1<<3)<<SwdpioloSHIFT),      /* pin direction */
276         Mdr             = ((1<<0)<<SwdpinshiSHIFT),     /* reset */
277         Mdro            = ((1<<0)<<SwdpiohiSHIFT),      /* pin direction */
278 };
279
280 enum {                                  /* Txcw */
281         TxcwFd          = 0x00000020,   /* Full Duplex */
282         TxcwHd          = 0x00000040,   /* Half Duplex */
283         TxcwPauseMASK   = 0x00000180,   /* Pause */
284         TxcwPauseSHIFT  = 7,
285         TxcwPs          = (1<<TxcwPauseSHIFT),  /* Pause Supported */
286         TxcwAs          = (2<<TxcwPauseSHIFT),  /* Asymmetric FC desired */
287         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
288         TxcwRfiSHIFT    = 12,
289         TxcwNpr         = 0x00008000,   /* Next Page Request */
290         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
291         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
292 };
293
294 enum {                                  /* Rxcw */
295         Rxword          = 0x0000FFFF,   /* Data from auto-negotiation process */
296         Rxnocarrier     = 0x04000000,   /* Carrier Sense indication */
297         Rxinvalid       = 0x08000000,   /* Invalid Symbol during configuration */
298         Rxchange        = 0x10000000,   /* Change to the Rxword indication */
299         Rxconfig        = 0x20000000,   /* /C/ order set reception indication */
300         Rxsync          = 0x40000000,   /* Lost bit synchronization indication */
301         Anc             = 0x80000000,   /* Auto Negotiation Complete */
302 };
303
304 enum {                                  /* Rctl */
305         Rrst            = 0x00000001,   /* Receiver Software Reset */
306         Ren             = 0x00000002,   /* Receiver Enable */
307         Sbp             = 0x00000004,   /* Store Bad Packets */
308         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
309         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
310         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
311         LbmMASK         = 0x000000C0,   /* Loopback Mode */
312         LbmOFF          = 0x00000000,   /* No Loopback */
313         LbmTBI          = 0x00000040,   /* TBI Loopback */
314         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
315         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
316         RdtmsMASK       = 0x00000300,   /* Rd Minimum Threshold Size */
317         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
318         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
319         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
320         MoMASK          = 0x00003000,   /* Multicast Offset */
321         Mo47b36         = 0x00000000,   /* bits [47:36] of received address */
322         Mo46b35         = 0x00001000,   /* bits [46:35] of received address */
323         Mo45b34         = 0x00002000,   /* bits [45:34] of received address */
324         Mo43b32         = 0x00003000,   /* bits [43:32] of received address */
325         Bam             = 0x00008000,   /* Broadcast Accept Mode */
326         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
327         Bsize2048       = 0x00000000,   /* Bsex = 0 */
328         Bsize1024       = 0x00010000,   /* Bsex = 0 */
329         Bsize512        = 0x00020000,   /* Bsex = 0 */
330         Bsize256        = 0x00030000,   /* Bsex = 0 */
331         Bsize16384      = 0x00010000,   /* Bsex = 1 */
332         Vfe             = 0x00040000,   /* VLAN Filter Enable */
333         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
334         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
335         Dpf             = 0x00400000,   /* Discard Pause Frames */
336         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
337         Bsex            = 0x02000000,   /* Buffer Size Extension */
338         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
339 };
340
341 enum {                                  /* Tctl */
342         Trst            = 0x00000001,   /* Transmitter Software Reset */
343         Ten             = 0x00000002,   /* Transmit Enable */
344         Psp             = 0x00000008,   /* Pad Short Packets */
345         CtMASK          = 0x00000FF0,   /* Collision Threshold */
346         CtSHIFT         = 4,
347         ColdMASK        = 0x003FF000,   /* Collision Distance */
348         ColdSHIFT       = 12,
349         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
350         Pbe             = 0x00800000,   /* Packet Burst Enable */
351         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
352         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
353 };
354
355 enum {                                  /* [RT]xdctl */
356         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
357         PthreshSHIFT    = 0,
358         HthreshMASK     = 0x00003F00,   /* Host Threshold */
359         HthreshSHIFT    = 8,
360         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
361         WthreshSHIFT    = 16,
362         Gran            = 0x01000000,   /* Granularity */
363         LthreshMASK     = 0xFE000000,   /* Low Threshold */
364         LthreshSHIFT    = 25,
365 };
366
367 enum {                                  /* Rxcsum */
368         PcssMASK        = 0x000000FF,   /* Packet Checksum Start */
369         PcssSHIFT       = 0,
370         Ipofl           = 0x00000100,   /* IP Checksum Off-load Enable */
371         Tuofl           = 0x00000200,   /* TCP/UDP Checksum Off-load Enable */
372 };
373
374 enum {                                  /* Manc */
375         Arpen           = 0x00002000,   /* Enable ARP Request Filtering */
376 };
377
378 enum {                                  /* Receive Delay Timer Ring */
379         DelayMASK       = 0x0000FFFF,   /* delay timer in 1.024nS increments */
380         DelaySHIFT      = 0,
381         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
382 };
383
384 typedef struct Rd {                     /* Receive Descriptor */
385         unsigned int    addr[2];
386         uint16_t        length;
387         uint16_t        checksum;
388         uint8_t status;
389         uint8_t errors;
390         uint16_t        special;
391 } Rd;
392
393 enum {                                  /* Rd status */
394         Rdd             = 0x01,         /* Descriptor Done */
395         Reop            = 0x02,         /* End of Packet */
396         Ixsm            = 0x04,         /* Ignore Checksum Indication */
397         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
398         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
399         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
400         Pif             = 0x80,         /* Passed in-exact filter */
401 };
402
403 enum {                                  /* Rd errors */
404         Ce              = 0x01,         /* CRC Error or Alignment Error */
405         Se              = 0x02,         /* Symbol Error */
406         Seq             = 0x04,         /* Sequence Error */
407         Cxe             = 0x10,         /* Carrier Extension Error */
408         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
409         Ipe             = 0x40,         /* IP Checksum Error */
410         Rxe             = 0x80,         /* RX Data Error */
411 };
412
413 typedef struct Td Td;
414 struct Td {                             /* Transmit Descriptor */
415         union {
416                 unsigned int    addr[2];        /* Data */
417                 struct {                /* Context */
418                         uint8_t ipcss;
419                         uint8_t ipcso;
420                         uint16_t        ipcse;
421                         uint8_t tucss;
422                         uint8_t tucso;
423                         uint16_t        tucse;
424                 };
425         };
426         unsigned int    control;
427         unsigned int    status;
428 };
429
430 enum {                                  /* Td control */
431         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
432         LenSHIFT        = 0,
433         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
434         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
435         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
436         Teop            = 0x01000000,   /* End of Packet (DD) */
437         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
438         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
439         Tse             = 0x04000000,   /* TCP Segmentation Enable */
440         Rs              = 0x08000000,   /* Report Status */
441         Rps             = 0x10000000,   /* Report Status Sent */
442         Dext            = 0x20000000,   /* Descriptor Extension */
443         Vle             = 0x40000000,   /* VLAN Packet Enable */
444         Ide             = 0x80000000,   /* Interrupt Delay Enable */
445 };
446
447 enum {                                  /* Td status */
448         Tdd             = 0x00000001,   /* Descriptor Done */
449         Ec              = 0x00000002,   /* Excess Collisions */
450         Lc              = 0x00000004,   /* Late Collision */
451         Tu              = 0x00000008,   /* Transmit Underrun */
452         Iixsm           = 0x00000100,   /* Insert IP Checksum */
453         Itxsm           = 0x00000200,   /* Insert TCP/UDP Checksum */
454         HdrlenMASK      = 0x0000FF00,   /* Header Length (Tse) */
455         HdrlenSHIFT     = 8,
456         VlanMASK        = 0x0FFF0000,   /* VLAN Identifier */
457         VlanSHIFT       = 16,
458         Tcfi            = 0x10000000,   /* Canonical Form Indicator */
459         PriMASK         = 0xE0000000,   /* User Priority */
460         PriSHIFT        = 29,
461         MssMASK         = 0xFFFF0000,   /* Maximum Segment Size (Tse) */
462         MssSHIFT        = 16,
463 };
464
465 enum {
466         Nrd             = 256,          /* multiple of 8 */
467         Ntd             = 64,           /* multiple of 8 */
468         Rbsz            = 2048,
469 };
470
471 struct ctlr {
472         int     port;
473         struct pci_device *pci;
474         struct ctlr*    next;
475         struct ether*   edev;
476         int     active;
477         int     started;
478         int     id;
479         int     cls;
480         uint16_t        eeprom[0x40];
481
482         qlock_t alock;                  /* attach */
483         void*   alloc;                  /* receive/transmit descriptors */
484         int     nrd;
485         int     ntd;
486
487         int*    nic;
488         spinlock_t      imlock;
489         int     im;                     /* interrupt mask */
490
491         struct mii*     mii;
492         struct rendez   lrendez;
493         int     lim;
494
495         int     link;
496
497         qlock_t slock;
498         unsigned int    statistics[Nstatistics];
499         unsigned int    lsleep;
500         unsigned int    lintr;
501         unsigned int    rsleep;
502         unsigned int    rintr;
503         unsigned int    txdw;
504         unsigned int    tintr;
505         unsigned int    ixsm;
506         unsigned int    ipcs;
507         unsigned int    tcpcs;
508
509         uint8_t ra[Eaddrlen];           /* receive address */
510         uint32_t        mta[128];               /* multicast table array */
511
512         struct rendez   rrendez;
513         int     rim;
514         int     rdfree;
515         Rd*     rdba;                   /* receive descriptor base address */
516         struct block**  rb;                     /* receive buffers */
517         int     rdh;                    /* receive descriptor head */
518         int     rdt;                    /* receive descriptor tail */
519         int     rdtr;                   /* receive delay timer ring value */
520
521         spinlock_t      tlock;
522         int     tbusy;
523         int     tdfree;
524         Td*     tdba;                   /* transmit descriptor base address */
525         struct block**  tb;                     /* transmit buffers */
526         int     tdh;                    /* transmit descriptor head */
527         int     tdt;                    /* transmit descriptor tail */
528
529         int     txcw;
530         int     fcrtl;
531         int     fcrth;
532 };
533
534 static inline uint32_t csr32r(struct ctlr *c, uintptr_t reg)
535 {
536         return read_mmreg32((uintptr_t)(c->nic + (reg / 4)));
537 }
538
539 static inline void csr32w(struct ctlr *c, uintptr_t reg, uint32_t val)
540 {
541         write_mmreg32((uintptr_t)(c->nic + (reg / 4)), val);
542 }
543
544 static struct ctlr* igbectlrhead;
545 static struct ctlr* igbectlrtail;
546
547 static char* statistics[Nstatistics] = {
548         "CRC Error",
549         "Alignment Error",
550         "Symbol Error",
551         "RX Error",
552         "Missed Packets",
553         "Single Collision",
554         "Excessive Collisions",
555         "Multiple Collision",
556         "Late Collisions",
557         NULL,
558         "Collision",
559         "Transmit Underrun",
560         "Defer",
561         "Transmit - No CRS",
562         "Sequence Error",
563         "Carrier Extension Error",
564         "Receive Error Length",
565         NULL,
566         "XON Received",
567         "XON Transmitted",
568         "XOFF Received",
569         "XOFF Transmitted",
570         "FC Received Unsupported",
571         "Packets Received (64 Bytes)",
572         "Packets Received (65-127 Bytes)",
573         "Packets Received (128-255 Bytes)",
574         "Packets Received (256-511 Bytes)",
575         "Packets Received (512-1023 Bytes)",
576         "Packets Received (1024-1522 Bytes)",
577         "Good Packets Received",
578         "Broadcast Packets Received",
579         "Multicast Packets Received",
580         "Good Packets Transmitted",
581         NULL,
582         "Good Octets Received",
583         NULL,
584         "Good Octets Transmitted",
585         NULL,
586         NULL,
587         NULL,
588         "Receive No Buffers",
589         "Receive Undersize",
590         "Receive Fragment",
591         "Receive Oversize",
592         "Receive Jabber",
593         NULL,
594         NULL,
595         NULL,
596         "Total Octets Received",
597         NULL,
598         "Total Octets Transmitted",
599         NULL,
600         "Total Packets Received",
601         "Total Packets Transmitted",
602         "Packets Transmitted (64 Bytes)",
603         "Packets Transmitted (65-127 Bytes)",
604         "Packets Transmitted (128-255 Bytes)",
605         "Packets Transmitted (256-511 Bytes)",
606         "Packets Transmitted (512-1023 Bytes)",
607         "Packets Transmitted (1024-1522 Bytes)",
608         "Multicast Packets Transmitted",
609         "Broadcast Packets Transmitted",
610         "TCP Segmentation Context Transmitted",
611         "TCP Segmentation Context Fail",
612 };
613
614 static void igbe_print_rd(struct Rd *rd)
615 {
616         printk("Rd %p: stat 0x%02x, err 0x%02x, len 0x%04x, check 0x%04x, "
617                "spec 0x%04x, addr[1] 0x%08x, addr[0] 0x%08x\n", rd,
618                    rd->status,
619                    rd->errors,
620                    rd->length,
621                    rd->checksum,
622                    rd->special,
623                    rd->addr[1],
624                    rd->addr[0]);
625 }
626
627 static long
628 igbeifstat(struct ether* edev, void* a, long n, uint32_t offset)
629 {
630         struct ctlr *ctlr;
631         char *p, *s;
632         int i, l, r;
633         uint64_t tuvl, ruvl;
634
635         ctlr = edev->ctlr;
636         qlock(&ctlr->slock);
637         p = kzmalloc(READSTR, 0);
638         if(p == NULL) {
639                 qunlock(&ctlr->slock);
640                 error(ENOMEM, ERROR_FIXME);
641         }
642         l = 0;
643         for(i = 0; i < Nstatistics; i++){
644                 r = csr32r(ctlr, Statistics+i*4);
645                 if((s = statistics[i]) == NULL)
646                         continue;
647                 switch(i){
648                 case Gorcl:
649                 case Gotcl:
650                 case Torl:
651                 case Totl:
652                         ruvl = r;
653                         ruvl += ((uint64_t)csr32r(ctlr, Statistics+(i+1)*4))<<32;
654                         tuvl = ruvl;
655                         tuvl += ctlr->statistics[i];
656                         tuvl += ((uint64_t)ctlr->statistics[i+1])<<32;
657                         if(tuvl == 0)
658                                 continue;
659                         ctlr->statistics[i] = tuvl;
660                         ctlr->statistics[i+1] = tuvl>>32;
661                         l += snprintf(p+l, READSTR-l, "%s: %llud %llud\n",
662                                 s, tuvl, ruvl);
663                         i++;
664                         break;
665
666                 default:
667                         ctlr->statistics[i] += r;
668                         if(ctlr->statistics[i] == 0)
669                                 continue;
670                         l += snprintf(p+l, READSTR-l, "%s: %ud %ud\n",
671                                 s, ctlr->statistics[i], r);
672                         break;
673                 }
674         }
675
676         l += snprintf(p+l, READSTR-l, "lintr: %ud %ud\n",
677                 ctlr->lintr, ctlr->lsleep);
678         l += snprintf(p+l, READSTR-l, "rintr: %ud %ud\n",
679                 ctlr->rintr, ctlr->rsleep);
680         l += snprintf(p+l, READSTR-l, "tintr: %ud %ud\n",
681                 ctlr->tintr, ctlr->txdw);
682         l += snprintf(p+l, READSTR-l, "ixcs: %ud %ud %ud\n",
683                 ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
684         l += snprintf(p+l, READSTR-l, "rdtr: %ud\n", ctlr->rdtr);
685         l += snprintf(p+l, READSTR-l, "Ctrlext: %08x\n", csr32r(ctlr, Ctrlext));
686
687         l += snprintf(p+l, READSTR-l, "eeprom:");
688         for(i = 0; i < 0x40; i++){
689                 if(i && ((i & 0x07) == 0))
690                         l += snprintf(p+l, READSTR-l, "\n       ");
691                 l += snprintf(p+l, READSTR-l, " %4.4uX", ctlr->eeprom[i]);
692         }
693         l += snprintf(p+l, READSTR-l, "\n");
694
695         if(ctlr->mii != NULL && ctlr->mii->curphy != NULL){
696                 l += snprintf(p+l, READSTR-l, "phy:   ");
697                 for(i = 0; i < NMiiPhyr; i++){
698                         if(i && ((i & 0x07) == 0))
699                                 l += snprintf(p+l, READSTR-l, "\n       ");
700                         r = miimir(ctlr->mii, i);
701                         l += snprintf(p+l, READSTR-l, " %4.4uX", r);
702                 }
703                 snprintf(p+l, READSTR-l, "\n");
704         }
705         n = readstr(offset, a, n, p);
706         kfree(p);
707         qunlock(&ctlr->slock);
708
709         return n;
710 }
711
712 enum {
713         CMrdtr,
714 };
715
716 static struct cmdtab igbectlmsg[] = {
717         {CMrdtr,        "rdtr", 2},
718 };
719
720 static long
721 igbectl(struct ether* edev, void* buf, long n)
722 {
723         ERRSTACK(2);
724         int v;
725         char *p;
726         struct ctlr *ctlr;
727         struct cmdbuf *cb;
728         struct cmdtab *ct;
729
730         if((ctlr = edev->ctlr) == NULL)
731                 error(ENODEV, ERROR_FIXME);
732
733         cb = parsecmd(buf, n);
734         if(waserror()){
735                 kfree(cb);
736                 nexterror();
737         }
738
739         ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
740         switch(ct->index){
741         case CMrdtr:
742                 v = strtol(cb->f[1], &p, 0);
743                 if(v < 0 || p == cb->f[1] || v > 0xFFFF)
744                         error(EINVAL, ERROR_FIXME);
745                 ctlr->rdtr = v;
746                 csr32w(ctlr, Rdtr, Fpd|v);
747                 break;
748         }
749         kfree(cb);
750         poperror();
751
752         return n;
753 }
754
755 static void
756 igbepromiscuous(void* arg, int on)
757 {
758         int rctl;
759         struct ctlr *ctlr;
760         struct ether *edev;
761
762         edev = arg;
763         ctlr = edev->ctlr;
764
765         rctl = csr32r(ctlr, Rctl);
766         rctl &= ~MoMASK;
767         rctl |= Mo47b36;
768         if(on)
769                 rctl |= Upe|Mpe;
770         else
771                 rctl &= ~(Upe|Mpe);
772         csr32w(ctlr, Rctl, rctl|Mpe);   /* temporarily keep Mpe on */
773 }
774
775 static void
776 igbemulticast(void* arg, uint8_t* addr, int add)
777 {
778         int bit, x;
779         struct ctlr *ctlr;
780         struct ether *edev;
781
782         edev = arg;
783         ctlr = edev->ctlr;
784
785         x = addr[5]>>1;
786         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
787         /*
788          * multiple ether addresses can hash to the same filter bit,
789          * so it's never safe to clear a filter bit.
790          * if we want to clear filter bits, we need to keep track of
791          * all the multicast addresses in use, clear all the filter bits,
792          * then set the ones corresponding to in-use addresses.
793          */
794         if(add)
795                 ctlr->mta[x] |= 1<<bit;
796 //      else
797 //              ctlr->mta[x] &= ~(1<<bit);
798
799         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
800 }
801
802 static void
803 igbeim(struct ctlr* ctlr, int im)
804 {
805         ilock(&ctlr->imlock);
806         ctlr->im |= im;
807         csr32w(ctlr, Ims, ctlr->im);
808         iunlock(&ctlr->imlock);
809 }
810
811 static int
812 igbelim(void* ctlr)
813 {
814         return ((struct ctlr*)ctlr)->lim != 0;
815 }
816
817 static void
818 igbelproc(void* arg)
819 {
820         struct ctlr *ctlr;
821         struct ether *edev;
822         struct miiphy *phy;
823         int ctrl, r;
824
825         edev = arg;
826         ctlr = edev->ctlr;
827         for(;;){
828                 /* plan9 originally had a busy loop here (just called continue).  though
829                  * either you have the mii or you don't.  i don't think it'll magically
830                  * show up later (it should have been initialized during pnp/pci, which
831                  * is before attach, which is before lproc).  -brho */
832                 if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
833                         printk("[kernel] igbelproc can't find a mii/curphy, aborting!\n");
834                         /* name alloc'd in attach */
835                         kfree(per_cpu_info[core_id()].cur_kthread->name);
836                         return;
837                 }
838                 /*
839                  * To do:
840                  *      logic to manage status change,
841                  *      this is incomplete but should work
842                  *      one time to set up the hardware.
843                  *
844                  *      MiiPhy.speed, etc. should be in Mii.
845                  */
846                 if(miistatus(ctlr->mii) < 0)
847                         //continue;     /* this comment out was plan9, not brho */
848                         goto enable;
849
850                 phy = ctlr->mii->curphy;
851                 ctrl = csr32r(ctlr, Ctrl);
852
853                 switch(ctlr->id){
854                 case i82543gc:
855                 case i82544ei:
856                 case i82544eif:
857                 default:
858                         if(!(ctrl & Asde)){
859                                 ctrl &= ~(SspeedMASK|Ilos|Fd);
860                                 ctrl |= Frcdplx|Frcspd;
861                                 if(phy->speed == 1000)
862                                         ctrl |= Sspeed1000;
863                                 else if(phy->speed == 100)
864                                         ctrl |= Sspeed100;
865                                 if(phy->fd)
866                                         ctrl |= Fd;
867                         }
868                         break;
869
870                 case i82540em:
871                 case i82540eplp:
872                 case i82547gi:
873                 case i82541gi:
874                 case i82541gi2:
875                 case i82541pi:
876                         break;
877                 }
878
879                 /*
880                  * Collision Distance.
881                  */
882                 r = csr32r(ctlr, Tctl);
883                 r &= ~ColdMASK;
884                 if(phy->fd)
885                         r |= 64<<ColdSHIFT;
886                 else
887                         r |= 512<<ColdSHIFT;
888                 csr32w(ctlr, Tctl, r);
889
890                 /*
891                  * Flow control.
892                  */
893                 if(phy->rfc)
894                         ctrl |= Rfce;
895                 if(phy->tfc)
896                         ctrl |= Tfce;
897                 csr32w(ctlr, Ctrl, ctrl);
898
899 enable:
900                 netif_carrier_on(edev);
901                 ctlr->lim = 0;
902                 igbeim(ctlr, Lsc);
903
904                 ctlr->lsleep++;
905                 rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
906         }
907 }
908
909 static void
910 igbetxinit(struct ctlr* ctlr)
911 {
912         int i, r;
913         struct block *bp;
914
915         csr32w(ctlr, Tctl, (0x0F<<CtSHIFT)|Psp|(66<<ColdSHIFT));
916         switch(ctlr->id){
917         default:
918                 r = 6;
919                 break;
920         case i82543gc:
921         case i82544ei:
922         case i82544eif:
923         case i82544gc:
924         case i82540em:
925         case i82540eplp:
926         case i82541ei:
927         case i82541gi:
928         case i82541gi2:
929         case i82541pi:
930         case i82545em:
931         case i82545gmc:
932         case i82546gb:
933         case i82546eb:
934         case i82547ei:
935         case i82547gi:
936                 r = 8;
937                 break;
938         }
939         csr32w(ctlr, Tipg, (6<<20)|(8<<10)|r);
940         csr32w(ctlr, Ait, 0);
941         csr32w(ctlr, Txdmac, 0);
942         csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
943         csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
944         csr32w(ctlr, Tdlen, ctlr->ntd*sizeof(Td));
945         ctlr->tdh = PREV_RING(0, ctlr->ntd);
946         csr32w(ctlr, Tdh, 0);
947         ctlr->tdt = 0;
948         csr32w(ctlr, Tdt, 0);
949
950         for(i = 0; i < ctlr->ntd; i++){
951                 if((bp = ctlr->tb[i]) != NULL){
952                         ctlr->tb[i] = NULL;
953                         freeb(bp);
954                 }
955                 memset(&ctlr->tdba[i], 0, sizeof(Td));
956         }
957         ctlr->tdfree = ctlr->ntd;
958
959         csr32w(ctlr, Tidv, 128);
960         r = (4<<WthreshSHIFT)|(4<<HthreshSHIFT)|(8<<PthreshSHIFT);
961
962         switch(ctlr->id){
963         default:
964                 break;
965         case i82540em:
966         case i82540eplp:
967         case i82547gi:
968         case i82545em:
969         case i82545gmc:
970         case i82546gb:
971         case i82546eb:
972         case i82541gi:
973         case i82541gi2:
974         case i82541pi:
975                 r = csr32r(ctlr, Txdctl);
976                 r &= ~WthreshMASK;
977                 r |= Gran|(4<<WthreshSHIFT);
978
979                 csr32w(ctlr, Tadv, 64);
980                 break;
981         }
982
983         csr32w(ctlr, Txdctl, r);
984
985         r = csr32r(ctlr, Tctl);
986         r |= Ten;
987         csr32w(ctlr, Tctl, r);
988 }
989
990 static void
991 igbetransmit(struct ether* edev)
992 {
993         Td *td;
994         struct block *bp;
995         struct ctlr *ctlr;
996         int tdh, tdt;
997
998         ctlr = edev->ctlr;
999
1000         ilock(&ctlr->tlock);
1001
1002         /*
1003          * Free any completed packets
1004          */
1005         tdh = ctlr->tdh;
1006         while(NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)){
1007                 if((bp = ctlr->tb[tdh]) != NULL){
1008                         ctlr->tb[tdh] = NULL;
1009                         freeb(bp);
1010                 }
1011                 memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1012                 tdh = NEXT_RING(tdh, ctlr->ntd);
1013         }
1014         ctlr->tdh = tdh;
1015
1016         /*
1017          * Try to fill the ring back up.
1018          */
1019         tdt = ctlr->tdt;
1020         while(NEXT_RING(tdt, ctlr->ntd) != tdh){
1021                 if((bp = qget(edev->oq)) == NULL)
1022                         break;
1023                 td = &ctlr->tdba[tdt];
1024                 td->addr[0] = paddr_low32(bp->rp);
1025                 td->addr[1] = paddr_high32(bp->rp);
1026                 td->control = ((BLEN(bp) & LenMASK)<<LenSHIFT);
1027                 td->control |= Dext|Ifcs|Teop|DtypeDD;
1028                 ctlr->tb[tdt] = bp;
1029                 tdt = NEXT_RING(tdt, ctlr->ntd);
1030                 if(NEXT_RING(tdt, ctlr->ntd) == tdh){
1031                         td->control |= Rs;
1032                         ctlr->txdw++;
1033                         ctlr->tdt = tdt;
1034                         csr32w(ctlr, Tdt, tdt);
1035                         igbeim(ctlr, Txdw);
1036                         break;
1037                 }
1038                 ctlr->tdt = tdt;
1039                 csr32w(ctlr, Tdt, tdt);
1040         }
1041
1042         iunlock(&ctlr->tlock);
1043 }
1044
1045 static void
1046 igbereplenish(struct ctlr* ctlr)
1047 {
1048         Rd *rd;
1049         int rdt;
1050         struct block *bp;
1051
1052         rdt = ctlr->rdt;
1053         while(NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh){
1054                 rd = &ctlr->rdba[rdt];
1055                 if(ctlr->rb[rdt] == NULL){
1056                         bp = block_alloc(Rbsz, MEM_ATOMIC);
1057                         if(bp == NULL){
1058                                 /* needs to be a safe print for interrupt level */
1059                                 printk("#l%d: igbereplenish: no available buffers\n",
1060                                         ctlr->edev->ctlrno);
1061                                 break;
1062                         }
1063                         ctlr->rb[rdt] = bp;
1064                         rd->addr[0] = paddr_low32(bp->rp);
1065                         rd->addr[1] = paddr_high32(bp->rp);
1066                 }
1067                 wmb();  /* ensure prev rd writes come before status = 0. */
1068                 rd->status = 0;
1069                 rdt = NEXT_RING(rdt, ctlr->nrd);
1070                 ctlr->rdfree++;
1071         }
1072         ctlr->rdt = rdt;
1073         csr32w(ctlr, Rdt, rdt);
1074 }
1075
1076 static void
1077 igberxinit(struct ctlr* ctlr)
1078 {
1079         int i;
1080         struct block *bp;
1081
1082         /* temporarily keep Mpe on */
1083         csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF|Mpe);
1084         csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1085         csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1086         csr32w(ctlr, Rdlen, ctlr->nrd*sizeof(Rd));
1087         ctlr->rdh = 0;
1088         csr32w(ctlr, Rdh, 0);
1089         ctlr->rdt = 0;
1090         csr32w(ctlr, Rdt, 0);
1091         ctlr->rdtr = 0;
1092         csr32w(ctlr, Rdtr, Fpd|0);
1093
1094         for(i = 0; i < ctlr->nrd; i++){
1095                 if((bp = ctlr->rb[i]) != NULL){
1096                         ctlr->rb[i] = NULL;
1097                         freeb(bp);
1098                 }
1099         }
1100         igbereplenish(ctlr);
1101
1102         switch(ctlr->id){
1103         case i82540em:
1104         case i82540eplp:
1105         case i82541gi:
1106         case i82541gi2:
1107         case i82541pi:
1108         case i82545em:
1109         case i82545gmc:
1110         case i82546gb:
1111         case i82546eb:
1112         case i82547gi:
1113                 csr32w(ctlr, Radv, 64);
1114                 break;
1115         }
1116         csr32w(ctlr, Rxdctl, (8<<WthreshSHIFT)|(8<<HthreshSHIFT)|4);
1117
1118         /*
1119          * Enable checksum offload.
1120          */
1121         csr32w(ctlr, Rxcsum, Tuofl|Ipofl|(ETHERHDRSIZE<<PcssSHIFT));
1122 }
1123
1124 static int
1125 igberim(void* ctlr)
1126 {
1127         return ((struct ctlr*)ctlr)->rim != 0;
1128 }
1129
1130 static void
1131 igberproc(void* arg)
1132 {
1133         Rd *rd;
1134         struct block *bp;
1135         struct ctlr *ctlr;
1136         int r, rdh;
1137         struct ether *edev;
1138
1139         edev = arg;
1140         ctlr = edev->ctlr;
1141
1142         igberxinit(ctlr);
1143         r = csr32r(ctlr, Rctl);
1144         r |= Ren;
1145         csr32w(ctlr, Rctl, r);
1146
1147         for(;;){
1148                 ctlr->rim = 0;
1149                 igbeim(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq);
1150                 ctlr->rsleep++;
1151                 rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1152
1153                 rdh = ctlr->rdh;
1154                 for(;;){
1155                         rd = &ctlr->rdba[rdh];
1156
1157                         if(!(rd->status & Rdd))
1158                                 break;
1159
1160                         /*
1161                          * Accept eop packets with no errors.
1162                          * With no errors and the Ixsm bit set,
1163                          * the descriptor status Tpcs and Ipcs bits give
1164                          * an indication of whether the checksums were
1165                          * calculated and valid.
1166                          */
1167                         if((rd->status & Reop) && rd->errors == 0){
1168                                 bp = ctlr->rb[rdh];
1169                                 ctlr->rb[rdh] = NULL;
1170                                 bp->wp += rd->length;
1171                                 bp->next = NULL;
1172                                 if(!(rd->status & Ixsm)){
1173                                         ctlr->ixsm++;
1174                                         if(rd->status & Ipcs){
1175                                                 /*
1176                                                  * IP checksum calculated
1177                                                  * (and valid as errors == 0).
1178                                                  */
1179                                                 ctlr->ipcs++;
1180                                                 bp->flag |= Bipck;
1181                                         }
1182                                         if(rd->status & Tcpcs){
1183                                                 /*
1184                                                  * TCP/UDP checksum calculated
1185                                                  * (and valid as errors == 0).
1186                                                  */
1187                                                 ctlr->tcpcs++;
1188                                                 bp->flag |= Btcpck|Budpck;
1189                                         }
1190                                         bp->flag |= Bpktck;
1191                                 }
1192                                 etheriq(edev, bp, 1);
1193                         }
1194                         else if(ctlr->rb[rdh] != NULL){
1195                                 freeb(ctlr->rb[rdh]);
1196                                 ctlr->rb[rdh] = NULL;
1197                         }
1198
1199                         memset(rd, 0, sizeof(Rd));
1200                         wmb();  /* make sure the zeroing happens before free (i think) */
1201                         ctlr->rdfree--;
1202                         rdh = NEXT_RING(rdh, ctlr->nrd);
1203                 }
1204                 ctlr->rdh = rdh;
1205
1206                 if(ctlr->rdfree < ctlr->nrd/2 || (ctlr->rim & Rxdmt0))
1207                         igbereplenish(ctlr);
1208         }
1209 }
1210
1211 static void
1212 igbeattach(struct ether* edev)
1213 {
1214         ERRSTACK(1);
1215         struct block *bp;
1216         struct ctlr *ctlr;
1217         char *name;
1218
1219         ctlr = edev->ctlr;
1220         ctlr->edev = edev;                      /* point back to Ether* */
1221         qlock(&ctlr->alock);
1222         if(ctlr->alloc != NULL){                        /* already allocated? */
1223                 qunlock(&ctlr->alock);
1224                 return;
1225         }
1226
1227         ctlr->tb = NULL;
1228         ctlr->rb = NULL;
1229         ctlr->alloc = NULL;
1230         if(waserror()){
1231                 kfree(ctlr->tb);
1232                 ctlr->tb = NULL;
1233                 kfree(ctlr->rb);
1234                 ctlr->rb = NULL;
1235                 kfree(ctlr->alloc);
1236                 ctlr->alloc = NULL;
1237                 qunlock(&ctlr->alock);
1238                 nexterror();
1239         }
1240
1241         ctlr->nrd = Nrd;
1242         ctlr->ntd = Ntd;
1243         ctlr->alloc = kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1244         if(ctlr->alloc == NULL) {
1245                 printd("igbe: can't allocate ctlr->alloc\n");
1246                 error(ENOMEM, ERROR_FIXME);
1247         }
1248         ctlr->rdba = (Rd*)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1249         ctlr->tdba = (Td*)(ctlr->rdba+ctlr->nrd);
1250
1251         ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1252         ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1253         if (ctlr->rb == NULL || ctlr->tb == NULL) {
1254                 printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1255                 error(ENOMEM, ERROR_FIXME);
1256         }
1257
1258         /* the ktasks should free these names, if they ever exit */
1259         name = kmalloc(KNAMELEN, MEM_WAIT);
1260         snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1261         ktask(name, igbelproc, edev);
1262
1263         name = kmalloc(KNAMELEN, MEM_WAIT);
1264         snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1265         ktask(name, igberproc, edev);
1266
1267         igbetxinit(ctlr);
1268
1269         qunlock(&ctlr->alock);
1270         poperror();
1271 }
1272
1273 static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1274 {
1275         struct ctlr *ctlr;
1276         struct ether *edev;
1277         int icr, im, txdw;
1278
1279         edev = arg;
1280         ctlr = edev->ctlr;
1281
1282         ilock(&ctlr->imlock);
1283         csr32w(ctlr, Imc, ~0);
1284         im = ctlr->im;
1285         txdw = 0;
1286
1287         while((icr = csr32r(ctlr, Icr) & ctlr->im) != 0){
1288                 if(icr & Lsc){
1289                         im &= ~Lsc;
1290                         ctlr->lim = icr & Lsc;
1291                         rendez_wakeup(&ctlr->lrendez);
1292                         ctlr->lintr++;
1293                 }
1294                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq)){
1295                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq);
1296                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq);
1297                         rendez_wakeup(&ctlr->rrendez);
1298                         ctlr->rintr++;
1299                 }
1300                 if(icr & Txdw){
1301                         im &= ~Txdw;
1302                         txdw++;
1303                         ctlr->tintr++;
1304                 }
1305         }
1306
1307         ctlr->im = im;
1308         csr32w(ctlr, Ims, im);
1309         iunlock(&ctlr->imlock);
1310
1311         if(txdw)
1312                 igbetransmit(edev);
1313 }
1314
1315 static int
1316 i82543mdior(struct ctlr* ctlr, int n)
1317 {
1318         int ctrl, data, i, r;
1319
1320         /*
1321          * Read n bits from the Management Data I/O Interface.
1322          */
1323         ctrl = csr32r(ctlr, Ctrl);
1324         r = (ctrl & ~Mddo)|Mdco;
1325         data = 0;
1326         for(i = n-1; i >= 0; i--){
1327                 if(csr32r(ctlr, Ctrl) & Mdd)
1328                         data |= (1<<i);
1329                 csr32w(ctlr, Ctrl, Mdc|r);
1330                 csr32w(ctlr, Ctrl, r);
1331         }
1332         csr32w(ctlr, Ctrl, ctrl);
1333
1334         return data;
1335 }
1336
1337 static int
1338 i82543mdiow(struct ctlr* ctlr, int bits, int n)
1339 {
1340         int ctrl, i, r;
1341
1342         /*
1343          * Write n bits to the Management Data I/O Interface.
1344          */
1345         ctrl = csr32r(ctlr, Ctrl);
1346         r = Mdco|Mddo|ctrl;
1347         for(i = n-1; i >= 0; i--){
1348                 if(bits & (1<<i))
1349                         r |= Mdd;
1350                 else
1351                         r &= ~Mdd;
1352                 csr32w(ctlr, Ctrl, Mdc|r);
1353                 csr32w(ctlr, Ctrl, r);
1354         }
1355         csr32w(ctlr, Ctrl, ctrl);
1356
1357         return 0;
1358 }
1359
1360 static int
1361 i82543miimir(struct mii* mii, int pa, int ra)
1362 {
1363         int data;
1364         struct ctlr *ctlr;
1365
1366         ctlr = mii->ctlr;
1367
1368         /*
1369          * MII Management Interface Read.
1370          *
1371          * Preamble;
1372          * ST+OP+PHYAD+REGAD;
1373          * TA + 16 data bits.
1374          */
1375         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1376         i82543mdiow(ctlr, 0x1800|(pa<<5)|ra, 14);
1377         data = i82543mdior(ctlr, 18);
1378
1379         if(data & 0x10000)
1380                 return -1;
1381
1382         return data & 0xFFFF;
1383 }
1384
1385 static int
1386 i82543miimiw(struct mii* mii, int pa, int ra, int data)
1387 {
1388         struct ctlr *ctlr;
1389
1390         ctlr = mii->ctlr;
1391
1392         /*
1393          * MII Management Interface Write.
1394          *
1395          * Preamble;
1396          * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1397          * Z.
1398          */
1399         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1400         data &= 0xFFFF;
1401         data |= (0x05<<(5+5+2+16))|(pa<<(5+2+16))|(ra<<(2+16))|(0x02<<16);
1402         i82543mdiow(ctlr, data, 32);
1403
1404         return 0;
1405 }
1406
1407 static int
1408 igbemiimir(struct mii* mii, int pa, int ra)
1409 {
1410         struct ctlr *ctlr;
1411         int mdic, timo;
1412
1413         ctlr = mii->ctlr;
1414
1415         csr32w(ctlr, Mdic, MDIrop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT));
1416         mdic = 0;
1417         for(timo = 64; timo; timo--){
1418                 mdic = csr32r(ctlr, Mdic);
1419                 if(mdic & (MDIe|MDIready))
1420                         break;
1421                 udelay(1);
1422         }
1423
1424         if((mdic & (MDIe|MDIready)) == MDIready)
1425                 return mdic & 0xFFFF;
1426         return -1;
1427 }
1428
1429 static int
1430 igbemiimiw(struct mii* mii, int pa, int ra, int data)
1431 {
1432         struct ctlr *ctlr;
1433         int mdic, timo;
1434
1435         ctlr = mii->ctlr;
1436
1437         data &= MDIdMASK;
1438         csr32w(ctlr, Mdic, MDIwop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT)|data);
1439         mdic = 0;
1440         for(timo = 64; timo; timo--){
1441                 mdic = csr32r(ctlr, Mdic);
1442                 if(mdic & (MDIe|MDIready))
1443                         break;
1444                 udelay(1);
1445         }
1446         if((mdic & (MDIe|MDIready)) == MDIready)
1447                 return 0;
1448         return -1;
1449 }
1450
1451 static int
1452 i82543miirw(struct mii* mii, int write, int pa, int ra, int data)
1453 {
1454         if(write)
1455                 return i82543miimiw(mii, pa, ra, data);
1456
1457         return i82543miimir(mii, pa, ra);
1458 }
1459
1460 static int
1461 igbemiirw(struct mii* mii, int write, int pa, int ra, int data)
1462 {
1463         if(write)
1464                 return igbemiimiw(mii, pa, ra, data);
1465
1466         return igbemiimir(mii, pa, ra);
1467 }
1468
1469 static int
1470 igbemii(struct ctlr* ctlr)
1471 {
1472         int ctrl, p, r;
1473         int (*rw)(struct mii*, int unused_int, int, int, int);
1474
1475         r = csr32r(ctlr, Status);
1476         if(r & Tbimode)
1477                 return -1;
1478
1479         ctrl = csr32r(ctlr, Ctrl);
1480         ctrl |= Slu;
1481
1482         switch(ctlr->id){
1483         case i82543gc:
1484                 ctrl |= Frcdplx|Frcspd;
1485                 csr32w(ctlr, Ctrl, ctrl);
1486
1487                 /*
1488                  * The reset pin direction (Mdro) should already
1489                  * be set from the EEPROM load.
1490                  * If it's not set this configuration is unexpected
1491                  * so bail.
1492                  */
1493                 r = csr32r(ctlr, Ctrlext);
1494                 if(!(r & Mdro))
1495                         return -1;
1496                 csr32w(ctlr, Ctrlext, r);
1497                 udelay(20*1000);
1498                 r = csr32r(ctlr, Ctrlext);
1499                 r &= ~Mdr;
1500                 csr32w(ctlr, Ctrlext, r);
1501                 udelay(20*1000);
1502                 r = csr32r(ctlr, Ctrlext);
1503                 r |= Mdr;
1504                 csr32w(ctlr, Ctrlext, r);
1505                 udelay(20*1000);
1506
1507                 rw = i82543miirw;
1508                 break;
1509         case i82544ei:
1510         case i82544eif:
1511         case i82544gc:
1512         case i82540em:
1513         case i82540eplp:
1514         case i82547ei:
1515         case i82547gi:
1516         case i82541ei:
1517         case i82541gi:
1518         case i82541gi2:
1519         case i82541pi:
1520         case i82545em:
1521         case i82545gmc:
1522         case i82546gb:
1523         case i82546eb:
1524                 ctrl &= ~(Frcdplx|Frcspd);
1525                 csr32w(ctlr, Ctrl, ctrl);
1526                 rw = igbemiirw;
1527                 break;
1528         default:
1529                 return -1;
1530         }
1531
1532         if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1533                 return -1;
1534         // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1535
1536         /*
1537          * 8254X-specific PHY registers not in 802.3:
1538          *      0x10    PHY specific control
1539          *      0x14    extended PHY specific control
1540          * Set appropriate values then reset the PHY to have
1541          * changes noted.
1542          */
1543         switch(ctlr->id){
1544         case i82547gi:
1545         case i82541gi:
1546         case i82541gi2:
1547         case i82541pi:
1548         case i82545em:
1549         case i82545gmc:
1550         case i82546gb:
1551         case i82546eb:
1552                 break;
1553         default:
1554                 r = miimir(ctlr->mii, 16);
1555                 r |= 0x0800;                    /* assert CRS on Tx */
1556                 r |= 0x0060;                    /* auto-crossover all speeds */
1557                 r |= 0x0002;                    /* polarity reversal enabled */
1558                 miimiw(ctlr->mii, 16, r);
1559
1560                 r = miimir(ctlr->mii, 20);
1561                 r |= 0x0070;                    /* +25MHz clock */
1562                 r &= ~0x0F00;
1563                 r |= 0x0100;                    /* 1x downshift */
1564                 miimiw(ctlr->mii, 20, r);
1565
1566                 miireset(ctlr->mii);
1567                 p = 0;
1568                 if(ctlr->txcw & TxcwPs)
1569                         p |= AnaP;
1570                 if(ctlr->txcw & TxcwAs)
1571                         p |= AnaAP;
1572                 miiane(ctlr->mii, ~0, p, ~0);
1573                 break;
1574         }
1575         return 0;
1576 }
1577
1578 static int
1579 at93c46io(struct ctlr* ctlr, char* op, int data)
1580 {
1581         char *lp, *p;
1582         int i, loop, eecd, r;
1583
1584         eecd = csr32r(ctlr, Eecd);
1585
1586         r = 0;
1587         loop = -1;
1588         lp = NULL;
1589         for(p = op; *p != '\0'; p++){
1590                 switch(*p){
1591                 default:
1592                         return -1;
1593                 case ' ':
1594                         continue;
1595                 case ':':                       /* start of loop */
1596                         loop = strtol(p+1, &lp, 0)-1;
1597                         lp--;
1598                         if(p == lp)
1599                                 loop = 7;
1600                         p = lp;
1601                         continue;
1602                 case ';':                       /* end of loop */
1603                         if(lp == NULL)
1604                                 return -1;
1605                         loop--;
1606                         if(loop >= 0)
1607                                 p = lp;
1608                         else
1609                                 lp = NULL;
1610                         continue;
1611                 case 'C':                       /* assert clock */
1612                         eecd |= Sk;
1613                         break;
1614                 case 'c':                       /* deassert clock */
1615                         eecd &= ~Sk;
1616                         break;
1617                 case 'D':                       /* next bit in 'data' byte */
1618                         if(loop < 0)
1619                                 return -1;
1620                         if(data & (1<<loop))
1621                                 eecd |= Di;
1622                         else
1623                                 eecd &= ~Di;
1624                         break;
1625                 case 'O':                       /* collect data output */
1626                         i = (csr32r(ctlr, Eecd) & Do) != 0;
1627                         if(loop >= 0)
1628                                 r |= (i<<loop);
1629                         else
1630                                 r = i;
1631                         continue;
1632                 case 'I':                       /* assert data input */
1633                         eecd |= Di;
1634                         break;
1635                 case 'i':                       /* deassert data input */
1636                         eecd &= ~Di;
1637                         break;
1638                 case 'S':                       /* enable chip select */
1639                         eecd |= Cs;
1640                         break;
1641                 case 's':                       /* disable chip select */
1642                         eecd &= ~Cs;
1643                         break;
1644                 }
1645                 csr32w(ctlr, Eecd, eecd);
1646                 udelay(50);
1647         }
1648         if(loop >= 0)
1649                 return -1;
1650         return r;
1651 }
1652
1653 static int
1654 at93c46r(struct ctlr* ctlr)
1655 {
1656         uint16_t sum;
1657         char rop[20];
1658         int addr, areq, bits, data, eecd, i;
1659
1660         eecd = csr32r(ctlr, Eecd);
1661         if(eecd & Spi){
1662                 printd("igbe: SPI EEPROM access not implemented\n");
1663                 return 0;
1664         }
1665         if(eecd & (Eeszaddr|Eesz256))
1666                 bits = 8;
1667         else
1668                 bits = 6;
1669
1670         sum = 0;
1671
1672         switch(ctlr->id){
1673         default:
1674                 areq = 0;
1675                 break;
1676         case i82540em:
1677         case i82540eplp:
1678         case i82541ei:
1679         case i82541gi:
1680         case i82541gi2:
1681         case i82541pi:
1682         case i82545em:
1683         case i82545gmc:
1684         case i82546gb:
1685         case i82546eb:
1686         case i82547ei:
1687         case i82547gi:
1688                 areq = 1;
1689                 csr32w(ctlr, Eecd, eecd|Areq);
1690                 for(i = 0; i < 1000; i++){
1691                         if((eecd = csr32r(ctlr, Eecd)) & Agnt)
1692                                 break;
1693                         udelay(5);
1694                 }
1695                 if(!(eecd & Agnt)){
1696                         printd("igbe: not granted EEPROM access\n");
1697                         goto release;
1698                 }
1699                 break;
1700         }
1701         snprintf(rop, sizeof(rop), "S :%dDCc;", bits+3);
1702
1703         for(addr = 0; addr < 0x40; addr++){
1704                 /*
1705                  * Read a word at address 'addr' from the Atmel AT93C46
1706                  * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1707                  * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1708                  * for protocol details.
1709                  */
1710                 if(at93c46io(ctlr, rop, (0x06<<bits)|addr) != 0){
1711                         printd("igbe: can't set EEPROM address 0x%2.2X\n", addr);
1712                         goto release;
1713                 }
1714                 data = at93c46io(ctlr, ":16COc;", 0);
1715                 at93c46io(ctlr, "sic", 0);
1716                 ctlr->eeprom[addr] = data;
1717                 sum += data;
1718         }
1719
1720 release:
1721         if(areq)
1722                 csr32w(ctlr, Eecd, eecd & ~Areq);
1723         return sum;
1724 }
1725
1726 static int
1727 igbedetach(struct ctlr* ctlr)
1728 {
1729         int r, timeo;
1730
1731         /*
1732          * Perform a device reset to get the chip back to the
1733          * power-on state, followed by an EEPROM reset to read
1734          * the defaults for some internal registers.
1735          */
1736         csr32w(ctlr, Imc, ~0);
1737         csr32w(ctlr, Rctl, 0);
1738         csr32w(ctlr, Tctl, 0);
1739
1740         udelay(10*1000);
1741
1742         csr32w(ctlr, Ctrl, Devrst);
1743         udelay(1*1000);
1744         for(timeo = 0; timeo < 1000; timeo++){
1745                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1746                         break;
1747                 udelay(1*1000);
1748         }
1749         if(csr32r(ctlr, Ctrl) & Devrst)
1750                 return -1;
1751         r = csr32r(ctlr, Ctrlext);
1752         csr32w(ctlr, Ctrlext, r|Eerst);
1753         udelay(1*1000);
1754
1755         for(timeo = 0; timeo < 1000; timeo++){
1756                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1757                         break;
1758                 udelay(1*1000);
1759         }
1760         if(csr32r(ctlr, Ctrlext) & Eerst)
1761                 return -1;
1762
1763         switch(ctlr->id){
1764         default:
1765                 break;
1766         case i82540em:
1767         case i82540eplp:
1768         case i82541gi:
1769         case i82541gi2:
1770         case i82541pi:
1771         case i82545em:
1772         case i82545gmc:
1773         case i82547gi:
1774         case i82546gb:
1775         case i82546eb:
1776                 r = csr32r(ctlr, Manc);
1777                 r &= ~Arpen;
1778                 csr32w(ctlr, Manc, r);
1779                 break;
1780         }
1781
1782         csr32w(ctlr, Imc, ~0);
1783         udelay(1*1000);
1784         for(timeo = 0; timeo < 1000; timeo++){
1785                 if(!csr32r(ctlr, Icr))
1786                         break;
1787                 udelay(1*1000);
1788         }
1789         if(csr32r(ctlr, Icr))
1790                 return -1;
1791
1792         return 0;
1793 }
1794
1795 static void
1796 igbeshutdown(struct ether* ether)
1797 {
1798         igbedetach(ether->ctlr);
1799 }
1800
1801 static int
1802 igbereset(struct ctlr* ctlr)
1803 {
1804         int ctrl, i, pause, r, swdpio, txcw;
1805
1806         if(igbedetach(ctlr))
1807                 return -1;
1808
1809         /*
1810          * Read the EEPROM, validate the checksum
1811          * then get the device back to a power-on state.
1812          */
1813         if((r = at93c46r(ctlr)) != 0xBABA){
1814                 printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1815                 return -1;
1816         }
1817
1818         /*
1819          * Snarf and set up the receive addresses.
1820          * There are 16 addresses. The first should be the MAC address.
1821          * The others are cleared and not marked valid (MS bit of Rah).
1822          */
1823         if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1824             (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1825                  pci_config_addr(0, 1, 0, 0)))
1826                 ctlr->eeprom[Ea+2] += 0x100;            /* second interface */
1827         if(ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1828                 ctlr->eeprom[Ea] = 0xD000;
1829         for(i = Ea; i < Eaddrlen/2; i++){
1830                 ctlr->ra[2*i] = ctlr->eeprom[i];
1831                 ctlr->ra[2*i+1] = ctlr->eeprom[i]>>8;
1832         }
1833         /* lan id seems to vary on 82543gc; don't use it */
1834         if (ctlr->id != i82543gc) {
1835                 r = (csr32r(ctlr, Status) & Lanid) >> 2;
1836                 ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1837         }
1838
1839         r = (ctlr->ra[3]<<24)|(ctlr->ra[2]<<16)|(ctlr->ra[1]<<8)|ctlr->ra[0];
1840         csr32w(ctlr, Ral, r);
1841         r = 0x80000000|(ctlr->ra[5]<<8)|ctlr->ra[4];
1842         csr32w(ctlr, Rah, r);
1843         for(i = 1; i < 16; i++){
1844                 csr32w(ctlr, Ral+i*8, 0);
1845                 csr32w(ctlr, Rah+i*8, 0);
1846         }
1847
1848         /*
1849          * Clear the Multicast Table Array.
1850          * It's a 4096 bit vector accessed as 128 32-bit registers.
1851          */
1852         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1853         for(i = 0; i < 128; i++)
1854                 csr32w(ctlr, Mta+i*4, 0);
1855
1856         /*
1857          * Just in case the Eerst didn't load the defaults
1858          * (doesn't appear to fully on the 82543GC), do it manually.
1859          */
1860         if (ctlr->id == i82543gc) {
1861                 txcw = csr32r(ctlr, Txcw);
1862                 txcw &= ~(TxcwAne|TxcwPauseMASK|TxcwFd);
1863                 ctrl = csr32r(ctlr, Ctrl);
1864                 ctrl &= ~(SwdpioloMASK|Frcspd|Ilos|Lrst|Fd);
1865
1866                 if(ctlr->eeprom[Icw1] & 0x0400){
1867                         ctrl |= Fd;
1868                         txcw |= TxcwFd;
1869                 }
1870                 if(ctlr->eeprom[Icw1] & 0x0200)
1871                         ctrl |= Lrst;
1872                 if(ctlr->eeprom[Icw1] & 0x0010)
1873                         ctrl |= Ilos;
1874                 if(ctlr->eeprom[Icw1] & 0x0800)
1875                         ctrl |= Frcspd;
1876                 swdpio = (ctlr->eeprom[Icw1] & 0x01E0)>>5;
1877                 ctrl |= swdpio<<SwdpioloSHIFT;
1878                 csr32w(ctlr, Ctrl, ctrl);
1879
1880                 ctrl = csr32r(ctlr, Ctrlext);
1881                 ctrl &= ~(Ips|SwdpiohiMASK);
1882                 swdpio = (ctlr->eeprom[Icw2] & 0x00F0)>>4;
1883                 if(ctlr->eeprom[Icw1] & 0x1000)
1884                         ctrl |= Ips;
1885                 ctrl |= swdpio<<SwdpiohiSHIFT;
1886                 csr32w(ctlr, Ctrlext, ctrl);
1887
1888                 if(ctlr->eeprom[Icw2] & 0x0800)
1889                         txcw |= TxcwAne;
1890                 pause = (ctlr->eeprom[Icw2] & 0x3000)>>12;
1891                 txcw |= pause<<TxcwPauseSHIFT;
1892                 switch(pause){
1893                 default:
1894                         ctlr->fcrtl = 0x00002000;
1895                         ctlr->fcrth = 0x00004000;
1896                         txcw |= TxcwAs|TxcwPs;
1897                         break;
1898                 case 0:
1899                         ctlr->fcrtl = 0x00002000;
1900                         ctlr->fcrth = 0x00004000;
1901                         break;
1902                 case 2:
1903                         ctlr->fcrtl = 0;
1904                         ctlr->fcrth = 0;
1905                         txcw |= TxcwAs;
1906                         break;
1907                 }
1908                 ctlr->txcw = txcw;
1909                 csr32w(ctlr, Txcw, txcw);
1910         }
1911
1912
1913         /*
1914          * Flow control - values from the datasheet.
1915          */
1916         csr32w(ctlr, Fcal, 0x00C28001);
1917         csr32w(ctlr, Fcah, 0x00000100);
1918         csr32w(ctlr, Fct, 0x00008808);
1919         csr32w(ctlr, Fcttv, 0x00000100);
1920
1921         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1922         csr32w(ctlr, Fcrth, ctlr->fcrth);
1923
1924         /* FYI, igbemii checks status right away too. */
1925         if(!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1926                 printk("igbemii failed!  igbe failing to reset!\n");
1927                 return -1;
1928         }
1929
1930         return 0;
1931 }
1932
1933 static void
1934 igbepci(void)
1935 {
1936         int id;
1937         struct pci_device *pcidev;
1938         struct ctlr *ctlr;
1939         void *mem;
1940         uintptr_t mmio_paddr;
1941
1942         STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
1943                 /* This checks that pcidev is a Network Controller for Ethernet */
1944                 if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1945                         continue;
1946                 id = pcidev->dev_id << 16 | pcidev->ven_id;
1947                 switch (id) {
1948                 default:
1949                         continue;
1950                 case i82543gc:
1951                 case i82544ei:
1952                 case i82544eif:
1953                 case i82544gc:
1954                 case i82547ei:
1955                 case i82547gi:
1956                 case i82540em:
1957                 case i82540eplp:
1958                 case i82541ei:
1959                 case i82541gi:
1960                 case i82541gi2:
1961                 case i82541pi:
1962                 case i82545em:
1963                 case i82545gmc:
1964                 case i82546gb:
1965                 case i82546eb:
1966                         break;
1967                 }
1968                 printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
1969                        pcidev->ven_id, pcidev->dev_id,
1970                        pcidev->bus, pcidev->dev, pcidev->func);
1971
1972                 mmio_paddr = pcidev->bar[0].mmio_base32 ? pcidev->bar[0].mmio_base32 :
1973                                                           pcidev->bar[0].mmio_base64;
1974                 mem = (void*)vmap_pmem_nocache(mmio_paddr, pcidev->bar[0].mmio_sz);
1975                 if(mem == NULL){
1976                         printd("igbe: can't map %p\n", pcidev->bar[0].mmio_base32);
1977                         continue;
1978                 }
1979                 pci_set_cacheline_size(pcidev);
1980                 ctlr = kzmalloc(sizeof(struct ctlr), 0);
1981                 if(ctlr == NULL) {
1982                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
1983                         error(ENOMEM, ERROR_FIXME);
1984                 }
1985                 spinlock_init_irqsave(&ctlr->imlock);
1986                 spinlock_init_irqsave(&ctlr->tlock);
1987                 qlock_init(&ctlr->alock);
1988                 qlock_init(&ctlr->slock);
1989                 rendez_init(&ctlr->lrendez);
1990                 rendez_init(&ctlr->rrendez);
1991                 /* port seems to be unused, and only used for some comparison with edev.
1992                  * plan9 just used the top of the raw bar, regardless of the type. */
1993                 ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
1994                 ctlr->pci = pcidev;
1995                 ctlr->id = id;
1996                 ctlr->cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
1997                 ctlr->nic = mem;
1998
1999                 if(igbereset(ctlr)){
2000                         kfree(ctlr);
2001                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2002                         continue;
2003                 }
2004                 pci_set_bus_master(pcidev);
2005
2006                 if(igbectlrhead != NULL)
2007                         igbectlrtail->next = ctlr;
2008                 else
2009                         igbectlrhead = ctlr;
2010                 igbectlrtail = ctlr;
2011         }
2012 }
2013
2014 static int
2015 igbepnp(struct ether* edev)
2016 {
2017         struct ctlr *ctlr;
2018
2019         run_once(igbepci());
2020
2021         /*
2022          * Any adapter matches if no edev->port is supplied,
2023          * otherwise the ports must match.
2024          */
2025         for(ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next){
2026                 if(ctlr->active)
2027                         continue;
2028                 if(edev->port == 0 || edev->port == ctlr->port){
2029                         ctlr->active = 1;
2030                         break;
2031                 }
2032         }
2033         if(ctlr == NULL)
2034                 return -1;
2035
2036         edev->ctlr = ctlr;
2037         strlcpy(edev->drv_name, "igbe", KNAMELEN);
2038         edev->port = ctlr->port;
2039         edev->irq = ctlr->pci->irqline;
2040         edev->tbdf = pci_to_tbdf(ctlr->pci);
2041         edev->mbps = 1000;
2042         memmove(edev->ea, ctlr->ra, Eaddrlen);
2043         /* Jim or whoever have this turned on already.  We might be capable of other
2044          * features. */
2045         edev->feat = NETF_RXCSUM;
2046
2047         /*
2048          * Linkage to the generic ethernet driver.
2049          */
2050         edev->attach = igbeattach;
2051         edev->transmit = igbetransmit;
2052         edev->ifstat = igbeifstat;
2053         edev->ctl = igbectl;
2054         edev->shutdown = igbeshutdown;
2055
2056         edev->arg = edev;
2057         edev->promiscuous = igbepromiscuous;
2058         edev->multicast = igbemulticast;
2059
2060         register_irq(edev->irq, igbeinterrupt, edev, edev->tbdf);
2061         return 0;
2062 }
2063
2064 linker_func_3(etherigbelink)
2065 {
2066         addethercard("i82543", igbepnp);
2067         addethercard("igbe", igbepnp);
2068 }