4d409f7194d8ffcfaa1ebb3a18b6617d7064f0dd
[akaros.git] / kern / drivers / net / etherigbe.c
1 /* This file is part of the UCB release of Plan 9. It is subject to the license
2  * terms in the LICENSE file found in the top-level directory of this
3  * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
4  * part of the UCB release of Plan 9, including this file, may be copied,
5  * modified, propagated, or distributed except according to the terms contained
6  * in the LICENSE file. */
7
8 /*
9  * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
10  * as found on the Intel PRO/1000 series of adapters:
11  *      82543GC Intel PRO/1000 T
12  *      82544EI Intel PRO/1000 XT
13  *      82540EM Intel PRO/1000 MT
14  *      82541[GP]I
15  *      82547GI
16  *      82546GB
17  *      82546EB
18  * To Do:
19  *      finish autonegotiation code;
20  *      integrate fiber stuff back in (this ONLY handles
21  *      the CAT5 cards at the moment);
22  *      add checksum-offload;
23  *      add tuning control via ctl file;
24  *      this driver is little-endian specific.
25  *
26  * Modified by brho:
27  *      ported to Akaros
28  *      fixed mii bugs (allocation, startup, miirw, etc)
29  *      fixed CLS bug (continue -> break)
30  *      made sure igbepci only runs once, even if it fails */
31
32 #include <vfs.h>
33 #include <kfs.h>
34 #include <slab.h>
35 #include <kmalloc.h>
36 #include <kref.h>
37 #include <string.h>
38 #include <stdio.h>
39 #include <assert.h>
40 #include <error.h>
41 #include <cpio.h>
42 #include <pmap.h>
43 #include <smp.h>
44 #include <arch/pci.h>
45 #include <ip.h>
46 #include <ns.h>
47 #include "ethermii.h"
48
49 #define ilock(x) spin_lock_irqsave(x)
50 #define iunlock(x) spin_unlock_irqsave(x)
51
52 enum {
53         i82542          = (0x1000<<16)|0x8086,
54         i82543gc        = (0x1004<<16)|0x8086,
55         i82544ei        = (0x1008<<16)|0x8086,
56         i82544eif       = (0x1009<<16)|0x8086,
57         i82544gc        = (0x100d<<16)|0x8086,
58         i82540em        = (0x100E<<16)|0x8086,
59         i82540eplp      = (0x101E<<16)|0x8086,
60         i82545em        = (0x100F<<16)|0x8086,
61         i82545gmc       = (0x1026<<16)|0x8086,
62         i82547ei        = (0x1019<<16)|0x8086,
63         i82547gi        = (0x1075<<16)|0x8086,
64         i82541ei        = (0x1013<<16)|0x8086,
65         i82541gi        = (0x1076<<16)|0x8086,
66         i82541gi2       = (0x1077<<16)|0x8086,
67         i82541pi        = (0x107c<<16)|0x8086,
68         i82546gb        = (0x1079<<16)|0x8086,
69         i82546eb        = (0x1010<<16)|0x8086,
70 };
71
72 enum {
73         Ctrl            = 0x00000000,   /* Device Control */
74         Ctrldup         = 0x00000004,   /* Device Control Duplicate */
75         Status          = 0x00000008,   /* Device Status */
76         Eecd            = 0x00000010,   /* EEPROM/Flash Control/Data */
77         Ctrlext         = 0x00000018,   /* Extended Device Control */
78         Mdic            = 0x00000020,   /* MDI Control */
79         Fcal            = 0x00000028,   /* Flow Control Address Low */
80         Fcah            = 0x0000002C,   /* Flow Control Address High */
81         Fct             = 0x00000030,   /* Flow Control Type */
82         Icr             = 0x000000C0,   /* Interrupt Cause Read */
83         Ics             = 0x000000C8,   /* Interrupt Cause Set */
84         Ims             = 0x000000D0,   /* Interrupt Mask Set/Read */
85         Imc             = 0x000000D8,   /* Interrupt mask Clear */
86         Rctl            = 0x00000100,   /* Receive Control */
87         Fcttv           = 0x00000170,   /* Flow Control Transmit Timer Value */
88         Txcw            = 0x00000178,   /* Transmit Configuration Word */
89         Rxcw            = 0x00000180,   /* Receive Configuration Word */
90         /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
91         Tctl            = 0x00000400,   /* Transmit Control */
92         Tipg            = 0x00000410,   /* Transmit IPG */
93         Tbt             = 0x00000448,   /* Transmit Burst Timer */
94         Ait             = 0x00000458,   /* Adaptive IFS Throttle */
95         Fcrtl           = 0x00002160,   /* Flow Control RX Threshold Low */
96         Fcrth           = 0x00002168,   /* Flow Control Rx Threshold High */
97         Rdfh            = 0x00002410,   /* Receive data fifo head */
98         Rdft            = 0x00002418,   /* Receive data fifo tail */
99         Rdfhs           = 0x00002420,   /* Receive data fifo head saved */
100         Rdfts           = 0x00002428,   /* Receive data fifo tail saved */
101         Rdfpc           = 0x00002430,   /* Receive data fifo packet count */
102         Rdbal           = 0x00002800,   /* Rd Base Address Low */
103         Rdbah           = 0x00002804,   /* Rd Base Address High */
104         Rdlen           = 0x00002808,   /* Receive Descriptor Length */
105         Rdh             = 0x00002810,   /* Receive Descriptor Head */
106         Rdt             = 0x00002818,   /* Receive Descriptor Tail */
107         Rdtr            = 0x00002820,   /* Receive Descriptor Timer Ring */
108         Rxdctl          = 0x00002828,   /* Receive Descriptor Control */
109         Radv            = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
110         Txdmac          = 0x00003000,   /* Transfer DMA Control */
111         Ett             = 0x00003008,   /* Early Transmit Control */
112         Tdfh            = 0x00003410,   /* Transmit data fifo head */
113         Tdft            = 0x00003418,   /* Transmit data fifo tail */
114         Tdfhs           = 0x00003420,   /* Transmit data Fifo Head saved */
115         Tdfts           = 0x00003428,   /* Transmit data fifo tail saved */
116         Tdfpc           = 0x00003430,   /* Trasnmit data Fifo packet count */
117         Tdbal           = 0x00003800,   /* Td Base Address Low */
118         Tdbah           = 0x00003804,   /* Td Base Address High */
119         Tdlen           = 0x00003808,   /* Transmit Descriptor Length */
120         Tdh             = 0x00003810,   /* Transmit Descriptor Head */
121         Tdt             = 0x00003818,   /* Transmit Descriptor Tail */
122         Tidv            = 0x00003820,   /* Transmit Interrupt Delay Value */
123         Txdctl          = 0x00003828,   /* Transmit Descriptor Control */
124         Tadv            = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
125
126         Statistics      = 0x00004000,   /* Start of Statistics Area */
127         Gorcl           = 0x88/4,       /* Good Octets Received Count */
128         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
129         Torl            = 0xC0/4,       /* Total Octets Received */
130         Totl            = 0xC8/4,       /* Total Octets Transmitted */
131         Nstatistics     = 64,
132
133         Rxcsum          = 0x00005000,   /* Receive Checksum Control */
134         Mta             = 0x00005200,   /* Multicast Table Array */
135         Ral             = 0x00005400,   /* Receive Address Low */
136         Rah             = 0x00005404,   /* Receive Address High */
137         Manc            = 0x00005820,   /* Management Control */
138 };
139
140 enum {                                  /* Ctrl */
141         Bem             = 0x00000002,   /* Big Endian Mode */
142         Prior           = 0x00000004,   /* Priority on the PCI bus */
143         Lrst            = 0x00000008,   /* Link Reset */
144         Asde            = 0x00000020,   /* Auto-Speed Detection Enable */
145         Slu             = 0x00000040,   /* Set Link Up */
146         Ilos            = 0x00000080,   /* Invert Loss of Signal (LOS) */
147         SspeedMASK      = 0x00000300,   /* Speed Selection */
148         SspeedSHIFT     = 8,
149         Sspeed10        = 0x00000000,   /* 10Mb/s */
150         Sspeed100       = 0x00000100,   /* 100Mb/s */
151         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
152         Frcspd          = 0x00000800,   /* Force Speed */
153         Frcdplx         = 0x00001000,   /* Force Duplex */
154         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
155         SwdpinsloSHIFT  = 18,
156         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
157         SwdpioloSHIFT   = 22,
158         Devrst          = 0x04000000,   /* Device Reset */
159         Rfce            = 0x08000000,   /* Receive Flow Control Enable */
160         Tfce            = 0x10000000,   /* Transmit Flow Control Enable */
161         Vme             = 0x40000000,   /* VLAN Mode Enable */
162 };
163
164 /*
165  * can't find Tckok nor Rbcok in any Intel docs,
166  * but even 82543gc docs define Lanid.
167  */
168 enum {                                  /* Status */
169         Lu              = 0x00000002,   /* Link Up */
170         Lanid           = 0x0000000C,   /* mask for Lan ID. (function id) */
171 //      Tckok           = 0x00000004,   /* Transmit clock is running */
172 //      Rbcok           = 0x00000008,   /* Receive clock is running */
173         Txoff           = 0x00000010,   /* Transmission Paused */
174         Tbimode         = 0x00000020,   /* TBI Mode Indication */
175         LspeedMASK      = 0x000000C0,   /* Link Speed Setting */
176         LspeedSHIFT     = 6,
177         Lspeed10        = 0x00000000,   /* 10Mb/s */
178         Lspeed100       = 0x00000040,   /* 100Mb/s */
179         Lspeed1000      = 0x00000080,   /* 1000Mb/s */
180         Mtxckok         = 0x00000400,   /* MTX clock is running */
181         Pci66           = 0x00000800,   /* PCI Bus speed indication */
182         Bus64           = 0x00001000,   /* PCI Bus width indication */
183         Pcixmode        = 0x00002000,   /* PCI-X mode */
184         PcixspeedMASK   = 0x0000C000,   /* PCI-X bus speed */
185         PcixspeedSHIFT  = 14,
186         Pcix66          = 0x00000000,   /* 50-66MHz */
187         Pcix100         = 0x00004000,   /* 66-100MHz */
188         Pcix133         = 0x00008000,   /* 100-133MHz */
189 };
190
191 enum {                                  /* Ctrl and Status */
192         Fd              = 0x00000001,   /* Full-Duplex */
193         AsdvMASK        = 0x00000300,
194         AsdvSHIFT       = 8,
195         Asdv10          = 0x00000000,   /* 10Mb/s */
196         Asdv100         = 0x00000100,   /* 100Mb/s */
197         Asdv1000        = 0x00000200,   /* 1000Mb/s */
198 };
199
200 enum {                                  /* Eecd */
201         Sk              = 0x00000001,   /* Clock input to the EEPROM */
202         Cs              = 0x00000002,   /* Chip Select */
203         Di              = 0x00000004,   /* Data Input to the EEPROM */
204         Do              = 0x00000008,   /* Data Output from the EEPROM */
205         Areq            = 0x00000040,   /* EEPROM Access Request */
206         Agnt            = 0x00000080,   /* EEPROM Access Grant */
207         Eepresent       = 0x00000100,   /* EEPROM Present */
208         Eesz256         = 0x00000200,   /* EEPROM is 256 words not 64 */
209         Eeszaddr        = 0x00000400,   /* EEPROM size for 8254[17] */
210         Spi             = 0x00002000,   /* EEPROM is SPI not Microwire */
211 };
212
213 enum {                                  /* Ctrlext */
214         Gpien           = 0x0000000F,   /* General Purpose Interrupt Enables */
215         SwdpinshiMASK   = 0x000000F0,   /* Software Defined Pins - hi nibble */
216         SwdpinshiSHIFT  = 4,
217         SwdpiohiMASK    = 0x00000F00,   /* Software Defined Pins - I or O */
218         SwdpiohiSHIFT   = 8,
219         Asdchk          = 0x00001000,   /* ASD Check */
220         Eerst           = 0x00002000,   /* EEPROM Reset */
221         Ips             = 0x00004000,   /* Invert Power State */
222         Spdbyps         = 0x00008000,   /* Speed Select Bypass */
223 };
224
225 enum {                                  /* EEPROM content offsets */
226         Ea              = 0x00,         /* Ethernet Address */
227         Cf              = 0x03,         /* Compatibility Field */
228         Pba             = 0x08,         /* Printed Board Assembly number */
229         Icw1            = 0x0A,         /* Initialization Control Word 1 */
230         Sid             = 0x0B,         /* Subsystem ID */
231         Svid            = 0x0C,         /* Subsystem Vendor ID */
232         Did             = 0x0D,         /* Device ID */
233         Vid             = 0x0E,         /* Vendor ID */
234         Icw2            = 0x0F,         /* Initialization Control Word 2 */
235 };
236
237 enum {                                  /* Mdic */
238         MDIdMASK        = 0x0000FFFF,   /* Data */
239         MDIdSHIFT       = 0,
240         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
241         MDIrSHIFT       = 16,
242         MDIpMASK        = 0x03E00000,   /* PHY Address */
243         MDIpSHIFT       = 21,
244         MDIwop          = 0x04000000,   /* Write Operation */
245         MDIrop          = 0x08000000,   /* Read Operation */
246         MDIready        = 0x10000000,   /* End of Transaction */
247         MDIie           = 0x20000000,   /* Interrupt Enable */
248         MDIe            = 0x40000000,   /* Error */
249 };
250
251 enum {                                  /* Icr, Ics, Ims, Imc */
252         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
253         Txqe            = 0x00000002,   /* Transmit Queue Empty */
254         Lsc             = 0x00000004,   /* Link Status Change */
255         Rxseq           = 0x00000008,   /* Receive Sequence Error */
256         Rxdmt0          = 0x00000010,   /* Rd Minimum Threshold Reached */
257         Rxo             = 0x00000040,   /* Receiver Overrun */
258         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
259         Mdac            = 0x00000200,   /* MDIO Access Completed */
260         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
261         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
262         Gpi1            = 0x00001000,
263         Gpi2            = 0x00002000,
264         Gpi3            = 0x00004000,
265 };
266
267 /*
268  * The Mdic register isn't implemented on the 82543GC,
269  * the software defined pins are used instead.
270  * These definitions work for the Intel PRO/1000 T Server Adapter.
271  * The direction pin bits are read from the EEPROM.
272  */
273 enum {
274         Mdd             = ((1<<2)<<SwdpinsloSHIFT),     /* data */
275         Mddo            = ((1<<2)<<SwdpioloSHIFT),      /* pin direction */
276         Mdc             = ((1<<3)<<SwdpinsloSHIFT),     /* clock */
277         Mdco            = ((1<<3)<<SwdpioloSHIFT),      /* pin direction */
278         Mdr             = ((1<<0)<<SwdpinshiSHIFT),     /* reset */
279         Mdro            = ((1<<0)<<SwdpiohiSHIFT),      /* pin direction */
280 };
281
282 enum {                                  /* Txcw */
283         TxcwFd          = 0x00000020,   /* Full Duplex */
284         TxcwHd          = 0x00000040,   /* Half Duplex */
285         TxcwPauseMASK   = 0x00000180,   /* Pause */
286         TxcwPauseSHIFT  = 7,
287         TxcwPs          = (1<<TxcwPauseSHIFT),  /* Pause Supported */
288         TxcwAs          = (2<<TxcwPauseSHIFT),  /* Asymmetric FC desired */
289         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
290         TxcwRfiSHIFT    = 12,
291         TxcwNpr         = 0x00008000,   /* Next Page Request */
292         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
293         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
294 };
295
296 enum {                                  /* Rxcw */
297         Rxword          = 0x0000FFFF,   /* Data from auto-negotiation process */
298         Rxnocarrier     = 0x04000000,   /* Carrier Sense indication */
299         Rxinvalid       = 0x08000000,   /* Invalid Symbol during configuration */
300         Rxchange        = 0x10000000,   /* Change to the Rxword indication */
301         Rxconfig        = 0x20000000,   /* /C/ order set reception indication */
302         Rxsync          = 0x40000000,   /* Lost bit synchronization indication */
303         Anc             = 0x80000000,   /* Auto Negotiation Complete */
304 };
305
306 enum {                                  /* Rctl */
307         Rrst            = 0x00000001,   /* Receiver Software Reset */
308         Ren             = 0x00000002,   /* Receiver Enable */
309         Sbp             = 0x00000004,   /* Store Bad Packets */
310         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
311         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
312         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
313         LbmMASK         = 0x000000C0,   /* Loopback Mode */
314         LbmOFF          = 0x00000000,   /* No Loopback */
315         LbmTBI          = 0x00000040,   /* TBI Loopback */
316         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
317         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
318         RdtmsMASK       = 0x00000300,   /* Rd Minimum Threshold Size */
319         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
320         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
321         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
322         MoMASK          = 0x00003000,   /* Multicast Offset */
323         Mo47b36         = 0x00000000,   /* bits [47:36] of received address */
324         Mo46b35         = 0x00001000,   /* bits [46:35] of received address */
325         Mo45b34         = 0x00002000,   /* bits [45:34] of received address */
326         Mo43b32         = 0x00003000,   /* bits [43:32] of received address */
327         Bam             = 0x00008000,   /* Broadcast Accept Mode */
328         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
329         Bsize2048       = 0x00000000,   /* Bsex = 0 */
330         Bsize1024       = 0x00010000,   /* Bsex = 0 */
331         Bsize512        = 0x00020000,   /* Bsex = 0 */
332         Bsize256        = 0x00030000,   /* Bsex = 0 */
333         Bsize16384      = 0x00010000,   /* Bsex = 1 */
334         Vfe             = 0x00040000,   /* VLAN Filter Enable */
335         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
336         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
337         Dpf             = 0x00400000,   /* Discard Pause Frames */
338         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
339         Bsex            = 0x02000000,   /* Buffer Size Extension */
340         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
341 };
342
343 enum {                                  /* Tctl */
344         Trst            = 0x00000001,   /* Transmitter Software Reset */
345         Ten             = 0x00000002,   /* Transmit Enable */
346         Psp             = 0x00000008,   /* Pad Short Packets */
347         CtMASK          = 0x00000FF0,   /* Collision Threshold */
348         CtSHIFT         = 4,
349         ColdMASK        = 0x003FF000,   /* Collision Distance */
350         ColdSHIFT       = 12,
351         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
352         Pbe             = 0x00800000,   /* Packet Burst Enable */
353         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
354         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
355 };
356
357 enum {                                  /* [RT]xdctl */
358         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
359         PthreshSHIFT    = 0,
360         HthreshMASK     = 0x00003F00,   /* Host Threshold */
361         HthreshSHIFT    = 8,
362         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
363         WthreshSHIFT    = 16,
364         Gran            = 0x01000000,   /* Granularity */
365         LthreshMASK     = 0xFE000000,   /* Low Threshold */
366         LthreshSHIFT    = 25,
367 };
368
369 enum {                                  /* Rxcsum */
370         PcssMASK        = 0x000000FF,   /* Packet Checksum Start */
371         PcssSHIFT       = 0,
372         Ipofl           = 0x00000100,   /* IP Checksum Off-load Enable */
373         Tuofl           = 0x00000200,   /* TCP/UDP Checksum Off-load Enable */
374 };
375
376 enum {                                  /* Manc */
377         Arpen           = 0x00002000,   /* Enable ARP Request Filtering */
378 };
379
380 enum {                                  /* Receive Delay Timer Ring */
381         DelayMASK       = 0x0000FFFF,   /* delay timer in 1.024nS increments */
382         DelaySHIFT      = 0,
383         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
384 };
385
386 typedef struct Rd {                     /* Receive Descriptor */
387         unsigned int    addr[2];
388         uint16_t        length;
389         uint16_t        checksum;
390         uint8_t status;
391         uint8_t errors;
392         uint16_t        special;
393 } Rd;
394
395 enum {                                  /* Rd status */
396         Rdd             = 0x01,         /* Descriptor Done */
397         Reop            = 0x02,         /* End of Packet */
398         Ixsm            = 0x04,         /* Ignore Checksum Indication */
399         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
400         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
401         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
402         Pif             = 0x80,         /* Passed in-exact filter */
403 };
404
405 enum {                                  /* Rd errors */
406         Ce              = 0x01,         /* CRC Error or Alignment Error */
407         Se              = 0x02,         /* Symbol Error */
408         Seq             = 0x04,         /* Sequence Error */
409         Cxe             = 0x10,         /* Carrier Extension Error */
410         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
411         Ipe             = 0x40,         /* IP Checksum Error */
412         Rxe             = 0x80,         /* RX Data Error */
413 };
414
415 typedef struct Td Td;
416 struct Td {                             /* Transmit Descriptor */
417         union {
418                 unsigned int    addr[2];        /* Data */
419                 struct {                /* Context */
420                         uint8_t ipcss;
421                         uint8_t ipcso;
422                         uint16_t        ipcse;
423                         uint8_t tucss;
424                         uint8_t tucso;
425                         uint16_t        tucse;
426                 };
427         };
428         unsigned int    control;
429         unsigned int    status;
430 };
431
432 enum {                                  /* Td control */
433         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
434         LenSHIFT        = 0,
435         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
436         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
437         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
438         Teop            = 0x01000000,   /* End of Packet (DD) */
439         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
440         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
441         Tse             = 0x04000000,   /* TCP Segmentation Enable */
442         Rs              = 0x08000000,   /* Report Status */
443         Rps             = 0x10000000,   /* Report Status Sent */
444         Dext            = 0x20000000,   /* Descriptor Extension */
445         Vle             = 0x40000000,   /* VLAN Packet Enable */
446         Ide             = 0x80000000,   /* Interrupt Delay Enable */
447 };
448
449 enum {                                  /* Td status */
450         Tdd             = 0x00000001,   /* Descriptor Done */
451         Ec              = 0x00000002,   /* Excess Collisions */
452         Lc              = 0x00000004,   /* Late Collision */
453         Tu              = 0x00000008,   /* Transmit Underrun */
454         Iixsm           = 0x00000100,   /* Insert IP Checksum */
455         Itxsm           = 0x00000200,   /* Insert TCP/UDP Checksum */
456         HdrlenMASK      = 0x0000FF00,   /* Header Length (Tse) */
457         HdrlenSHIFT     = 8,
458         VlanMASK        = 0x0FFF0000,   /* VLAN Identifier */
459         VlanSHIFT       = 16,
460         Tcfi            = 0x10000000,   /* Canonical Form Indicator */
461         PriMASK         = 0xE0000000,   /* User Priority */
462         PriSHIFT        = 29,
463         MssMASK         = 0xFFFF0000,   /* Maximum Segment Size (Tse) */
464         MssSHIFT        = 16,
465 };
466
467 enum {
468         Nrd             = 256,          /* multiple of 8 */
469         Ntd             = 64,           /* multiple of 8 */
470         Nrb             = 1024,         /* private receive buffers per Ctlr */
471         Rbsz            = 2048,
472 };
473
474 struct ctlr {
475         int     port;
476         struct pci_device *pci;
477         struct ctlr*    next;
478         struct ether*   edev;
479         int     active;
480         int     started;
481         int     id;
482         int     cls;
483         uint16_t        eeprom[0x40];
484
485         qlock_t alock;                  /* attach */
486         void*   alloc;                  /* receive/transmit descriptors */
487         int     nrd;
488         int     ntd;
489         int     nrb;                    /* how many this Ctlr has in the pool */
490
491         int*    nic;
492         spinlock_t      imlock;
493         int     im;                     /* interrupt mask */
494
495         struct mii*     mii;
496         struct rendez   lrendez;
497         int     lim;
498
499         int     link;
500
501         qlock_t slock;
502         unsigned int    statistics[Nstatistics];
503         unsigned int    lsleep;
504         unsigned int    lintr;
505         unsigned int    rsleep;
506         unsigned int    rintr;
507         unsigned int    txdw;
508         unsigned int    tintr;
509         unsigned int    ixsm;
510         unsigned int    ipcs;
511         unsigned int    tcpcs;
512
513         uint8_t ra[Eaddrlen];           /* receive address */
514         uint32_t        mta[128];               /* multicast table array */
515
516         struct rendez   rrendez;
517         int     rim;
518         int     rdfree;
519         Rd*     rdba;                   /* receive descriptor base address */
520         struct block**  rb;                     /* receive buffers */
521         int     rdh;                    /* receive descriptor head */
522         int     rdt;                    /* receive descriptor tail */
523         int     rdtr;                   /* receive delay timer ring value */
524
525         spinlock_t      tlock;
526         int     tbusy;
527         int     tdfree;
528         Td*     tdba;                   /* transmit descriptor base address */
529         struct block**  tb;                     /* transmit buffers */
530         int     tdh;                    /* transmit descriptor head */
531         int     tdt;                    /* transmit descriptor tail */
532
533         int     txcw;
534         int     fcrtl;
535         int     fcrth;
536 };
537
538 static inline uint32_t csr32r(struct ctlr *c, uintptr_t reg)
539 {
540         return read_mmreg32((uintptr_t)(c->nic + (reg / 4)));
541 }
542
543 static inline void csr32w(struct ctlr *c, uintptr_t reg, uint32_t val)
544 {
545         write_mmreg32((uintptr_t)(c->nic + (reg / 4)), val);
546 }
547
548 static struct ctlr* igbectlrhead;
549 static struct ctlr* igbectlrtail;
550
551 /* lock for igberpool (free receive Blocks) */
552 static spinlock_t igberblock = SPINLOCK_INITIALIZER_IRQSAVE;
553 static struct block* igberbpool;        /* receive Blocks for all igbe controllers */
554
555 static char* statistics[Nstatistics] = {
556         "CRC Error",
557         "Alignment Error",
558         "Symbol Error",
559         "RX Error",
560         "Missed Packets",
561         "Single Collision",
562         "Excessive Collisions",
563         "Multiple Collision",
564         "Late Collisions",
565         NULL,
566         "Collision",
567         "Transmit Underrun",
568         "Defer",
569         "Transmit - No CRS",
570         "Sequence Error",
571         "Carrier Extension Error",
572         "Receive Error Length",
573         NULL,
574         "XON Received",
575         "XON Transmitted",
576         "XOFF Received",
577         "XOFF Transmitted",
578         "FC Received Unsupported",
579         "Packets Received (64 Bytes)",
580         "Packets Received (65-127 Bytes)",
581         "Packets Received (128-255 Bytes)",
582         "Packets Received (256-511 Bytes)",
583         "Packets Received (512-1023 Bytes)",
584         "Packets Received (1024-1522 Bytes)",
585         "Good Packets Received",
586         "Broadcast Packets Received",
587         "Multicast Packets Received",
588         "Good Packets Transmitted",
589         NULL,
590         "Good Octets Received",
591         NULL,
592         "Good Octets Transmitted",
593         NULL,
594         NULL,
595         NULL,
596         "Receive No Buffers",
597         "Receive Undersize",
598         "Receive Fragment",
599         "Receive Oversize",
600         "Receive Jabber",
601         NULL,
602         NULL,
603         NULL,
604         "Total Octets Received",
605         NULL,
606         "Total Octets Transmitted",
607         NULL,
608         "Total Packets Received",
609         "Total Packets Transmitted",
610         "Packets Transmitted (64 Bytes)",
611         "Packets Transmitted (65-127 Bytes)",
612         "Packets Transmitted (128-255 Bytes)",
613         "Packets Transmitted (256-511 Bytes)",
614         "Packets Transmitted (512-1023 Bytes)",
615         "Packets Transmitted (1024-1522 Bytes)",
616         "Multicast Packets Transmitted",
617         "Broadcast Packets Transmitted",
618         "TCP Segmentation Context Transmitted",
619         "TCP Segmentation Context Fail",
620 };
621
622 static void igbe_print_rd(struct Rd *rd)
623 {
624         printk("Rd %p: stat 0x%02x, err 0x%02x, len 0x%04x, check 0x%04x, "
625                "spec 0x%04x, addr[1] 0x%08x, addr[0] 0x%08x\n", rd,
626                    rd->status,
627                    rd->errors,
628                    rd->length,
629                    rd->checksum,
630                    rd->special,
631                    rd->addr[1],
632                    rd->addr[0]);
633 }
634
635 static long
636 igbeifstat(struct ether* edev, void* a, long n, uint32_t offset)
637 {
638         struct ctlr *ctlr;
639         char *p, *s;
640         int i, l, r;
641         uint64_t tuvl, ruvl;
642
643         ctlr = edev->ctlr;
644         qlock(&ctlr->slock);
645         p = kzmalloc(READSTR, 0);
646         if(p == NULL) {
647                 qunlock(&ctlr->slock);
648                 error(Enomem);
649         }
650         l = 0;
651         for(i = 0; i < Nstatistics; i++){
652                 r = csr32r(ctlr, Statistics+i*4);
653                 if((s = statistics[i]) == NULL)
654                         continue;
655                 switch(i){
656                 case Gorcl:
657                 case Gotcl:
658                 case Torl:
659                 case Totl:
660                         ruvl = r;
661                         ruvl += ((uint64_t)csr32r(ctlr, Statistics+(i+1)*4))<<32;
662                         tuvl = ruvl;
663                         tuvl += ctlr->statistics[i];
664                         tuvl += ((uint64_t)ctlr->statistics[i+1])<<32;
665                         if(tuvl == 0)
666                                 continue;
667                         ctlr->statistics[i] = tuvl;
668                         ctlr->statistics[i+1] = tuvl>>32;
669                         l += snprintf(p+l, READSTR-l, "%s: %llud %llud\n",
670                                 s, tuvl, ruvl);
671                         i++;
672                         break;
673
674                 default:
675                         ctlr->statistics[i] += r;
676                         if(ctlr->statistics[i] == 0)
677                                 continue;
678                         l += snprintf(p+l, READSTR-l, "%s: %ud %ud\n",
679                                 s, ctlr->statistics[i], r);
680                         break;
681                 }
682         }
683
684         l += snprintf(p+l, READSTR-l, "lintr: %ud %ud\n",
685                 ctlr->lintr, ctlr->lsleep);
686         l += snprintf(p+l, READSTR-l, "rintr: %ud %ud\n",
687                 ctlr->rintr, ctlr->rsleep);
688         l += snprintf(p+l, READSTR-l, "tintr: %ud %ud\n",
689                 ctlr->tintr, ctlr->txdw);
690         l += snprintf(p+l, READSTR-l, "ixcs: %ud %ud %ud\n",
691                 ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
692         l += snprintf(p+l, READSTR-l, "rdtr: %ud\n", ctlr->rdtr);
693         l += snprintf(p+l, READSTR-l, "Ctrlext: %08x\n", csr32r(ctlr, Ctrlext));
694
695         l += snprintf(p+l, READSTR-l, "eeprom:");
696         for(i = 0; i < 0x40; i++){
697                 if(i && ((i & 0x07) == 0))
698                         l += snprintf(p+l, READSTR-l, "\n       ");
699                 l += snprintf(p+l, READSTR-l, " %4.4uX", ctlr->eeprom[i]);
700         }
701         l += snprintf(p+l, READSTR-l, "\n");
702
703         if(ctlr->mii != NULL && ctlr->mii->curphy != NULL){
704                 l += snprintf(p+l, READSTR-l, "phy:   ");
705                 for(i = 0; i < NMiiPhyr; i++){
706                         if(i && ((i & 0x07) == 0))
707                                 l += snprintf(p+l, READSTR-l, "\n       ");
708                         r = miimir(ctlr->mii, i);
709                         l += snprintf(p+l, READSTR-l, " %4.4uX", r);
710                 }
711                 snprintf(p+l, READSTR-l, "\n");
712         }
713         n = readstr(offset, a, n, p);
714         kfree(p);
715         qunlock(&ctlr->slock);
716
717         return n;
718 }
719
720 enum {
721         CMrdtr,
722 };
723
724 static struct cmdtab igbectlmsg[] = {
725         {CMrdtr,        "rdtr", 2},
726 };
727
728 static long
729 igbectl(struct ether* edev, void* buf, long n)
730 {
731         ERRSTACK(2);
732         int v;
733         char *p;
734         struct ctlr *ctlr;
735         struct cmdbuf *cb;
736         struct cmdtab *ct;
737
738         if((ctlr = edev->ctlr) == NULL)
739                 error(Enonexist);
740
741         cb = parsecmd(buf, n);
742         if(waserror()){
743                 kfree(cb);
744                 nexterror();
745         }
746
747         ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
748         switch(ct->index){
749         case CMrdtr:
750                 v = strtol(cb->f[1], &p, 0);
751                 if(v < 0 || p == cb->f[1] || v > 0xFFFF)
752                         error(Ebadarg);
753                 ctlr->rdtr = v;
754                 csr32w(ctlr, Rdtr, Fpd|v);
755                 break;
756         }
757         kfree(cb);
758         poperror();
759
760         return n;
761 }
762
763 static void
764 igbepromiscuous(void* arg, int on)
765 {
766         int rctl;
767         struct ctlr *ctlr;
768         struct ether *edev;
769
770         edev = arg;
771         ctlr = edev->ctlr;
772
773         rctl = csr32r(ctlr, Rctl);
774         rctl &= ~MoMASK;
775         rctl |= Mo47b36;
776         if(on)
777                 rctl |= Upe|Mpe;
778         else
779                 rctl &= ~(Upe|Mpe);
780         csr32w(ctlr, Rctl, rctl|Mpe);   /* temporarily keep Mpe on */
781 }
782
783 static void
784 igbemulticast(void* arg, uint8_t* addr, int add)
785 {
786         int bit, x;
787         struct ctlr *ctlr;
788         struct ether *edev;
789
790         edev = arg;
791         ctlr = edev->ctlr;
792
793         x = addr[5]>>1;
794         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
795         /*
796          * multiple ether addresses can hash to the same filter bit,
797          * so it's never safe to clear a filter bit.
798          * if we want to clear filter bits, we need to keep track of
799          * all the multicast addresses in use, clear all the filter bits,
800          * then set the ones corresponding to in-use addresses.
801          */
802         if(add)
803                 ctlr->mta[x] |= 1<<bit;
804 //      else
805 //              ctlr->mta[x] &= ~(1<<bit);
806
807         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
808 }
809
810 static struct block*
811 igberballoc(void)
812 {
813         struct block *bp;
814
815         ilock(&igberblock);
816         if((bp = igberbpool) != NULL){
817                 igberbpool = bp->next;
818                 bp->next = NULL;
819                 /* _xinc(&bp->ref);     prevent bp from being freed */
820         }
821         iunlock(&igberblock);
822
823         return bp;
824 }
825
826 static void
827 igberbfree(struct block* bp)
828 {
829         bp->rp = bp->lim - Rbsz;
830         bp->wp = bp->rp;
831         bp->flag &= ~BCKSUM_FLAGS;
832
833         ilock(&igberblock);
834         bp->next = igberbpool;
835         igberbpool = bp;
836         iunlock(&igberblock);
837 }
838
839 static void
840 igbeim(struct ctlr* ctlr, int im)
841 {
842         ilock(&ctlr->imlock);
843         ctlr->im |= im;
844         csr32w(ctlr, Ims, ctlr->im);
845         iunlock(&ctlr->imlock);
846 }
847
848 static int
849 igbelim(void* ctlr)
850 {
851         return ((struct ctlr*)ctlr)->lim != 0;
852 }
853
854 static void
855 igbelproc(void* arg)
856 {
857         struct ctlr *ctlr;
858         struct ether *edev;
859         struct miiphy *phy;
860         int ctrl, r;
861
862         edev = arg;
863         ctlr = edev->ctlr;
864         for(;;){
865                 /* plan9 originally had a busy loop here (just called continue).  though
866                  * either you have the mii or you don't.  i don't think it'll magically
867                  * show up later (it should have been initialized during pnp/pci, which
868                  * is before attach, which is before lproc).  -brho */
869                 if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
870                         printk("[kernel] igbelproc can't find a mii/curphy, aborting!\n");
871                         /* name alloc'd in attach */
872                         kfree(per_cpu_info[core_id()].cur_kthread->name);
873                         return;
874                 }
875                 /*
876                  * To do:
877                  *      logic to manage status change,
878                  *      this is incomplete but should work
879                  *      one time to set up the hardware.
880                  *
881                  *      MiiPhy.speed, etc. should be in Mii.
882                  */
883                 if(miistatus(ctlr->mii) < 0)
884                         //continue;     /* this comment out was plan9, not brho */
885                         goto enable;
886
887                 phy = ctlr->mii->curphy;
888                 ctrl = csr32r(ctlr, Ctrl);
889
890                 switch(ctlr->id){
891                 case i82543gc:
892                 case i82544ei:
893                 case i82544eif:
894                 default:
895                         if(!(ctrl & Asde)){
896                                 ctrl &= ~(SspeedMASK|Ilos|Fd);
897                                 ctrl |= Frcdplx|Frcspd;
898                                 if(phy->speed == 1000)
899                                         ctrl |= Sspeed1000;
900                                 else if(phy->speed == 100)
901                                         ctrl |= Sspeed100;
902                                 if(phy->fd)
903                                         ctrl |= Fd;
904                         }
905                         break;
906
907                 case i82540em:
908                 case i82540eplp:
909                 case i82547gi:
910                 case i82541gi:
911                 case i82541gi2:
912                 case i82541pi:
913                         break;
914                 }
915
916                 /*
917                  * Collision Distance.
918                  */
919                 r = csr32r(ctlr, Tctl);
920                 r &= ~ColdMASK;
921                 if(phy->fd)
922                         r |= 64<<ColdSHIFT;
923                 else
924                         r |= 512<<ColdSHIFT;
925                 csr32w(ctlr, Tctl, r);
926
927                 /*
928                  * Flow control.
929                  */
930                 if(phy->rfc)
931                         ctrl |= Rfce;
932                 if(phy->tfc)
933                         ctrl |= Tfce;
934                 csr32w(ctlr, Ctrl, ctrl);
935
936 enable:
937                 ctlr->lim = 0;
938                 igbeim(ctlr, Lsc);
939
940                 ctlr->lsleep++;
941                 rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
942         }
943 }
944
945 static void
946 igbetxinit(struct ctlr* ctlr)
947 {
948         int i, r;
949         struct block *bp;
950
951         csr32w(ctlr, Tctl, (0x0F<<CtSHIFT)|Psp|(66<<ColdSHIFT));
952         switch(ctlr->id){
953         default:
954                 r = 6;
955                 break;
956         case i82543gc:
957         case i82544ei:
958         case i82544eif:
959         case i82544gc:
960         case i82540em:
961         case i82540eplp:
962         case i82541ei:
963         case i82541gi:
964         case i82541gi2:
965         case i82541pi:
966         case i82545em:
967         case i82545gmc:
968         case i82546gb:
969         case i82546eb:
970         case i82547ei:
971         case i82547gi:
972                 r = 8;
973                 break;
974         }
975         csr32w(ctlr, Tipg, (6<<20)|(8<<10)|r);
976         csr32w(ctlr, Ait, 0);
977         csr32w(ctlr, Txdmac, 0);
978         csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
979         csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
980         csr32w(ctlr, Tdlen, ctlr->ntd*sizeof(Td));
981         ctlr->tdh = PREV_RING(0, ctlr->ntd);
982         csr32w(ctlr, Tdh, 0);
983         ctlr->tdt = 0;
984         csr32w(ctlr, Tdt, 0);
985
986         for(i = 0; i < ctlr->ntd; i++){
987                 if((bp = ctlr->tb[i]) != NULL){
988                         ctlr->tb[i] = NULL;
989                         freeb(bp);
990                 }
991                 memset(&ctlr->tdba[i], 0, sizeof(Td));
992         }
993         ctlr->tdfree = ctlr->ntd;
994
995         csr32w(ctlr, Tidv, 128);
996         r = (4<<WthreshSHIFT)|(4<<HthreshSHIFT)|(8<<PthreshSHIFT);
997
998         switch(ctlr->id){
999         default:
1000                 break;
1001         case i82540em:
1002         case i82540eplp:
1003         case i82547gi:
1004         case i82545em:
1005         case i82545gmc:
1006         case i82546gb:
1007         case i82546eb:
1008         case i82541gi:
1009         case i82541gi2:
1010         case i82541pi:
1011                 r = csr32r(ctlr, Txdctl);
1012                 r &= ~WthreshMASK;
1013                 r |= Gran|(4<<WthreshSHIFT);
1014
1015                 csr32w(ctlr, Tadv, 64);
1016                 break;
1017         }
1018
1019         csr32w(ctlr, Txdctl, r);
1020
1021         r = csr32r(ctlr, Tctl);
1022         r |= Ten;
1023         csr32w(ctlr, Tctl, r);
1024 }
1025
1026 static void
1027 igbetransmit(struct ether* edev)
1028 {
1029         Td *td;
1030         struct block *bp;
1031         struct ctlr *ctlr;
1032         int tdh, tdt;
1033
1034         ctlr = edev->ctlr;
1035
1036         ilock(&ctlr->tlock);
1037
1038         /*
1039          * Free any completed packets
1040          */
1041         tdh = ctlr->tdh;
1042         while(NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)){
1043                 if((bp = ctlr->tb[tdh]) != NULL){
1044                         ctlr->tb[tdh] = NULL;
1045                         freeb(bp);
1046                 }
1047                 memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1048                 tdh = NEXT_RING(tdh, ctlr->ntd);
1049         }
1050         ctlr->tdh = tdh;
1051
1052         /*
1053          * Try to fill the ring back up.
1054          */
1055         tdt = ctlr->tdt;
1056         while(NEXT_RING(tdt, ctlr->ntd) != tdh){
1057                 if((bp = qget(edev->oq)) == NULL)
1058                         break;
1059                 td = &ctlr->tdba[tdt];
1060                 td->addr[0] = paddr_low32(bp->rp);
1061                 td->addr[1] = paddr_high32(bp->rp);
1062                 td->control = ((BLEN(bp) & LenMASK)<<LenSHIFT);
1063                 td->control |= Dext|Ifcs|Teop|DtypeDD;
1064                 ctlr->tb[tdt] = bp;
1065                 tdt = NEXT_RING(tdt, ctlr->ntd);
1066                 if(NEXT_RING(tdt, ctlr->ntd) == tdh){
1067                         td->control |= Rs;
1068                         ctlr->txdw++;
1069                         ctlr->tdt = tdt;
1070                         csr32w(ctlr, Tdt, tdt);
1071                         igbeim(ctlr, Txdw);
1072                         break;
1073                 }
1074                 ctlr->tdt = tdt;
1075                 csr32w(ctlr, Tdt, tdt);
1076         }
1077
1078         iunlock(&ctlr->tlock);
1079 }
1080
1081 static void
1082 igbereplenish(struct ctlr* ctlr)
1083 {
1084         Rd *rd;
1085         int rdt;
1086         struct block *bp;
1087
1088         rdt = ctlr->rdt;
1089         while(NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh){
1090                 rd = &ctlr->rdba[rdt];
1091                 if(ctlr->rb[rdt] == NULL){
1092                         bp = igberballoc();
1093                         if(bp == NULL){
1094                                 /* needs to be a safe print for interrupt level */
1095                                 printk("#l%d: igbereplenish: no available buffers\n",
1096                                         ctlr->edev->ctlrno);
1097                                 break;
1098                         }
1099                         ctlr->rb[rdt] = bp;
1100                         rd->addr[0] = paddr_low32(bp->rp);
1101                         rd->addr[1] = paddr_high32(bp->rp);
1102                 }
1103                 wmb();  /* ensure prev rd writes come before status = 0. */
1104                 rd->status = 0;
1105                 rdt = NEXT_RING(rdt, ctlr->nrd);
1106                 ctlr->rdfree++;
1107         }
1108         ctlr->rdt = rdt;
1109         csr32w(ctlr, Rdt, rdt);
1110 }
1111
1112 static void
1113 igberxinit(struct ctlr* ctlr)
1114 {
1115         int i;
1116         struct block *bp;
1117
1118         /* temporarily keep Mpe on */
1119         csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF|Mpe);
1120         csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1121         csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1122         csr32w(ctlr, Rdlen, ctlr->nrd*sizeof(Rd));
1123         ctlr->rdh = 0;
1124         csr32w(ctlr, Rdh, 0);
1125         ctlr->rdt = 0;
1126         csr32w(ctlr, Rdt, 0);
1127         ctlr->rdtr = 0;
1128         csr32w(ctlr, Rdtr, Fpd|0);
1129
1130         for(i = 0; i < ctlr->nrd; i++){
1131                 if((bp = ctlr->rb[i]) != NULL){
1132                         ctlr->rb[i] = NULL;
1133                         freeb(bp);
1134                 }
1135         }
1136         igbereplenish(ctlr);
1137
1138         switch(ctlr->id){
1139         case i82540em:
1140         case i82540eplp:
1141         case i82541gi:
1142         case i82541gi2:
1143         case i82541pi:
1144         case i82545em:
1145         case i82545gmc:
1146         case i82546gb:
1147         case i82546eb:
1148         case i82547gi:
1149                 csr32w(ctlr, Radv, 64);
1150                 break;
1151         }
1152         csr32w(ctlr, Rxdctl, (8<<WthreshSHIFT)|(8<<HthreshSHIFT)|4);
1153
1154         /*
1155          * Enable checksum offload.
1156          */
1157         csr32w(ctlr, Rxcsum, Tuofl|Ipofl|(ETHERHDRSIZE<<PcssSHIFT));
1158 }
1159
1160 static int
1161 igberim(void* ctlr)
1162 {
1163         return ((struct ctlr*)ctlr)->rim != 0;
1164 }
1165
1166 static void
1167 igberproc(void* arg)
1168 {
1169         Rd *rd;
1170         struct block *bp;
1171         struct ctlr *ctlr;
1172         int r, rdh;
1173         struct ether *edev;
1174
1175         edev = arg;
1176         ctlr = edev->ctlr;
1177
1178         igberxinit(ctlr);
1179         r = csr32r(ctlr, Rctl);
1180         r |= Ren;
1181         csr32w(ctlr, Rctl, r);
1182
1183         for(;;){
1184                 ctlr->rim = 0;
1185                 igbeim(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq);
1186                 ctlr->rsleep++;
1187                 rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1188
1189                 rdh = ctlr->rdh;
1190                 for(;;){
1191                         rd = &ctlr->rdba[rdh];
1192
1193                         if(!(rd->status & Rdd))
1194                                 break;
1195
1196                         /*
1197                          * Accept eop packets with no errors.
1198                          * With no errors and the Ixsm bit set,
1199                          * the descriptor status Tpcs and Ipcs bits give
1200                          * an indication of whether the checksums were
1201                          * calculated and valid.
1202                          */
1203                         if((rd->status & Reop) && rd->errors == 0){
1204                                 bp = ctlr->rb[rdh];
1205                                 ctlr->rb[rdh] = NULL;
1206                                 bp->wp += rd->length;
1207                                 bp->next = NULL;
1208                                 if(!(rd->status & Ixsm)){
1209                                         ctlr->ixsm++;
1210                                         if(rd->status & Ipcs){
1211                                                 /*
1212                                                  * IP checksum calculated
1213                                                  * (and valid as errors == 0).
1214                                                  */
1215                                                 ctlr->ipcs++;
1216                                                 bp->flag |= Bipck;
1217                                         }
1218                                         if(rd->status & Tcpcs){
1219                                                 /*
1220                                                  * TCP/UDP checksum calculated
1221                                                  * (and valid as errors == 0).
1222                                                  */
1223                                                 ctlr->tcpcs++;
1224                                                 bp->flag |= Btcpck|Budpck;
1225                                         }
1226                                         bp->checksum = rd->checksum;
1227                                         bp->flag |= Bpktck;
1228                                 }
1229                                 etheriq(edev, bp, 1);
1230                         }
1231                         else if(ctlr->rb[rdh] != NULL){
1232                                 freeb(ctlr->rb[rdh]);
1233                                 ctlr->rb[rdh] = NULL;
1234                         }
1235
1236                         memset(rd, 0, sizeof(Rd));
1237                         wmb();  /* make sure the zeroing happens before free (i think) */
1238                         ctlr->rdfree--;
1239                         rdh = NEXT_RING(rdh, ctlr->nrd);
1240                 }
1241                 ctlr->rdh = rdh;
1242
1243                 if(ctlr->rdfree < ctlr->nrd/2 || (ctlr->rim & Rxdmt0))
1244                         igbereplenish(ctlr);
1245         }
1246 }
1247
1248 static void
1249 igbeattach(struct ether* edev)
1250 {
1251         ERRSTACK(1);
1252         struct block *bp;
1253         struct ctlr *ctlr;
1254         char *name;
1255
1256         ctlr = edev->ctlr;
1257         ctlr->edev = edev;                      /* point back to Ether* */
1258         qlock(&ctlr->alock);
1259         if(ctlr->alloc != NULL){                        /* already allocated? */
1260                 qunlock(&ctlr->alock);
1261                 return;
1262         }
1263
1264         ctlr->tb = NULL;
1265         ctlr->rb = NULL;
1266         ctlr->alloc = NULL;
1267         ctlr->nrb = 0;
1268         if(waserror()){
1269                 while(ctlr->nrb > 0){
1270                         bp = igberballoc();
1271                         bp->free = NULL;
1272                         freeb(bp);
1273                         ctlr->nrb--;
1274                 }
1275                 kfree(ctlr->tb);
1276                 ctlr->tb = NULL;
1277                 kfree(ctlr->rb);
1278                 ctlr->rb = NULL;
1279                 kfree(ctlr->alloc);
1280                 ctlr->alloc = NULL;
1281                 qunlock(&ctlr->alock);
1282                 nexterror();
1283         }
1284
1285         ctlr->nrd = Nrd;
1286         ctlr->ntd = Ntd;
1287         ctlr->alloc = kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1288         if(ctlr->alloc == NULL) {
1289                 printd("igbe: can't allocate ctlr->alloc\n");
1290                 error(Enomem);
1291         }
1292         ctlr->rdba = (Rd*)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1293         ctlr->tdba = (Td*)(ctlr->rdba+ctlr->nrd);
1294
1295         ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1296         ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1297         if (ctlr->rb == NULL || ctlr->tb == NULL) {
1298                 printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1299                 error(Enomem);
1300         }
1301
1302         for(ctlr->nrb = 0; ctlr->nrb < Nrb; ctlr->nrb++){
1303                 if((bp = allocb(Rbsz)) == NULL)
1304                         break;
1305                 bp->free = igberbfree;
1306                 freeb(bp);
1307         }
1308
1309         /* the ktasks should free these names, if they ever exit */
1310         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1311         snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1312         ktask(name, igbelproc, edev);
1313
1314         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1315         snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1316         ktask(name, igberproc, edev);
1317
1318         igbetxinit(ctlr);
1319
1320         qunlock(&ctlr->alock);
1321         poperror();
1322 }
1323
1324 static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1325 {
1326         struct ctlr *ctlr;
1327         struct ether *edev;
1328         int icr, im, txdw;
1329
1330         edev = arg;
1331         ctlr = edev->ctlr;
1332
1333         ilock(&ctlr->imlock);
1334         csr32w(ctlr, Imc, ~0);
1335         im = ctlr->im;
1336         txdw = 0;
1337
1338         while((icr = csr32r(ctlr, Icr) & ctlr->im) != 0){
1339                 if(icr & Lsc){
1340                         im &= ~Lsc;
1341                         ctlr->lim = icr & Lsc;
1342                         rendez_wakeup(&ctlr->lrendez);
1343                         ctlr->lintr++;
1344                 }
1345                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq)){
1346                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq);
1347                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq);
1348                         rendez_wakeup(&ctlr->rrendez);
1349                         ctlr->rintr++;
1350                 }
1351                 if(icr & Txdw){
1352                         im &= ~Txdw;
1353                         txdw++;
1354                         ctlr->tintr++;
1355                 }
1356         }
1357
1358         ctlr->im = im;
1359         csr32w(ctlr, Ims, im);
1360         iunlock(&ctlr->imlock);
1361
1362         if(txdw)
1363                 igbetransmit(edev);
1364 }
1365
1366 static int
1367 i82543mdior(struct ctlr* ctlr, int n)
1368 {
1369         int ctrl, data, i, r;
1370
1371         /*
1372          * Read n bits from the Management Data I/O Interface.
1373          */
1374         ctrl = csr32r(ctlr, Ctrl);
1375         r = (ctrl & ~Mddo)|Mdco;
1376         data = 0;
1377         for(i = n-1; i >= 0; i--){
1378                 if(csr32r(ctlr, Ctrl) & Mdd)
1379                         data |= (1<<i);
1380                 csr32w(ctlr, Ctrl, Mdc|r);
1381                 csr32w(ctlr, Ctrl, r);
1382         }
1383         csr32w(ctlr, Ctrl, ctrl);
1384
1385         return data;
1386 }
1387
1388 static int
1389 i82543mdiow(struct ctlr* ctlr, int bits, int n)
1390 {
1391         int ctrl, i, r;
1392
1393         /*
1394          * Write n bits to the Management Data I/O Interface.
1395          */
1396         ctrl = csr32r(ctlr, Ctrl);
1397         r = Mdco|Mddo|ctrl;
1398         for(i = n-1; i >= 0; i--){
1399                 if(bits & (1<<i))
1400                         r |= Mdd;
1401                 else
1402                         r &= ~Mdd;
1403                 csr32w(ctlr, Ctrl, Mdc|r);
1404                 csr32w(ctlr, Ctrl, r);
1405         }
1406         csr32w(ctlr, Ctrl, ctrl);
1407
1408         return 0;
1409 }
1410
1411 static int
1412 i82543miimir(struct mii* mii, int pa, int ra)
1413 {
1414         int data;
1415         struct ctlr *ctlr;
1416
1417         ctlr = mii->ctlr;
1418
1419         /*
1420          * MII Management Interface Read.
1421          *
1422          * Preamble;
1423          * ST+OP+PHYAD+REGAD;
1424          * TA + 16 data bits.
1425          */
1426         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1427         i82543mdiow(ctlr, 0x1800|(pa<<5)|ra, 14);
1428         data = i82543mdior(ctlr, 18);
1429
1430         if(data & 0x10000)
1431                 return -1;
1432
1433         return data & 0xFFFF;
1434 }
1435
1436 static int
1437 i82543miimiw(struct mii* mii, int pa, int ra, int data)
1438 {
1439         struct ctlr *ctlr;
1440
1441         ctlr = mii->ctlr;
1442
1443         /*
1444          * MII Management Interface Write.
1445          *
1446          * Preamble;
1447          * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1448          * Z.
1449          */
1450         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1451         data &= 0xFFFF;
1452         data |= (0x05<<(5+5+2+16))|(pa<<(5+2+16))|(ra<<(2+16))|(0x02<<16);
1453         i82543mdiow(ctlr, data, 32);
1454
1455         return 0;
1456 }
1457
1458 static int
1459 igbemiimir(struct mii* mii, int pa, int ra)
1460 {
1461         struct ctlr *ctlr;
1462         int mdic, timo;
1463
1464         ctlr = mii->ctlr;
1465
1466         csr32w(ctlr, Mdic, MDIrop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT));
1467         mdic = 0;
1468         for(timo = 64; timo; timo--){
1469                 mdic = csr32r(ctlr, Mdic);
1470                 if(mdic & (MDIe|MDIready))
1471                         break;
1472                 udelay(1);
1473         }
1474
1475         if((mdic & (MDIe|MDIready)) == MDIready)
1476                 return mdic & 0xFFFF;
1477         return -1;
1478 }
1479
1480 static int
1481 igbemiimiw(struct mii* mii, int pa, int ra, int data)
1482 {
1483         struct ctlr *ctlr;
1484         int mdic, timo;
1485
1486         ctlr = mii->ctlr;
1487
1488         data &= MDIdMASK;
1489         csr32w(ctlr, Mdic, MDIwop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT)|data);
1490         mdic = 0;
1491         for(timo = 64; timo; timo--){
1492                 mdic = csr32r(ctlr, Mdic);
1493                 if(mdic & (MDIe|MDIready))
1494                         break;
1495                 udelay(1);
1496         }
1497         if((mdic & (MDIe|MDIready)) == MDIready)
1498                 return 0;
1499         return -1;
1500 }
1501
1502 static int
1503 i82543miirw(struct mii* mii, int write, int pa, int ra, int data)
1504 {
1505         if(write)
1506                 return i82543miimiw(mii, pa, ra, data);
1507
1508         return i82543miimir(mii, pa, ra);
1509 }
1510
1511 static int
1512 igbemiirw(struct mii* mii, int write, int pa, int ra, int data)
1513 {
1514         if(write)
1515                 return igbemiimiw(mii, pa, ra, data);
1516
1517         return igbemiimir(mii, pa, ra);
1518 }
1519
1520 static int
1521 igbemii(struct ctlr* ctlr)
1522 {
1523         int ctrl, p, r;
1524         int (*rw)(struct mii*, int unused_int, int, int, int);
1525
1526         r = csr32r(ctlr, Status);
1527         if(r & Tbimode)
1528                 return -1;
1529
1530         ctrl = csr32r(ctlr, Ctrl);
1531         ctrl |= Slu;
1532
1533         switch(ctlr->id){
1534         case i82543gc:
1535                 ctrl |= Frcdplx|Frcspd;
1536                 csr32w(ctlr, Ctrl, ctrl);
1537
1538                 /*
1539                  * The reset pin direction (Mdro) should already
1540                  * be set from the EEPROM load.
1541                  * If it's not set this configuration is unexpected
1542                  * so bail.
1543                  */
1544                 r = csr32r(ctlr, Ctrlext);
1545                 if(!(r & Mdro))
1546                         return -1;
1547                 csr32w(ctlr, Ctrlext, r);
1548                 udelay(20*1000);
1549                 r = csr32r(ctlr, Ctrlext);
1550                 r &= ~Mdr;
1551                 csr32w(ctlr, Ctrlext, r);
1552                 udelay(20*1000);
1553                 r = csr32r(ctlr, Ctrlext);
1554                 r |= Mdr;
1555                 csr32w(ctlr, Ctrlext, r);
1556                 udelay(20*1000);
1557
1558                 rw = i82543miirw;
1559                 break;
1560         case i82544ei:
1561         case i82544eif:
1562         case i82544gc:
1563         case i82540em:
1564         case i82540eplp:
1565         case i82547ei:
1566         case i82547gi:
1567         case i82541ei:
1568         case i82541gi:
1569         case i82541gi2:
1570         case i82541pi:
1571         case i82545em:
1572         case i82545gmc:
1573         case i82546gb:
1574         case i82546eb:
1575                 ctrl &= ~(Frcdplx|Frcspd);
1576                 csr32w(ctlr, Ctrl, ctrl);
1577                 rw = igbemiirw;
1578                 break;
1579         default:
1580                 return -1;
1581         }
1582
1583         if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1584                 return -1;
1585         // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1586
1587         /*
1588          * 8254X-specific PHY registers not in 802.3:
1589          *      0x10    PHY specific control
1590          *      0x14    extended PHY specific control
1591          * Set appropriate values then reset the PHY to have
1592          * changes noted.
1593          */
1594         switch(ctlr->id){
1595         case i82547gi:
1596         case i82541gi:
1597         case i82541gi2:
1598         case i82541pi:
1599         case i82545em:
1600         case i82545gmc:
1601         case i82546gb:
1602         case i82546eb:
1603                 break;
1604         default:
1605                 r = miimir(ctlr->mii, 16);
1606                 r |= 0x0800;                    /* assert CRS on Tx */
1607                 r |= 0x0060;                    /* auto-crossover all speeds */
1608                 r |= 0x0002;                    /* polarity reversal enabled */
1609                 miimiw(ctlr->mii, 16, r);
1610
1611                 r = miimir(ctlr->mii, 20);
1612                 r |= 0x0070;                    /* +25MHz clock */
1613                 r &= ~0x0F00;
1614                 r |= 0x0100;                    /* 1x downshift */
1615                 miimiw(ctlr->mii, 20, r);
1616
1617                 miireset(ctlr->mii);
1618                 p = 0;
1619                 if(ctlr->txcw & TxcwPs)
1620                         p |= AnaP;
1621                 if(ctlr->txcw & TxcwAs)
1622                         p |= AnaAP;
1623                 miiane(ctlr->mii, ~0, p, ~0);
1624                 break;
1625         }
1626         return 0;
1627 }
1628
1629 static int
1630 at93c46io(struct ctlr* ctlr, char* op, int data)
1631 {
1632         char *lp, *p;
1633         int i, loop, eecd, r;
1634
1635         eecd = csr32r(ctlr, Eecd);
1636
1637         r = 0;
1638         loop = -1;
1639         lp = NULL;
1640         for(p = op; *p != '\0'; p++){
1641                 switch(*p){
1642                 default:
1643                         return -1;
1644                 case ' ':
1645                         continue;
1646                 case ':':                       /* start of loop */
1647                         loop = strtol(p+1, &lp, 0)-1;
1648                         lp--;
1649                         if(p == lp)
1650                                 loop = 7;
1651                         p = lp;
1652                         continue;
1653                 case ';':                       /* end of loop */
1654                         if(lp == NULL)
1655                                 return -1;
1656                         loop--;
1657                         if(loop >= 0)
1658                                 p = lp;
1659                         else
1660                                 lp = NULL;
1661                         continue;
1662                 case 'C':                       /* assert clock */
1663                         eecd |= Sk;
1664                         break;
1665                 case 'c':                       /* deassert clock */
1666                         eecd &= ~Sk;
1667                         break;
1668                 case 'D':                       /* next bit in 'data' byte */
1669                         if(loop < 0)
1670                                 return -1;
1671                         if(data & (1<<loop))
1672                                 eecd |= Di;
1673                         else
1674                                 eecd &= ~Di;
1675                         break;
1676                 case 'O':                       /* collect data output */
1677                         i = (csr32r(ctlr, Eecd) & Do) != 0;
1678                         if(loop >= 0)
1679                                 r |= (i<<loop);
1680                         else
1681                                 r = i;
1682                         continue;
1683                 case 'I':                       /* assert data input */
1684                         eecd |= Di;
1685                         break;
1686                 case 'i':                       /* deassert data input */
1687                         eecd &= ~Di;
1688                         break;
1689                 case 'S':                       /* enable chip select */
1690                         eecd |= Cs;
1691                         break;
1692                 case 's':                       /* disable chip select */
1693                         eecd &= ~Cs;
1694                         break;
1695                 }
1696                 csr32w(ctlr, Eecd, eecd);
1697                 udelay(50);
1698         }
1699         if(loop >= 0)
1700                 return -1;
1701         return r;
1702 }
1703
1704 static int
1705 at93c46r(struct ctlr* ctlr)
1706 {
1707         uint16_t sum;
1708         char rop[20];
1709         int addr, areq, bits, data, eecd, i;
1710
1711         eecd = csr32r(ctlr, Eecd);
1712         if(eecd & Spi){
1713                 printd("igbe: SPI EEPROM access not implemented\n");
1714                 return 0;
1715         }
1716         if(eecd & (Eeszaddr|Eesz256))
1717                 bits = 8;
1718         else
1719                 bits = 6;
1720
1721         sum = 0;
1722
1723         switch(ctlr->id){
1724         default:
1725                 areq = 0;
1726                 break;
1727         case i82540em:
1728         case i82540eplp:
1729         case i82541ei:
1730         case i82541gi:
1731         case i82541gi2:
1732         case i82541pi:
1733         case i82545em:
1734         case i82545gmc:
1735         case i82546gb:
1736         case i82546eb:
1737         case i82547ei:
1738         case i82547gi:
1739                 areq = 1;
1740                 csr32w(ctlr, Eecd, eecd|Areq);
1741                 for(i = 0; i < 1000; i++){
1742                         if((eecd = csr32r(ctlr, Eecd)) & Agnt)
1743                                 break;
1744                         udelay(5);
1745                 }
1746                 if(!(eecd & Agnt)){
1747                         printd("igbe: not granted EEPROM access\n");
1748                         goto release;
1749                 }
1750                 break;
1751         }
1752         snprintf(rop, sizeof(rop), "S :%dDCc;", bits+3);
1753
1754         for(addr = 0; addr < 0x40; addr++){
1755                 /*
1756                  * Read a word at address 'addr' from the Atmel AT93C46
1757                  * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1758                  * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1759                  * for protocol details.
1760                  */
1761                 if(at93c46io(ctlr, rop, (0x06<<bits)|addr) != 0){
1762                         printd("igbe: can't set EEPROM address 0x%2.2X\n", addr);
1763                         goto release;
1764                 }
1765                 data = at93c46io(ctlr, ":16COc;", 0);
1766                 at93c46io(ctlr, "sic", 0);
1767                 ctlr->eeprom[addr] = data;
1768                 sum += data;
1769         }
1770
1771 release:
1772         if(areq)
1773                 csr32w(ctlr, Eecd, eecd & ~Areq);
1774         return sum;
1775 }
1776
1777 static int
1778 igbedetach(struct ctlr* ctlr)
1779 {
1780         int r, timeo;
1781
1782         /*
1783          * Perform a device reset to get the chip back to the
1784          * power-on state, followed by an EEPROM reset to read
1785          * the defaults for some internal registers.
1786          */
1787         csr32w(ctlr, Imc, ~0);
1788         csr32w(ctlr, Rctl, 0);
1789         csr32w(ctlr, Tctl, 0);
1790
1791         udelay(10*1000);
1792
1793         csr32w(ctlr, Ctrl, Devrst);
1794         udelay(1*1000);
1795         for(timeo = 0; timeo < 1000; timeo++){
1796                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1797                         break;
1798                 udelay(1*1000);
1799         }
1800         if(csr32r(ctlr, Ctrl) & Devrst)
1801                 return -1;
1802         r = csr32r(ctlr, Ctrlext);
1803         csr32w(ctlr, Ctrlext, r|Eerst);
1804         udelay(1*1000);
1805
1806         for(timeo = 0; timeo < 1000; timeo++){
1807                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1808                         break;
1809                 udelay(1*1000);
1810         }
1811         if(csr32r(ctlr, Ctrlext) & Eerst)
1812                 return -1;
1813
1814         switch(ctlr->id){
1815         default:
1816                 break;
1817         case i82540em:
1818         case i82540eplp:
1819         case i82541gi:
1820         case i82541gi2:
1821         case i82541pi:
1822         case i82545em:
1823         case i82545gmc:
1824         case i82547gi:
1825         case i82546gb:
1826         case i82546eb:
1827                 r = csr32r(ctlr, Manc);
1828                 r &= ~Arpen;
1829                 csr32w(ctlr, Manc, r);
1830                 break;
1831         }
1832
1833         csr32w(ctlr, Imc, ~0);
1834         udelay(1*1000);
1835         for(timeo = 0; timeo < 1000; timeo++){
1836                 if(!csr32r(ctlr, Icr))
1837                         break;
1838                 udelay(1*1000);
1839         }
1840         if(csr32r(ctlr, Icr))
1841                 return -1;
1842
1843         return 0;
1844 }
1845
1846 static void
1847 igbeshutdown(struct ether* ether)
1848 {
1849         igbedetach(ether->ctlr);
1850 }
1851
1852 static int
1853 igbereset(struct ctlr* ctlr)
1854 {
1855         int ctrl, i, pause, r, swdpio, txcw;
1856
1857         if(igbedetach(ctlr))
1858                 return -1;
1859
1860         /*
1861          * Read the EEPROM, validate the checksum
1862          * then get the device back to a power-on state.
1863          */
1864         if((r = at93c46r(ctlr)) != 0xBABA){
1865                 printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1866                 return -1;
1867         }
1868
1869         /*
1870          * Snarf and set up the receive addresses.
1871          * There are 16 addresses. The first should be the MAC address.
1872          * The others are cleared and not marked valid (MS bit of Rah).
1873          */
1874         if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1875             (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1876                  pci_config_addr(0, 1, 0, 0)))
1877                 ctlr->eeprom[Ea+2] += 0x100;            /* second interface */
1878         if(ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1879                 ctlr->eeprom[Ea] = 0xD000;
1880         for(i = Ea; i < Eaddrlen/2; i++){
1881                 ctlr->ra[2*i] = ctlr->eeprom[i];
1882                 ctlr->ra[2*i+1] = ctlr->eeprom[i]>>8;
1883         }
1884         /* lan id seems to vary on 82543gc; don't use it */
1885         if (ctlr->id != i82543gc) {
1886                 r = (csr32r(ctlr, Status) & Lanid) >> 2;
1887                 ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1888         }
1889
1890         r = (ctlr->ra[3]<<24)|(ctlr->ra[2]<<16)|(ctlr->ra[1]<<8)|ctlr->ra[0];
1891         csr32w(ctlr, Ral, r);
1892         r = 0x80000000|(ctlr->ra[5]<<8)|ctlr->ra[4];
1893         csr32w(ctlr, Rah, r);
1894         for(i = 1; i < 16; i++){
1895                 csr32w(ctlr, Ral+i*8, 0);
1896                 csr32w(ctlr, Rah+i*8, 0);
1897         }
1898
1899         /*
1900          * Clear the Multicast Table Array.
1901          * It's a 4096 bit vector accessed as 128 32-bit registers.
1902          */
1903         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1904         for(i = 0; i < 128; i++)
1905                 csr32w(ctlr, Mta+i*4, 0);
1906
1907         /*
1908          * Just in case the Eerst didn't load the defaults
1909          * (doesn't appear to fully on the 82543GC), do it manually.
1910          */
1911         if (ctlr->id == i82543gc) {
1912                 txcw = csr32r(ctlr, Txcw);
1913                 txcw &= ~(TxcwAne|TxcwPauseMASK|TxcwFd);
1914                 ctrl = csr32r(ctlr, Ctrl);
1915                 ctrl &= ~(SwdpioloMASK|Frcspd|Ilos|Lrst|Fd);
1916
1917                 if(ctlr->eeprom[Icw1] & 0x0400){
1918                         ctrl |= Fd;
1919                         txcw |= TxcwFd;
1920                 }
1921                 if(ctlr->eeprom[Icw1] & 0x0200)
1922                         ctrl |= Lrst;
1923                 if(ctlr->eeprom[Icw1] & 0x0010)
1924                         ctrl |= Ilos;
1925                 if(ctlr->eeprom[Icw1] & 0x0800)
1926                         ctrl |= Frcspd;
1927                 swdpio = (ctlr->eeprom[Icw1] & 0x01E0)>>5;
1928                 ctrl |= swdpio<<SwdpioloSHIFT;
1929                 csr32w(ctlr, Ctrl, ctrl);
1930
1931                 ctrl = csr32r(ctlr, Ctrlext);
1932                 ctrl &= ~(Ips|SwdpiohiMASK);
1933                 swdpio = (ctlr->eeprom[Icw2] & 0x00F0)>>4;
1934                 if(ctlr->eeprom[Icw1] & 0x1000)
1935                         ctrl |= Ips;
1936                 ctrl |= swdpio<<SwdpiohiSHIFT;
1937                 csr32w(ctlr, Ctrlext, ctrl);
1938
1939                 if(ctlr->eeprom[Icw2] & 0x0800)
1940                         txcw |= TxcwAne;
1941                 pause = (ctlr->eeprom[Icw2] & 0x3000)>>12;
1942                 txcw |= pause<<TxcwPauseSHIFT;
1943                 switch(pause){
1944                 default:
1945                         ctlr->fcrtl = 0x00002000;
1946                         ctlr->fcrth = 0x00004000;
1947                         txcw |= TxcwAs|TxcwPs;
1948                         break;
1949                 case 0:
1950                         ctlr->fcrtl = 0x00002000;
1951                         ctlr->fcrth = 0x00004000;
1952                         break;
1953                 case 2:
1954                         ctlr->fcrtl = 0;
1955                         ctlr->fcrth = 0;
1956                         txcw |= TxcwAs;
1957                         break;
1958                 }
1959                 ctlr->txcw = txcw;
1960                 csr32w(ctlr, Txcw, txcw);
1961         }
1962
1963
1964         /*
1965          * Flow control - values from the datasheet.
1966          */
1967         csr32w(ctlr, Fcal, 0x00C28001);
1968         csr32w(ctlr, Fcah, 0x00000100);
1969         csr32w(ctlr, Fct, 0x00008808);
1970         csr32w(ctlr, Fcttv, 0x00000100);
1971
1972         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1973         csr32w(ctlr, Fcrth, ctlr->fcrth);
1974
1975         /* FYI, igbemii checks status right away too. */
1976         if(!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1977                 printk("igbemii failed!  igbe failing to reset!\n");
1978                 return -1;
1979         }
1980
1981         return 0;
1982 }
1983
1984 static void
1985 igbepci(void)
1986 {
1987         int cls, id;
1988         struct pci_device *pcidev;
1989         struct ctlr *ctlr;
1990         void *mem;
1991         uintptr_t mmio_paddr;
1992
1993         STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
1994                 /* This checks that pcidev is a Network Controller for Ethernet */
1995                 if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1996                         continue;
1997                 id = pcidev->dev_id << 16 | pcidev->ven_id;
1998                 switch (id) {
1999                 default:
2000                         continue;
2001                 case i82543gc:
2002                 case i82544ei:
2003                 case i82544eif:
2004                 case i82544gc:
2005                 case i82547ei:
2006                 case i82547gi:
2007                 case i82540em:
2008                 case i82540eplp:
2009                 case i82541ei:
2010                 case i82541gi:
2011                 case i82541gi2:
2012                 case i82541pi:
2013                 case i82545em:
2014                 case i82545gmc:
2015                 case i82546gb:
2016                 case i82546eb:
2017                         break;
2018                 }
2019                 printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
2020                        pcidev->ven_id, pcidev->dev_id,
2021                        pcidev->bus, pcidev->dev, pcidev->func);
2022
2023                 mmio_paddr = pcidev->bar[0].mmio_base32 ? pcidev->bar[0].mmio_base32 : 
2024                                                           pcidev->bar[0].mmio_base64;
2025                 mem = (void*)vmap_pmem_nocache(mmio_paddr, pcidev->bar[0].mmio_sz);
2026                 if(mem == NULL){
2027                         printd("igbe: can't map %p\n", pcidev->bar[0].mmio_base32);
2028                         continue;
2029                 }
2030                 cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
2031                 switch(cls){
2032                         default:
2033                                 printd("igbe: unexpected CLS - %d\n", cls*4);
2034                                 break;
2035                         case 0x00:
2036                         case 0xFF:
2037                                 /* bogus value; use a sane default.  cls is set in DWORD (u32)
2038                                  * units. */
2039                                 cls = ARCH_CL_SIZE / sizeof(long);
2040                                 pcidev_write8(pcidev, PCI_CLSZ_REG, cls);
2041                                 break;
2042                         case 0x08:
2043                         case 0x10:
2044                                 break;
2045                 }
2046                 ctlr = kzmalloc(sizeof(struct ctlr), 0);
2047                 if(ctlr == NULL) {
2048                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2049                         error(Enomem);
2050                 }
2051                 spinlock_init_irqsave(&ctlr->imlock);
2052                 spinlock_init_irqsave(&ctlr->tlock);
2053                 qlock_init(&ctlr->alock);
2054                 qlock_init(&ctlr->slock);
2055                 rendez_init(&ctlr->lrendez);
2056                 rendez_init(&ctlr->rrendez);
2057                 /* port seems to be unused, and only used for some comparison with edev.
2058                  * plan9 just used the top of the raw bar, regardless of the type. */
2059                 ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
2060                 ctlr->pci = pcidev;
2061                 ctlr->id = id;
2062                 ctlr->cls = cls * sizeof(long);
2063                 ctlr->nic = mem;
2064
2065                 if(igbereset(ctlr)){
2066                         kfree(ctlr);
2067                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2068                         continue;
2069                 }
2070                 pci_set_bus_master(pcidev);
2071
2072                 if(igbectlrhead != NULL)
2073                         igbectlrtail->next = ctlr;
2074                 else
2075                         igbectlrhead = ctlr;
2076                 igbectlrtail = ctlr;
2077         }
2078 }
2079
2080 static int
2081 igbepnp(struct ether* edev)
2082 {
2083         struct ctlr *ctlr;
2084
2085         run_once(igbepci());
2086
2087         /*
2088          * Any adapter matches if no edev->port is supplied,
2089          * otherwise the ports must match.
2090          */
2091         for(ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next){
2092                 if(ctlr->active)
2093                         continue;
2094                 if(edev->port == 0 || edev->port == ctlr->port){
2095                         ctlr->active = 1;
2096                         break;
2097                 }
2098         }
2099         if(ctlr == NULL)
2100                 return -1;
2101
2102         edev->ctlr = ctlr;
2103         edev->port = ctlr->port;
2104         edev->irq = ctlr->pci->irqline;
2105         edev->tbdf = MKBUS(BusPCI, ctlr->pci->bus, ctlr->pci->dev, ctlr->pci->func);
2106         edev->netif.mbps = 1000;
2107         memmove(edev->ea, ctlr->ra, Eaddrlen);
2108
2109         /*
2110          * Linkage to the generic ethernet driver.
2111          */
2112         edev->attach = igbeattach;
2113         edev->transmit = igbetransmit;
2114         edev->ifstat = igbeifstat;
2115         edev->ctl = igbectl;
2116         edev->shutdown = igbeshutdown;
2117
2118         edev->netif.arg = edev;
2119         edev->netif.promiscuous = igbepromiscuous;
2120         edev->netif.multicast = igbemulticast;
2121
2122         register_irq(edev->irq, igbeinterrupt, edev, edev->tbdf);
2123         return 0;
2124 }
2125
2126 linker_func_3(etherigbelink)
2127 {
2128         addethercard("i82543", igbepnp);
2129         addethercard("igbe", igbepnp);
2130 }