igbe: use safer MMIO register access
[akaros.git] / kern / drivers / net / etherigbe.c
1 /* This file is part of the UCB release of Plan 9. It is subject to the license
2  * terms in the LICENSE file found in the top-level directory of this
3  * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
4  * part of the UCB release of Plan 9, including this file, may be copied,
5  * modified, propagated, or distributed except according to the terms contained
6  * in the LICENSE file. */
7
8 /*
9  * Intel 8254[340]NN Gigabit Ethernet PCI Controllers
10  * as found on the Intel PRO/1000 series of adapters:
11  *      82543GC Intel PRO/1000 T
12  *      82544EI Intel PRO/1000 XT
13  *      82540EM Intel PRO/1000 MT
14  *      82541[GP]I
15  *      82547GI
16  *      82546GB
17  *      82546EB
18  * To Do:
19  *      finish autonegotiation code;
20  *      integrate fiber stuff back in (this ONLY handles
21  *      the CAT5 cards at the moment);
22  *      add checksum-offload;
23  *      add tuning control via ctl file;
24  *      this driver is little-endian specific.
25  *
26  * Modified by brho:
27  *      ported to Akaros
28  *      fixed mii bugs (allocation, startup, miirw, etc)
29  *      fixed CLS bug (continue -> break)
30  *      made sure igbepci only runs once, even if it fails */
31
32 #include <vfs.h>
33 #include <kfs.h>
34 #include <slab.h>
35 #include <kmalloc.h>
36 #include <kref.h>
37 #include <string.h>
38 #include <stdio.h>
39 #include <assert.h>
40 #include <error.h>
41 #include <cpio.h>
42 #include <pmap.h>
43 #include <smp.h>
44 #include <arch/pci.h>
45 #include <ip.h>
46 #include <ns.h>
47 #include "ethermii.h"
48
49 #define ilock(x) spin_lock_irqsave(x)
50 #define iunlock(x) spin_unlock_irqsave(x)
51
52 enum {
53         i82542          = (0x1000<<16)|0x8086,
54         i82543gc        = (0x1004<<16)|0x8086,
55         i82544ei        = (0x1008<<16)|0x8086,
56         i82544eif       = (0x1009<<16)|0x8086,
57         i82544gc        = (0x100d<<16)|0x8086,
58         i82540em        = (0x100E<<16)|0x8086,
59         i82540eplp      = (0x101E<<16)|0x8086,
60         i82545em        = (0x100F<<16)|0x8086,
61         i82545gmc       = (0x1026<<16)|0x8086,
62         i82547ei        = (0x1019<<16)|0x8086,
63         i82547gi        = (0x1075<<16)|0x8086,
64         i82541ei        = (0x1013<<16)|0x8086,
65         i82541gi        = (0x1076<<16)|0x8086,
66         i82541gi2       = (0x1077<<16)|0x8086,
67         i82541pi        = (0x107c<<16)|0x8086,
68         i82546gb        = (0x1079<<16)|0x8086,
69         i82546eb        = (0x1010<<16)|0x8086,
70 };
71
72 enum {
73         Ctrl            = 0x00000000,   /* Device Control */
74         Ctrldup         = 0x00000004,   /* Device Control Duplicate */
75         Status          = 0x00000008,   /* Device Status */
76         Eecd            = 0x00000010,   /* EEPROM/Flash Control/Data */
77         Ctrlext         = 0x00000018,   /* Extended Device Control */
78         Mdic            = 0x00000020,   /* MDI Control */
79         Fcal            = 0x00000028,   /* Flow Control Address Low */
80         Fcah            = 0x0000002C,   /* Flow Control Address High */
81         Fct             = 0x00000030,   /* Flow Control Type */
82         Icr             = 0x000000C0,   /* Interrupt Cause Read */
83         Ics             = 0x000000C8,   /* Interrupt Cause Set */
84         Ims             = 0x000000D0,   /* Interrupt Mask Set/Read */
85         Imc             = 0x000000D8,   /* Interrupt mask Clear */
86         Rctl            = 0x00000100,   /* Receive Control */
87         Fcttv           = 0x00000170,   /* Flow Control Transmit Timer Value */
88         Txcw            = 0x00000178,   /* Transmit Configuration Word */
89         Rxcw            = 0x00000180,   /* Receive Configuration Word */
90         /* on the oldest cards (8254[23]), the Mta register is at 0x200 */
91         Tctl            = 0x00000400,   /* Transmit Control */
92         Tipg            = 0x00000410,   /* Transmit IPG */
93         Tbt             = 0x00000448,   /* Transmit Burst Timer */
94         Ait             = 0x00000458,   /* Adaptive IFS Throttle */
95         Fcrtl           = 0x00002160,   /* Flow Control RX Threshold Low */
96         Fcrth           = 0x00002168,   /* Flow Control Rx Threshold High */
97         Rdfh            = 0x00002410,   /* Receive data fifo head */
98         Rdft            = 0x00002418,   /* Receive data fifo tail */
99         Rdfhs           = 0x00002420,   /* Receive data fifo head saved */
100         Rdfts           = 0x00002428,   /* Receive data fifo tail saved */
101         Rdfpc           = 0x00002430,   /* Receive data fifo packet count */
102         Rdbal           = 0x00002800,   /* Rd Base Address Low */
103         Rdbah           = 0x00002804,   /* Rd Base Address High */
104         Rdlen           = 0x00002808,   /* Receive Descriptor Length */
105         Rdh             = 0x00002810,   /* Receive Descriptor Head */
106         Rdt             = 0x00002818,   /* Receive Descriptor Tail */
107         Rdtr            = 0x00002820,   /* Receive Descriptor Timer Ring */
108         Rxdctl          = 0x00002828,   /* Receive Descriptor Control */
109         Radv            = 0x0000282C,   /* Receive Interrupt Absolute Delay Timer */
110         Txdmac          = 0x00003000,   /* Transfer DMA Control */
111         Ett             = 0x00003008,   /* Early Transmit Control */
112         Tdfh            = 0x00003410,   /* Transmit data fifo head */
113         Tdft            = 0x00003418,   /* Transmit data fifo tail */
114         Tdfhs           = 0x00003420,   /* Transmit data Fifo Head saved */
115         Tdfts           = 0x00003428,   /* Transmit data fifo tail saved */
116         Tdfpc           = 0x00003430,   /* Trasnmit data Fifo packet count */
117         Tdbal           = 0x00003800,   /* Td Base Address Low */
118         Tdbah           = 0x00003804,   /* Td Base Address High */
119         Tdlen           = 0x00003808,   /* Transmit Descriptor Length */
120         Tdh             = 0x00003810,   /* Transmit Descriptor Head */
121         Tdt             = 0x00003818,   /* Transmit Descriptor Tail */
122         Tidv            = 0x00003820,   /* Transmit Interrupt Delay Value */
123         Txdctl          = 0x00003828,   /* Transmit Descriptor Control */
124         Tadv            = 0x0000382C,   /* Transmit Interrupt Absolute Delay Timer */
125
126         Statistics      = 0x00004000,   /* Start of Statistics Area */
127         Gorcl           = 0x88/4,       /* Good Octets Received Count */
128         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
129         Torl            = 0xC0/4,       /* Total Octets Received */
130         Totl            = 0xC8/4,       /* Total Octets Transmitted */
131         Nstatistics     = 64,
132
133         Rxcsum          = 0x00005000,   /* Receive Checksum Control */
134         Mta             = 0x00005200,   /* Multicast Table Array */
135         Ral             = 0x00005400,   /* Receive Address Low */
136         Rah             = 0x00005404,   /* Receive Address High */
137         Manc            = 0x00005820,   /* Management Control */
138 };
139
140 enum {                                  /* Ctrl */
141         Bem             = 0x00000002,   /* Big Endian Mode */
142         Prior           = 0x00000004,   /* Priority on the PCI bus */
143         Lrst            = 0x00000008,   /* Link Reset */
144         Asde            = 0x00000020,   /* Auto-Speed Detection Enable */
145         Slu             = 0x00000040,   /* Set Link Up */
146         Ilos            = 0x00000080,   /* Invert Loss of Signal (LOS) */
147         SspeedMASK      = 0x00000300,   /* Speed Selection */
148         SspeedSHIFT     = 8,
149         Sspeed10        = 0x00000000,   /* 10Mb/s */
150         Sspeed100       = 0x00000100,   /* 100Mb/s */
151         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
152         Frcspd          = 0x00000800,   /* Force Speed */
153         Frcdplx         = 0x00001000,   /* Force Duplex */
154         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
155         SwdpinsloSHIFT  = 18,
156         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
157         SwdpioloSHIFT   = 22,
158         Devrst          = 0x04000000,   /* Device Reset */
159         Rfce            = 0x08000000,   /* Receive Flow Control Enable */
160         Tfce            = 0x10000000,   /* Transmit Flow Control Enable */
161         Vme             = 0x40000000,   /* VLAN Mode Enable */
162 };
163
164 /*
165  * can't find Tckok nor Rbcok in any Intel docs,
166  * but even 82543gc docs define Lanid.
167  */
168 enum {                                  /* Status */
169         Lu              = 0x00000002,   /* Link Up */
170         Lanid           = 0x0000000C,   /* mask for Lan ID. (function id) */
171 //      Tckok           = 0x00000004,   /* Transmit clock is running */
172 //      Rbcok           = 0x00000008,   /* Receive clock is running */
173         Txoff           = 0x00000010,   /* Transmission Paused */
174         Tbimode         = 0x00000020,   /* TBI Mode Indication */
175         LspeedMASK      = 0x000000C0,   /* Link Speed Setting */
176         LspeedSHIFT     = 6,
177         Lspeed10        = 0x00000000,   /* 10Mb/s */
178         Lspeed100       = 0x00000040,   /* 100Mb/s */
179         Lspeed1000      = 0x00000080,   /* 1000Mb/s */
180         Mtxckok         = 0x00000400,   /* MTX clock is running */
181         Pci66           = 0x00000800,   /* PCI Bus speed indication */
182         Bus64           = 0x00001000,   /* PCI Bus width indication */
183         Pcixmode        = 0x00002000,   /* PCI-X mode */
184         PcixspeedMASK   = 0x0000C000,   /* PCI-X bus speed */
185         PcixspeedSHIFT  = 14,
186         Pcix66          = 0x00000000,   /* 50-66MHz */
187         Pcix100         = 0x00004000,   /* 66-100MHz */
188         Pcix133         = 0x00008000,   /* 100-133MHz */
189 };
190
191 enum {                                  /* Ctrl and Status */
192         Fd              = 0x00000001,   /* Full-Duplex */
193         AsdvMASK        = 0x00000300,
194         AsdvSHIFT       = 8,
195         Asdv10          = 0x00000000,   /* 10Mb/s */
196         Asdv100         = 0x00000100,   /* 100Mb/s */
197         Asdv1000        = 0x00000200,   /* 1000Mb/s */
198 };
199
200 enum {                                  /* Eecd */
201         Sk              = 0x00000001,   /* Clock input to the EEPROM */
202         Cs              = 0x00000002,   /* Chip Select */
203         Di              = 0x00000004,   /* Data Input to the EEPROM */
204         Do              = 0x00000008,   /* Data Output from the EEPROM */
205         Areq            = 0x00000040,   /* EEPROM Access Request */
206         Agnt            = 0x00000080,   /* EEPROM Access Grant */
207         Eepresent       = 0x00000100,   /* EEPROM Present */
208         Eesz256         = 0x00000200,   /* EEPROM is 256 words not 64 */
209         Eeszaddr        = 0x00000400,   /* EEPROM size for 8254[17] */
210         Spi             = 0x00002000,   /* EEPROM is SPI not Microwire */
211 };
212
213 enum {                                  /* Ctrlext */
214         Gpien           = 0x0000000F,   /* General Purpose Interrupt Enables */
215         SwdpinshiMASK   = 0x000000F0,   /* Software Defined Pins - hi nibble */
216         SwdpinshiSHIFT  = 4,
217         SwdpiohiMASK    = 0x00000F00,   /* Software Defined Pins - I or O */
218         SwdpiohiSHIFT   = 8,
219         Asdchk          = 0x00001000,   /* ASD Check */
220         Eerst           = 0x00002000,   /* EEPROM Reset */
221         Ips             = 0x00004000,   /* Invert Power State */
222         Spdbyps         = 0x00008000,   /* Speed Select Bypass */
223 };
224
225 enum {                                  /* EEPROM content offsets */
226         Ea              = 0x00,         /* Ethernet Address */
227         Cf              = 0x03,         /* Compatibility Field */
228         Pba             = 0x08,         /* Printed Board Assembly number */
229         Icw1            = 0x0A,         /* Initialization Control Word 1 */
230         Sid             = 0x0B,         /* Subsystem ID */
231         Svid            = 0x0C,         /* Subsystem Vendor ID */
232         Did             = 0x0D,         /* Device ID */
233         Vid             = 0x0E,         /* Vendor ID */
234         Icw2            = 0x0F,         /* Initialization Control Word 2 */
235 };
236
237 enum {                                  /* Mdic */
238         MDIdMASK        = 0x0000FFFF,   /* Data */
239         MDIdSHIFT       = 0,
240         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
241         MDIrSHIFT       = 16,
242         MDIpMASK        = 0x03E00000,   /* PHY Address */
243         MDIpSHIFT       = 21,
244         MDIwop          = 0x04000000,   /* Write Operation */
245         MDIrop          = 0x08000000,   /* Read Operation */
246         MDIready        = 0x10000000,   /* End of Transaction */
247         MDIie           = 0x20000000,   /* Interrupt Enable */
248         MDIe            = 0x40000000,   /* Error */
249 };
250
251 enum {                                  /* Icr, Ics, Ims, Imc */
252         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
253         Txqe            = 0x00000002,   /* Transmit Queue Empty */
254         Lsc             = 0x00000004,   /* Link Status Change */
255         Rxseq           = 0x00000008,   /* Receive Sequence Error */
256         Rxdmt0          = 0x00000010,   /* Rd Minimum Threshold Reached */
257         Rxo             = 0x00000040,   /* Receiver Overrun */
258         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
259         Mdac            = 0x00000200,   /* MDIO Access Completed */
260         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
261         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
262         Gpi1            = 0x00001000,
263         Gpi2            = 0x00002000,
264         Gpi3            = 0x00004000,
265 };
266
267 /*
268  * The Mdic register isn't implemented on the 82543GC,
269  * the software defined pins are used instead.
270  * These definitions work for the Intel PRO/1000 T Server Adapter.
271  * The direction pin bits are read from the EEPROM.
272  */
273 enum {
274         Mdd             = ((1<<2)<<SwdpinsloSHIFT),     /* data */
275         Mddo            = ((1<<2)<<SwdpioloSHIFT),      /* pin direction */
276         Mdc             = ((1<<3)<<SwdpinsloSHIFT),     /* clock */
277         Mdco            = ((1<<3)<<SwdpioloSHIFT),      /* pin direction */
278         Mdr             = ((1<<0)<<SwdpinshiSHIFT),     /* reset */
279         Mdro            = ((1<<0)<<SwdpiohiSHIFT),      /* pin direction */
280 };
281
282 enum {                                  /* Txcw */
283         TxcwFd          = 0x00000020,   /* Full Duplex */
284         TxcwHd          = 0x00000040,   /* Half Duplex */
285         TxcwPauseMASK   = 0x00000180,   /* Pause */
286         TxcwPauseSHIFT  = 7,
287         TxcwPs          = (1<<TxcwPauseSHIFT),  /* Pause Supported */
288         TxcwAs          = (2<<TxcwPauseSHIFT),  /* Asymmetric FC desired */
289         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
290         TxcwRfiSHIFT    = 12,
291         TxcwNpr         = 0x00008000,   /* Next Page Request */
292         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
293         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
294 };
295
296 enum {                                  /* Rxcw */
297         Rxword          = 0x0000FFFF,   /* Data from auto-negotiation process */
298         Rxnocarrier     = 0x04000000,   /* Carrier Sense indication */
299         Rxinvalid       = 0x08000000,   /* Invalid Symbol during configuration */
300         Rxchange        = 0x10000000,   /* Change to the Rxword indication */
301         Rxconfig        = 0x20000000,   /* /C/ order set reception indication */
302         Rxsync          = 0x40000000,   /* Lost bit synchronization indication */
303         Anc             = 0x80000000,   /* Auto Negotiation Complete */
304 };
305
306 enum {                                  /* Rctl */
307         Rrst            = 0x00000001,   /* Receiver Software Reset */
308         Ren             = 0x00000002,   /* Receiver Enable */
309         Sbp             = 0x00000004,   /* Store Bad Packets */
310         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
311         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
312         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
313         LbmMASK         = 0x000000C0,   /* Loopback Mode */
314         LbmOFF          = 0x00000000,   /* No Loopback */
315         LbmTBI          = 0x00000040,   /* TBI Loopback */
316         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
317         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
318         RdtmsMASK       = 0x00000300,   /* Rd Minimum Threshold Size */
319         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
320         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
321         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
322         MoMASK          = 0x00003000,   /* Multicast Offset */
323         Mo47b36         = 0x00000000,   /* bits [47:36] of received address */
324         Mo46b35         = 0x00001000,   /* bits [46:35] of received address */
325         Mo45b34         = 0x00002000,   /* bits [45:34] of received address */
326         Mo43b32         = 0x00003000,   /* bits [43:32] of received address */
327         Bam             = 0x00008000,   /* Broadcast Accept Mode */
328         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
329         Bsize2048       = 0x00000000,   /* Bsex = 0 */
330         Bsize1024       = 0x00010000,   /* Bsex = 0 */
331         Bsize512        = 0x00020000,   /* Bsex = 0 */
332         Bsize256        = 0x00030000,   /* Bsex = 0 */
333         Bsize16384      = 0x00010000,   /* Bsex = 1 */
334         Vfe             = 0x00040000,   /* VLAN Filter Enable */
335         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
336         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
337         Dpf             = 0x00400000,   /* Discard Pause Frames */
338         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
339         Bsex            = 0x02000000,   /* Buffer Size Extension */
340         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
341 };
342
343 enum {                                  /* Tctl */
344         Trst            = 0x00000001,   /* Transmitter Software Reset */
345         Ten             = 0x00000002,   /* Transmit Enable */
346         Psp             = 0x00000008,   /* Pad Short Packets */
347         CtMASK          = 0x00000FF0,   /* Collision Threshold */
348         CtSHIFT         = 4,
349         ColdMASK        = 0x003FF000,   /* Collision Distance */
350         ColdSHIFT       = 12,
351         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
352         Pbe             = 0x00800000,   /* Packet Burst Enable */
353         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
354         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
355 };
356
357 enum {                                  /* [RT]xdctl */
358         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
359         PthreshSHIFT    = 0,
360         HthreshMASK     = 0x00003F00,   /* Host Threshold */
361         HthreshSHIFT    = 8,
362         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
363         WthreshSHIFT    = 16,
364         Gran            = 0x01000000,   /* Granularity */
365         LthreshMASK     = 0xFE000000,   /* Low Threshold */
366         LthreshSHIFT    = 25,
367 };
368
369 enum {                                  /* Rxcsum */
370         PcssMASK        = 0x000000FF,   /* Packet Checksum Start */
371         PcssSHIFT       = 0,
372         Ipofl           = 0x00000100,   /* IP Checksum Off-load Enable */
373         Tuofl           = 0x00000200,   /* TCP/UDP Checksum Off-load Enable */
374 };
375
376 enum {                                  /* Manc */
377         Arpen           = 0x00002000,   /* Enable ARP Request Filtering */
378 };
379
380 enum {                                  /* Receive Delay Timer Ring */
381         DelayMASK       = 0x0000FFFF,   /* delay timer in 1.024nS increments */
382         DelaySHIFT      = 0,
383         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
384 };
385
386 typedef struct Rd {                     /* Receive Descriptor */
387         unsigned int    addr[2];
388         uint16_t        length;
389         uint16_t        checksum;
390         uint8_t status;
391         uint8_t errors;
392         uint16_t        special;
393 } Rd;
394
395 enum {                                  /* Rd status */
396         Rdd             = 0x01,         /* Descriptor Done */
397         Reop            = 0x02,         /* End of Packet */
398         Ixsm            = 0x04,         /* Ignore Checksum Indication */
399         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
400         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
401         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
402         Pif             = 0x80,         /* Passed in-exact filter */
403 };
404
405 enum {                                  /* Rd errors */
406         Ce              = 0x01,         /* CRC Error or Alignment Error */
407         Se              = 0x02,         /* Symbol Error */
408         Seq             = 0x04,         /* Sequence Error */
409         Cxe             = 0x10,         /* Carrier Extension Error */
410         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
411         Ipe             = 0x40,         /* IP Checksum Error */
412         Rxe             = 0x80,         /* RX Data Error */
413 };
414
415 typedef struct Td Td;
416 struct Td {                             /* Transmit Descriptor */
417         union {
418                 unsigned int    addr[2];        /* Data */
419                 struct {                /* Context */
420                         uint8_t ipcss;
421                         uint8_t ipcso;
422                         uint16_t        ipcse;
423                         uint8_t tucss;
424                         uint8_t tucso;
425                         uint16_t        tucse;
426                 };
427         };
428         unsigned int    control;
429         unsigned int    status;
430 };
431
432 enum {                                  /* Td control */
433         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
434         LenSHIFT        = 0,
435         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
436         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
437         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
438         Teop            = 0x01000000,   /* End of Packet (DD) */
439         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
440         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
441         Tse             = 0x04000000,   /* TCP Segmentation Enable */
442         Rs              = 0x08000000,   /* Report Status */
443         Rps             = 0x10000000,   /* Report Status Sent */
444         Dext            = 0x20000000,   /* Descriptor Extension */
445         Vle             = 0x40000000,   /* VLAN Packet Enable */
446         Ide             = 0x80000000,   /* Interrupt Delay Enable */
447 };
448
449 enum {                                  /* Td status */
450         Tdd             = 0x00000001,   /* Descriptor Done */
451         Ec              = 0x00000002,   /* Excess Collisions */
452         Lc              = 0x00000004,   /* Late Collision */
453         Tu              = 0x00000008,   /* Transmit Underrun */
454         Iixsm           = 0x00000100,   /* Insert IP Checksum */
455         Itxsm           = 0x00000200,   /* Insert TCP/UDP Checksum */
456         HdrlenMASK      = 0x0000FF00,   /* Header Length (Tse) */
457         HdrlenSHIFT     = 8,
458         VlanMASK        = 0x0FFF0000,   /* VLAN Identifier */
459         VlanSHIFT       = 16,
460         Tcfi            = 0x10000000,   /* Canonical Form Indicator */
461         PriMASK         = 0xE0000000,   /* User Priority */
462         PriSHIFT        = 29,
463         MssMASK         = 0xFFFF0000,   /* Maximum Segment Size (Tse) */
464         MssSHIFT        = 16,
465 };
466
467 enum {
468         Nrd             = 256,          /* multiple of 8 */
469         Ntd             = 64,           /* multiple of 8 */
470         Nrb             = 1024,         /* private receive buffers per Ctlr */
471         Rbsz            = 2048,
472 };
473
474 struct ctlr {
475         int     port;
476         struct pci_device *pci;
477         struct ctlr*    next;
478         struct ether*   edev;
479         int     active;
480         int     started;
481         int     id;
482         int     cls;
483         uint16_t        eeprom[0x40];
484
485         qlock_t alock;                  /* attach */
486         void*   alloc;                  /* receive/transmit descriptors */
487         int     nrd;
488         int     ntd;
489         int     nrb;                    /* how many this Ctlr has in the pool */
490
491         int*    nic;
492         spinlock_t      imlock;
493         int     im;                     /* interrupt mask */
494
495         struct mii*     mii;
496         struct rendez   lrendez;
497         int     lim;
498
499         int     link;
500
501         qlock_t slock;
502         unsigned int    statistics[Nstatistics];
503         unsigned int    lsleep;
504         unsigned int    lintr;
505         unsigned int    rsleep;
506         unsigned int    rintr;
507         unsigned int    txdw;
508         unsigned int    tintr;
509         unsigned int    ixsm;
510         unsigned int    ipcs;
511         unsigned int    tcpcs;
512
513         uint8_t ra[Eaddrlen];           /* receive address */
514         uint32_t        mta[128];               /* multicast table array */
515
516         struct rendez   rrendez;
517         int     rim;
518         int     rdfree;
519         Rd*     rdba;                   /* receive descriptor base address */
520         struct block**  rb;                     /* receive buffers */
521         int     rdh;                    /* receive descriptor head */
522         int     rdt;                    /* receive descriptor tail */
523         int     rdtr;                   /* receive delay timer ring value */
524
525         spinlock_t      tlock;
526         int     tbusy;
527         int     tdfree;
528         Td*     tdba;                   /* transmit descriptor base address */
529         struct block**  tb;                     /* transmit buffers */
530         int     tdh;                    /* transmit descriptor head */
531         int     tdt;                    /* transmit descriptor tail */
532
533         int     txcw;
534         int     fcrtl;
535         int     fcrth;
536 };
537
538 static inline uint32_t csr32r(struct ctlr *c, uintptr_t reg)
539 {
540         return read_mmreg32((uintptr_t)(c->nic + (reg / 4)));
541 }
542
543 static inline void csr32w(struct ctlr *c, uintptr_t reg, uint32_t val)
544 {
545         write_mmreg32((uintptr_t)(c->nic + (reg / 4)), val);
546 }
547
548 static struct ctlr* igbectlrhead;
549 static struct ctlr* igbectlrtail;
550
551 /* lock for igberpool (free receive Blocks) */
552 static spinlock_t igberblock = SPINLOCK_INITIALIZER_IRQSAVE;
553 static struct block* igberbpool;        /* receive Blocks for all igbe controllers */
554
555 static char* statistics[Nstatistics] = {
556         "CRC Error",
557         "Alignment Error",
558         "Symbol Error",
559         "RX Error",
560         "Missed Packets",
561         "Single Collision",
562         "Excessive Collisions",
563         "Multiple Collision",
564         "Late Collisions",
565         NULL,
566         "Collision",
567         "Transmit Underrun",
568         "Defer",
569         "Transmit - No CRS",
570         "Sequence Error",
571         "Carrier Extension Error",
572         "Receive Error Length",
573         NULL,
574         "XON Received",
575         "XON Transmitted",
576         "XOFF Received",
577         "XOFF Transmitted",
578         "FC Received Unsupported",
579         "Packets Received (64 Bytes)",
580         "Packets Received (65-127 Bytes)",
581         "Packets Received (128-255 Bytes)",
582         "Packets Received (256-511 Bytes)",
583         "Packets Received (512-1023 Bytes)",
584         "Packets Received (1024-1522 Bytes)",
585         "Good Packets Received",
586         "Broadcast Packets Received",
587         "Multicast Packets Received",
588         "Good Packets Transmitted",
589         NULL,
590         "Good Octets Received",
591         NULL,
592         "Good Octets Transmitted",
593         NULL,
594         NULL,
595         NULL,
596         "Receive No Buffers",
597         "Receive Undersize",
598         "Receive Fragment",
599         "Receive Oversize",
600         "Receive Jabber",
601         NULL,
602         NULL,
603         NULL,
604         "Total Octets Received",
605         NULL,
606         "Total Octets Transmitted",
607         NULL,
608         "Total Packets Received",
609         "Total Packets Transmitted",
610         "Packets Transmitted (64 Bytes)",
611         "Packets Transmitted (65-127 Bytes)",
612         "Packets Transmitted (128-255 Bytes)",
613         "Packets Transmitted (256-511 Bytes)",
614         "Packets Transmitted (512-1023 Bytes)",
615         "Packets Transmitted (1024-1522 Bytes)",
616         "Multicast Packets Transmitted",
617         "Broadcast Packets Transmitted",
618         "TCP Segmentation Context Transmitted",
619         "TCP Segmentation Context Fail",
620 };
621
622 static long
623 igbeifstat(struct ether* edev, void* a, long n, uint32_t offset)
624 {
625         struct ctlr *ctlr;
626         char *p, *s;
627         int i, l, r;
628         uint64_t tuvl, ruvl;
629
630         ctlr = edev->ctlr;
631         qlock(&ctlr->slock);
632         p = kzmalloc(READSTR, 0);
633         if(p == NULL) {
634                 qunlock(&ctlr->slock);
635                 error(Enomem);
636         }
637         l = 0;
638         for(i = 0; i < Nstatistics; i++){
639                 r = csr32r(ctlr, Statistics+i*4);
640                 if((s = statistics[i]) == NULL)
641                         continue;
642                 switch(i){
643                 case Gorcl:
644                 case Gotcl:
645                 case Torl:
646                 case Totl:
647                         ruvl = r;
648                         ruvl += ((uint64_t)csr32r(ctlr, Statistics+(i+1)*4))<<32;
649                         tuvl = ruvl;
650                         tuvl += ctlr->statistics[i];
651                         tuvl += ((uint64_t)ctlr->statistics[i+1])<<32;
652                         if(tuvl == 0)
653                                 continue;
654                         ctlr->statistics[i] = tuvl;
655                         ctlr->statistics[i+1] = tuvl>>32;
656                         l += snprintf(p+l, READSTR-l, "%s: %llud %llud\n",
657                                 s, tuvl, ruvl);
658                         i++;
659                         break;
660
661                 default:
662                         ctlr->statistics[i] += r;
663                         if(ctlr->statistics[i] == 0)
664                                 continue;
665                         l += snprintf(p+l, READSTR-l, "%s: %ud %ud\n",
666                                 s, ctlr->statistics[i], r);
667                         break;
668                 }
669         }
670
671         l += snprintf(p+l, READSTR-l, "lintr: %ud %ud\n",
672                 ctlr->lintr, ctlr->lsleep);
673         l += snprintf(p+l, READSTR-l, "rintr: %ud %ud\n",
674                 ctlr->rintr, ctlr->rsleep);
675         l += snprintf(p+l, READSTR-l, "tintr: %ud %ud\n",
676                 ctlr->tintr, ctlr->txdw);
677         l += snprintf(p+l, READSTR-l, "ixcs: %ud %ud %ud\n",
678                 ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
679         l += snprintf(p+l, READSTR-l, "rdtr: %ud\n", ctlr->rdtr);
680         l += snprintf(p+l, READSTR-l, "Ctrlext: %08x\n", csr32r(ctlr, Ctrlext));
681
682         l += snprintf(p+l, READSTR-l, "eeprom:");
683         for(i = 0; i < 0x40; i++){
684                 if(i && ((i & 0x07) == 0))
685                         l += snprintf(p+l, READSTR-l, "\n       ");
686                 l += snprintf(p+l, READSTR-l, " %4.4uX", ctlr->eeprom[i]);
687         }
688         l += snprintf(p+l, READSTR-l, "\n");
689
690         if(ctlr->mii != NULL && ctlr->mii->curphy != NULL){
691                 l += snprintf(p+l, READSTR-l, "phy:   ");
692                 for(i = 0; i < NMiiPhyr; i++){
693                         if(i && ((i & 0x07) == 0))
694                                 l += snprintf(p+l, READSTR-l, "\n       ");
695                         r = miimir(ctlr->mii, i);
696                         l += snprintf(p+l, READSTR-l, " %4.4uX", r);
697                 }
698                 snprintf(p+l, READSTR-l, "\n");
699         }
700         n = readstr(offset, a, n, p);
701         kfree(p);
702         qunlock(&ctlr->slock);
703
704         return n;
705 }
706
707 enum {
708         CMrdtr,
709 };
710
711 static struct cmdtab igbectlmsg[] = {
712         {CMrdtr,        "rdtr", 2},
713 };
714
715 static long
716 igbectl(struct ether* edev, void* buf, long n)
717 {
718         ERRSTACK(2);
719         int v;
720         char *p;
721         struct ctlr *ctlr;
722         struct cmdbuf *cb;
723         struct cmdtab *ct;
724
725         if((ctlr = edev->ctlr) == NULL)
726                 error(Enonexist);
727
728         cb = parsecmd(buf, n);
729         if(waserror()){
730                 kfree(cb);
731                 nexterror();
732         }
733
734         ct = lookupcmd(cb, igbectlmsg, ARRAY_SIZE(igbectlmsg));
735         switch(ct->index){
736         case CMrdtr:
737                 v = strtol(cb->f[1], &p, 0);
738                 if(v < 0 || p == cb->f[1] || v > 0xFFFF)
739                         error(Ebadarg);
740                 ctlr->rdtr = v;
741                 csr32w(ctlr, Rdtr, Fpd|v);
742                 break;
743         }
744         kfree(cb);
745         poperror();
746
747         return n;
748 }
749
750 static void
751 igbepromiscuous(void* arg, int on)
752 {
753         int rctl;
754         struct ctlr *ctlr;
755         struct ether *edev;
756
757         edev = arg;
758         ctlr = edev->ctlr;
759
760         rctl = csr32r(ctlr, Rctl);
761         rctl &= ~MoMASK;
762         rctl |= Mo47b36;
763         if(on)
764                 rctl |= Upe|Mpe;
765         else
766                 rctl &= ~(Upe|Mpe);
767         csr32w(ctlr, Rctl, rctl|Mpe);   /* temporarily keep Mpe on */
768 }
769
770 static void
771 igbemulticast(void* arg, uint8_t* addr, int add)
772 {
773         int bit, x;
774         struct ctlr *ctlr;
775         struct ether *edev;
776
777         edev = arg;
778         ctlr = edev->ctlr;
779
780         x = addr[5]>>1;
781         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
782         /*
783          * multiple ether addresses can hash to the same filter bit,
784          * so it's never safe to clear a filter bit.
785          * if we want to clear filter bits, we need to keep track of
786          * all the multicast addresses in use, clear all the filter bits,
787          * then set the ones corresponding to in-use addresses.
788          */
789         if(add)
790                 ctlr->mta[x] |= 1<<bit;
791 //      else
792 //              ctlr->mta[x] &= ~(1<<bit);
793
794         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
795 }
796
797 static struct block*
798 igberballoc(void)
799 {
800         struct block *bp;
801
802         ilock(&igberblock);
803         if((bp = igberbpool) != NULL){
804                 igberbpool = bp->next;
805                 bp->next = NULL;
806                 /* _xinc(&bp->ref);     prevent bp from being freed */
807         }
808         iunlock(&igberblock);
809
810         return bp;
811 }
812
813 static void
814 igberbfree(struct block* bp)
815 {
816         bp->rp = bp->lim - Rbsz;
817         bp->wp = bp->rp;
818         bp->flag &= ~(Bipck | Budpck | Btcpck | Bpktck);
819
820         ilock(&igberblock);
821         bp->next = igberbpool;
822         igberbpool = bp;
823         iunlock(&igberblock);
824 }
825
826 static void
827 igbeim(struct ctlr* ctlr, int im)
828 {
829         ilock(&ctlr->imlock);
830         ctlr->im |= im;
831         csr32w(ctlr, Ims, ctlr->im);
832         iunlock(&ctlr->imlock);
833 }
834
835 static int
836 igbelim(void* ctlr)
837 {
838         return ((struct ctlr*)ctlr)->lim != 0;
839 }
840
841 static void
842 igbelproc(void* arg)
843 {
844         struct ctlr *ctlr;
845         struct ether *edev;
846         struct miiphy *phy;
847         int ctrl, r;
848
849         edev = arg;
850         ctlr = edev->ctlr;
851         for(;;){
852                 /* plan9 originally had a busy loop here (just called continue).  though
853                  * either you have the mii or you don't.  i don't think it'll magically
854                  * show up later (it should have been initialized during pnp/pci, which
855                  * is before attach, which is before lproc).  -brho */
856                 if (ctlr->mii == NULL || ctlr->mii->curphy == NULL) {
857                         printk("[kernel] igbelproc can't find a mii/curphy, aborting!\n");
858                         /* name alloc'd in attach */
859                         kfree(per_cpu_info[core_id()].cur_kthread->name);
860                         return;
861                 }
862                 /*
863                  * To do:
864                  *      logic to manage status change,
865                  *      this is incomplete but should work
866                  *      one time to set up the hardware.
867                  *
868                  *      MiiPhy.speed, etc. should be in Mii.
869                  */
870                 if(miistatus(ctlr->mii) < 0)
871                         //continue;     /* this comment out was plan9, not brho */
872                         goto enable;
873
874                 phy = ctlr->mii->curphy;
875                 ctrl = csr32r(ctlr, Ctrl);
876
877                 switch(ctlr->id){
878                 case i82543gc:
879                 case i82544ei:
880                 case i82544eif:
881                 default:
882                         if(!(ctrl & Asde)){
883                                 ctrl &= ~(SspeedMASK|Ilos|Fd);
884                                 ctrl |= Frcdplx|Frcspd;
885                                 if(phy->speed == 1000)
886                                         ctrl |= Sspeed1000;
887                                 else if(phy->speed == 100)
888                                         ctrl |= Sspeed100;
889                                 if(phy->fd)
890                                         ctrl |= Fd;
891                         }
892                         break;
893
894                 case i82540em:
895                 case i82540eplp:
896                 case i82547gi:
897                 case i82541gi:
898                 case i82541gi2:
899                 case i82541pi:
900                         break;
901                 }
902
903                 /*
904                  * Collision Distance.
905                  */
906                 r = csr32r(ctlr, Tctl);
907                 r &= ~ColdMASK;
908                 if(phy->fd)
909                         r |= 64<<ColdSHIFT;
910                 else
911                         r |= 512<<ColdSHIFT;
912                 csr32w(ctlr, Tctl, r);
913
914                 /*
915                  * Flow control.
916                  */
917                 if(phy->rfc)
918                         ctrl |= Rfce;
919                 if(phy->tfc)
920                         ctrl |= Tfce;
921                 csr32w(ctlr, Ctrl, ctrl);
922
923 enable:
924                 ctlr->lim = 0;
925                 igbeim(ctlr, Lsc);
926
927                 ctlr->lsleep++;
928                 rendez_sleep(&ctlr->lrendez, igbelim, ctlr);
929         }
930 }
931
932 static void
933 igbetxinit(struct ctlr* ctlr)
934 {
935         int i, r;
936         struct block *bp;
937
938         csr32w(ctlr, Tctl, (0x0F<<CtSHIFT)|Psp|(66<<ColdSHIFT));
939         switch(ctlr->id){
940         default:
941                 r = 6;
942                 break;
943         case i82543gc:
944         case i82544ei:
945         case i82544eif:
946         case i82544gc:
947         case i82540em:
948         case i82540eplp:
949         case i82541ei:
950         case i82541gi:
951         case i82541gi2:
952         case i82541pi:
953         case i82545em:
954         case i82545gmc:
955         case i82546gb:
956         case i82546eb:
957         case i82547ei:
958         case i82547gi:
959                 r = 8;
960                 break;
961         }
962         csr32w(ctlr, Tipg, (6<<20)|(8<<10)|r);
963         csr32w(ctlr, Ait, 0);
964         csr32w(ctlr, Txdmac, 0);
965         csr32w(ctlr, Tdbal, paddr_low32(ctlr->tdba));
966         csr32w(ctlr, Tdbah, paddr_high32(ctlr->tdba));
967         csr32w(ctlr, Tdlen, ctlr->ntd*sizeof(Td));
968         ctlr->tdh = PREV_RING(0, ctlr->ntd);
969         csr32w(ctlr, Tdh, 0);
970         ctlr->tdt = 0;
971         csr32w(ctlr, Tdt, 0);
972
973         for(i = 0; i < ctlr->ntd; i++){
974                 if((bp = ctlr->tb[i]) != NULL){
975                         ctlr->tb[i] = NULL;
976                         freeb(bp);
977                 }
978                 memset(&ctlr->tdba[i], 0, sizeof(Td));
979         }
980         ctlr->tdfree = ctlr->ntd;
981
982         csr32w(ctlr, Tidv, 128);
983         r = (4<<WthreshSHIFT)|(4<<HthreshSHIFT)|(8<<PthreshSHIFT);
984
985         switch(ctlr->id){
986         default:
987                 break;
988         case i82540em:
989         case i82540eplp:
990         case i82547gi:
991         case i82545em:
992         case i82545gmc:
993         case i82546gb:
994         case i82546eb:
995         case i82541gi:
996         case i82541gi2:
997         case i82541pi:
998                 r = csr32r(ctlr, Txdctl);
999                 r &= ~WthreshMASK;
1000                 r |= Gran|(4<<WthreshSHIFT);
1001
1002                 csr32w(ctlr, Tadv, 64);
1003                 break;
1004         }
1005
1006         csr32w(ctlr, Txdctl, r);
1007
1008         r = csr32r(ctlr, Tctl);
1009         r |= Ten;
1010         csr32w(ctlr, Tctl, r);
1011 }
1012
1013 static void
1014 igbetransmit(struct ether* edev)
1015 {
1016         Td *td;
1017         struct block *bp;
1018         struct ctlr *ctlr;
1019         int tdh, tdt;
1020
1021         ctlr = edev->ctlr;
1022
1023         ilock(&ctlr->tlock);
1024
1025         /*
1026          * Free any completed packets
1027          */
1028         tdh = ctlr->tdh;
1029         while(NEXT_RING(tdh, ctlr->ntd) != csr32r(ctlr, Tdh)){
1030                 if((bp = ctlr->tb[tdh]) != NULL){
1031                         ctlr->tb[tdh] = NULL;
1032                         freeb(bp);
1033                 }
1034                 memset(&ctlr->tdba[tdh], 0, sizeof(Td));
1035                 tdh = NEXT_RING(tdh, ctlr->ntd);
1036         }
1037         ctlr->tdh = tdh;
1038
1039         /*
1040          * Try to fill the ring back up.
1041          */
1042         tdt = ctlr->tdt;
1043         while(NEXT_RING(tdt, ctlr->ntd) != tdh){
1044                 if((bp = qget(edev->oq)) == NULL)
1045                         break;
1046                 td = &ctlr->tdba[tdt];
1047                 td->addr[0] = paddr_low32(bp->rp);
1048                 td->addr[1] = paddr_high32(bp->rp);
1049                 td->control = ((BLEN(bp) & LenMASK)<<LenSHIFT);
1050                 td->control |= Dext|Ifcs|Teop|DtypeDD;
1051                 ctlr->tb[tdt] = bp;
1052                 tdt = NEXT_RING(tdt, ctlr->ntd);
1053                 if(NEXT_RING(tdt, ctlr->ntd) == tdh){
1054                         td->control |= Rs;
1055                         ctlr->txdw++;
1056                         ctlr->tdt = tdt;
1057                         csr32w(ctlr, Tdt, tdt);
1058                         igbeim(ctlr, Txdw);
1059                         break;
1060                 }
1061                 ctlr->tdt = tdt;
1062                 csr32w(ctlr, Tdt, tdt);
1063         }
1064
1065         iunlock(&ctlr->tlock);
1066 }
1067
1068 static void
1069 igbereplenish(struct ctlr* ctlr)
1070 {
1071         Rd *rd;
1072         int rdt;
1073         struct block *bp;
1074
1075         rdt = ctlr->rdt;
1076         while(NEXT_RING(rdt, ctlr->nrd) != ctlr->rdh){
1077                 rd = &ctlr->rdba[rdt];
1078                 if(ctlr->rb[rdt] == NULL){
1079                         bp = igberballoc();
1080                         if(bp == NULL){
1081                                 /* needs to be a safe print for interrupt level */
1082                                 printk("#l%d: igbereplenish: no available buffers\n",
1083                                         ctlr->edev->ctlrno);
1084                                 break;
1085                         }
1086                         ctlr->rb[rdt] = bp;
1087                         rd->addr[0] = paddr_low32(bp->rp);
1088                         rd->addr[1] = paddr_high32(bp->rp);
1089                 }
1090                 wmb();  /* ensure prev rd writes come before status = 0. */
1091                 rd->status = 0;
1092                 rdt = NEXT_RING(rdt, ctlr->nrd);
1093                 ctlr->rdfree++;
1094         }
1095         ctlr->rdt = rdt;
1096         csr32w(ctlr, Rdt, rdt);
1097 }
1098
1099 static void
1100 igberxinit(struct ctlr* ctlr)
1101 {
1102         int i;
1103         struct block *bp;
1104
1105         /* temporarily keep Mpe on */
1106         csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF|Mpe);
1107         csr32w(ctlr, Rdbal, paddr_low32(ctlr->rdba));
1108         csr32w(ctlr, Rdbah, paddr_high32(ctlr->rdba));
1109         csr32w(ctlr, Rdlen, ctlr->nrd*sizeof(Rd));
1110         ctlr->rdh = 0;
1111         csr32w(ctlr, Rdh, 0);
1112         ctlr->rdt = 0;
1113         csr32w(ctlr, Rdt, 0);
1114         ctlr->rdtr = 0;
1115         csr32w(ctlr, Rdtr, Fpd|0);
1116
1117         for(i = 0; i < ctlr->nrd; i++){
1118                 if((bp = ctlr->rb[i]) != NULL){
1119                         ctlr->rb[i] = NULL;
1120                         freeb(bp);
1121                 }
1122         }
1123         igbereplenish(ctlr);
1124
1125         switch(ctlr->id){
1126         case i82540em:
1127         case i82540eplp:
1128         case i82541gi:
1129         case i82541gi2:
1130         case i82541pi:
1131         case i82545em:
1132         case i82545gmc:
1133         case i82546gb:
1134         case i82546eb:
1135         case i82547gi:
1136                 csr32w(ctlr, Radv, 64);
1137                 break;
1138         }
1139         csr32w(ctlr, Rxdctl, (8<<WthreshSHIFT)|(8<<HthreshSHIFT)|4);
1140
1141         /*
1142          * Enable checksum offload.
1143          */
1144         csr32w(ctlr, Rxcsum, Tuofl|Ipofl|(ETHERHDRSIZE<<PcssSHIFT));
1145 }
1146
1147 static int
1148 igberim(void* ctlr)
1149 {
1150         return ((struct ctlr*)ctlr)->rim != 0;
1151 }
1152
1153 static void
1154 igberproc(void* arg)
1155 {
1156         Rd *rd;
1157         struct block *bp;
1158         struct ctlr *ctlr;
1159         int r, rdh;
1160         struct ether *edev;
1161
1162         edev = arg;
1163         ctlr = edev->ctlr;
1164
1165         igberxinit(ctlr);
1166         r = csr32r(ctlr, Rctl);
1167         r |= Ren;
1168         csr32w(ctlr, Rctl, r);
1169
1170         for(;;){
1171                 ctlr->rim = 0;
1172                 igbeim(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq);
1173                 ctlr->rsleep++;
1174                 rendez_sleep(&ctlr->rrendez, igberim, ctlr);
1175
1176                 rdh = ctlr->rdh;
1177                 for(;;){
1178                         rd = &ctlr->rdba[rdh];
1179
1180                         if(!(rd->status & Rdd))
1181                                 break;
1182
1183                         /*
1184                          * Accept eop packets with no errors.
1185                          * With no errors and the Ixsm bit set,
1186                          * the descriptor status Tpcs and Ipcs bits give
1187                          * an indication of whether the checksums were
1188                          * calculated and valid.
1189                          */
1190                         if((rd->status & Reop) && rd->errors == 0){
1191                                 bp = ctlr->rb[rdh];
1192                                 ctlr->rb[rdh] = NULL;
1193                                 bp->wp += rd->length;
1194                                 bp->next = NULL;
1195                                 if(!(rd->status & Ixsm)){
1196                                         ctlr->ixsm++;
1197                                         if(rd->status & Ipcs){
1198                                                 /*
1199                                                  * IP checksum calculated
1200                                                  * (and valid as errors == 0).
1201                                                  */
1202                                                 ctlr->ipcs++;
1203                                                 bp->flag |= Bipck;
1204                                         }
1205                                         if(rd->status & Tcpcs){
1206                                                 /*
1207                                                  * TCP/UDP checksum calculated
1208                                                  * (and valid as errors == 0).
1209                                                  */
1210                                                 ctlr->tcpcs++;
1211                                                 bp->flag |= Btcpck|Budpck;
1212                                         }
1213                                         bp->checksum = rd->checksum;
1214                                         bp->flag |= Bpktck;
1215                                 }
1216                                 etheriq(edev, bp, 1);
1217                         }
1218                         else if(ctlr->rb[rdh] != NULL){
1219                                 freeb(ctlr->rb[rdh]);
1220                                 ctlr->rb[rdh] = NULL;
1221                         }
1222
1223                         memset(rd, 0, sizeof(Rd));
1224                         wmb();  /* make sure the zeroing happens before free (i think) */
1225                         ctlr->rdfree--;
1226                         rdh = NEXT_RING(rdh, ctlr->nrd);
1227                 }
1228                 ctlr->rdh = rdh;
1229
1230                 if(ctlr->rdfree < ctlr->nrd/2 || (ctlr->rim & Rxdmt0))
1231                         igbereplenish(ctlr);
1232         }
1233 }
1234
1235 static void
1236 igbeattach(struct ether* edev)
1237 {
1238         ERRSTACK(1);
1239         struct block *bp;
1240         struct ctlr *ctlr;
1241         char *name;
1242
1243         ctlr = edev->ctlr;
1244         ctlr->edev = edev;                      /* point back to Ether* */
1245         qlock(&ctlr->alock);
1246         if(ctlr->alloc != NULL){                        /* already allocated? */
1247                 qunlock(&ctlr->alock);
1248                 return;
1249         }
1250
1251         ctlr->tb = NULL;
1252         ctlr->rb = NULL;
1253         ctlr->alloc = NULL;
1254         ctlr->nrb = 0;
1255         if(waserror()){
1256                 while(ctlr->nrb > 0){
1257                         bp = igberballoc();
1258                         bp->free = NULL;
1259                         freeb(bp);
1260                         ctlr->nrb--;
1261                 }
1262                 kfree(ctlr->tb);
1263                 ctlr->tb = NULL;
1264                 kfree(ctlr->rb);
1265                 ctlr->rb = NULL;
1266                 kfree(ctlr->alloc);
1267                 ctlr->alloc = NULL;
1268                 qunlock(&ctlr->alock);
1269                 nexterror();
1270         }
1271
1272         ctlr->nrd = Nrd;
1273         ctlr->ntd = Ntd;
1274         ctlr->alloc = kzmalloc(ctlr->nrd * sizeof(Rd) + ctlr->ntd * sizeof(Td) + 127, 0);
1275         if(ctlr->alloc == NULL) {
1276                 printd("igbe: can't allocate ctlr->alloc\n");
1277                 error(Enomem);
1278         }
1279         ctlr->rdba = (Rd*)ROUNDUP((uintptr_t)ctlr->alloc, 128);
1280         ctlr->tdba = (Td*)(ctlr->rdba+ctlr->nrd);
1281
1282         ctlr->rb = kzmalloc(ctlr->nrd * sizeof(struct block *), 0);
1283         ctlr->tb = kzmalloc(ctlr->ntd * sizeof(struct block *), 0);
1284         if (ctlr->rb == NULL || ctlr->tb == NULL) {
1285                 printd("igbe: can't allocate ctlr->rb or ctlr->tb\n");
1286                 error(Enomem);
1287         }
1288
1289         for(ctlr->nrb = 0; ctlr->nrb < Nrb; ctlr->nrb++){
1290                 if((bp = allocb(Rbsz)) == NULL)
1291                         break;
1292                 bp->free = igberbfree;
1293                 freeb(bp);
1294         }
1295
1296         /* the ktasks should free these names, if they ever exit */
1297         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1298         snprintf(name, KNAMELEN, "#l%dlproc", edev->ctlrno);
1299         ktask(name, igbelproc, edev);
1300
1301         name = kmalloc(KNAMELEN, KMALLOC_WAIT);
1302         snprintf(name, KNAMELEN, "#l%drproc", edev->ctlrno);
1303         ktask(name, igberproc, edev);
1304
1305         igbetxinit(ctlr);
1306
1307         qunlock(&ctlr->alock);
1308         poperror();
1309 }
1310
1311 static void igbeinterrupt(struct hw_trapframe *hw_tf, void *arg)
1312 {
1313         struct ctlr *ctlr;
1314         struct ether *edev;
1315         int icr, im, txdw;
1316
1317         edev = arg;
1318         ctlr = edev->ctlr;
1319
1320         ilock(&ctlr->imlock);
1321         csr32w(ctlr, Imc, ~0);
1322         im = ctlr->im;
1323         txdw = 0;
1324
1325         while((icr = csr32r(ctlr, Icr) & ctlr->im) != 0){
1326                 if(icr & Lsc){
1327                         im &= ~Lsc;
1328                         ctlr->lim = icr & Lsc;
1329                         rendez_wakeup(&ctlr->lrendez);
1330                         ctlr->lintr++;
1331                 }
1332                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq)){
1333                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq);
1334                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq);
1335                         rendez_wakeup(&ctlr->rrendez);
1336                         ctlr->rintr++;
1337                 }
1338                 if(icr & Txdw){
1339                         im &= ~Txdw;
1340                         txdw++;
1341                         ctlr->tintr++;
1342                 }
1343         }
1344
1345         ctlr->im = im;
1346         csr32w(ctlr, Ims, im);
1347         iunlock(&ctlr->imlock);
1348
1349         if(txdw)
1350                 igbetransmit(edev);
1351 }
1352
1353 static int
1354 i82543mdior(struct ctlr* ctlr, int n)
1355 {
1356         int ctrl, data, i, r;
1357
1358         /*
1359          * Read n bits from the Management Data I/O Interface.
1360          */
1361         ctrl = csr32r(ctlr, Ctrl);
1362         r = (ctrl & ~Mddo)|Mdco;
1363         data = 0;
1364         for(i = n-1; i >= 0; i--){
1365                 if(csr32r(ctlr, Ctrl) & Mdd)
1366                         data |= (1<<i);
1367                 csr32w(ctlr, Ctrl, Mdc|r);
1368                 csr32w(ctlr, Ctrl, r);
1369         }
1370         csr32w(ctlr, Ctrl, ctrl);
1371
1372         return data;
1373 }
1374
1375 static int
1376 i82543mdiow(struct ctlr* ctlr, int bits, int n)
1377 {
1378         int ctrl, i, r;
1379
1380         /*
1381          * Write n bits to the Management Data I/O Interface.
1382          */
1383         ctrl = csr32r(ctlr, Ctrl);
1384         r = Mdco|Mddo|ctrl;
1385         for(i = n-1; i >= 0; i--){
1386                 if(bits & (1<<i))
1387                         r |= Mdd;
1388                 else
1389                         r &= ~Mdd;
1390                 csr32w(ctlr, Ctrl, Mdc|r);
1391                 csr32w(ctlr, Ctrl, r);
1392         }
1393         csr32w(ctlr, Ctrl, ctrl);
1394
1395         return 0;
1396 }
1397
1398 static int
1399 i82543miimir(struct mii* mii, int pa, int ra)
1400 {
1401         int data;
1402         struct ctlr *ctlr;
1403
1404         ctlr = mii->ctlr;
1405
1406         /*
1407          * MII Management Interface Read.
1408          *
1409          * Preamble;
1410          * ST+OP+PHYAD+REGAD;
1411          * TA + 16 data bits.
1412          */
1413         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1414         i82543mdiow(ctlr, 0x1800|(pa<<5)|ra, 14);
1415         data = i82543mdior(ctlr, 18);
1416
1417         if(data & 0x10000)
1418                 return -1;
1419
1420         return data & 0xFFFF;
1421 }
1422
1423 static int
1424 i82543miimiw(struct mii* mii, int pa, int ra, int data)
1425 {
1426         struct ctlr *ctlr;
1427
1428         ctlr = mii->ctlr;
1429
1430         /*
1431          * MII Management Interface Write.
1432          *
1433          * Preamble;
1434          * ST+OP+PHYAD+REGAD+TA + 16 data bits;
1435          * Z.
1436          */
1437         i82543mdiow(ctlr, 0xFFFFFFFF, 32);
1438         data &= 0xFFFF;
1439         data |= (0x05<<(5+5+2+16))|(pa<<(5+2+16))|(ra<<(2+16))|(0x02<<16);
1440         i82543mdiow(ctlr, data, 32);
1441
1442         return 0;
1443 }
1444
1445 static int
1446 igbemiimir(struct mii* mii, int pa, int ra)
1447 {
1448         struct ctlr *ctlr;
1449         int mdic, timo;
1450
1451         ctlr = mii->ctlr;
1452
1453         csr32w(ctlr, Mdic, MDIrop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT));
1454         mdic = 0;
1455         for(timo = 64; timo; timo--){
1456                 mdic = csr32r(ctlr, Mdic);
1457                 if(mdic & (MDIe|MDIready))
1458                         break;
1459                 udelay(1);
1460         }
1461
1462         if((mdic & (MDIe|MDIready)) == MDIready)
1463                 return mdic & 0xFFFF;
1464         return -1;
1465 }
1466
1467 static int
1468 igbemiimiw(struct mii* mii, int pa, int ra, int data)
1469 {
1470         struct ctlr *ctlr;
1471         int mdic, timo;
1472
1473         ctlr = mii->ctlr;
1474
1475         data &= MDIdMASK;
1476         csr32w(ctlr, Mdic, MDIwop|(pa<<MDIpSHIFT)|(ra<<MDIrSHIFT)|data);
1477         mdic = 0;
1478         for(timo = 64; timo; timo--){
1479                 mdic = csr32r(ctlr, Mdic);
1480                 if(mdic & (MDIe|MDIready))
1481                         break;
1482                 udelay(1);
1483         }
1484         if((mdic & (MDIe|MDIready)) == MDIready)
1485                 return 0;
1486         return -1;
1487 }
1488
1489 static int
1490 i82543miirw(struct mii* mii, int write, int pa, int ra, int data)
1491 {
1492         if(write)
1493                 return i82543miimiw(mii, pa, ra, data);
1494
1495         return i82543miimir(mii, pa, ra);
1496 }
1497
1498 static int
1499 igbemiirw(struct mii* mii, int write, int pa, int ra, int data)
1500 {
1501         if(write)
1502                 return igbemiimiw(mii, pa, ra, data);
1503
1504         return igbemiimir(mii, pa, ra);
1505 }
1506
1507 static int
1508 igbemii(struct ctlr* ctlr)
1509 {
1510         int ctrl, p, r;
1511         int (*rw)(struct mii*, int unused_int, int, int, int);
1512
1513         r = csr32r(ctlr, Status);
1514         if(r & Tbimode)
1515                 return -1;
1516
1517         ctrl = csr32r(ctlr, Ctrl);
1518         ctrl |= Slu;
1519
1520         switch(ctlr->id){
1521         case i82543gc:
1522                 ctrl |= Frcdplx|Frcspd;
1523                 csr32w(ctlr, Ctrl, ctrl);
1524
1525                 /*
1526                  * The reset pin direction (Mdro) should already
1527                  * be set from the EEPROM load.
1528                  * If it's not set this configuration is unexpected
1529                  * so bail.
1530                  */
1531                 r = csr32r(ctlr, Ctrlext);
1532                 if(!(r & Mdro))
1533                         return -1;
1534                 csr32w(ctlr, Ctrlext, r);
1535                 udelay(20*1000);
1536                 r = csr32r(ctlr, Ctrlext);
1537                 r &= ~Mdr;
1538                 csr32w(ctlr, Ctrlext, r);
1539                 udelay(20*1000);
1540                 r = csr32r(ctlr, Ctrlext);
1541                 r |= Mdr;
1542                 csr32w(ctlr, Ctrlext, r);
1543                 udelay(20*1000);
1544
1545                 rw = i82543miirw;
1546                 break;
1547         case i82544ei:
1548         case i82544eif:
1549         case i82544gc:
1550         case i82540em:
1551         case i82540eplp:
1552         case i82547ei:
1553         case i82547gi:
1554         case i82541ei:
1555         case i82541gi:
1556         case i82541gi2:
1557         case i82541pi:
1558         case i82545em:
1559         case i82545gmc:
1560         case i82546gb:
1561         case i82546eb:
1562                 ctrl &= ~(Frcdplx|Frcspd);
1563                 csr32w(ctlr, Ctrl, ctrl);
1564                 rw = igbemiirw;
1565                 break;
1566         default:
1567                 return -1;
1568         }
1569
1570         if (!(ctlr->mii = miiattach(ctlr, ~0, rw)))
1571                 return -1;
1572         // print("oui %X phyno %d\n", phy->oui, phy->phyno);
1573
1574         /*
1575          * 8254X-specific PHY registers not in 802.3:
1576          *      0x10    PHY specific control
1577          *      0x14    extended PHY specific control
1578          * Set appropriate values then reset the PHY to have
1579          * changes noted.
1580          */
1581         switch(ctlr->id){
1582         case i82547gi:
1583         case i82541gi:
1584         case i82541gi2:
1585         case i82541pi:
1586         case i82545em:
1587         case i82545gmc:
1588         case i82546gb:
1589         case i82546eb:
1590                 break;
1591         default:
1592                 r = miimir(ctlr->mii, 16);
1593                 r |= 0x0800;                    /* assert CRS on Tx */
1594                 r |= 0x0060;                    /* auto-crossover all speeds */
1595                 r |= 0x0002;                    /* polarity reversal enabled */
1596                 miimiw(ctlr->mii, 16, r);
1597
1598                 r = miimir(ctlr->mii, 20);
1599                 r |= 0x0070;                    /* +25MHz clock */
1600                 r &= ~0x0F00;
1601                 r |= 0x0100;                    /* 1x downshift */
1602                 miimiw(ctlr->mii, 20, r);
1603
1604                 miireset(ctlr->mii);
1605                 p = 0;
1606                 if(ctlr->txcw & TxcwPs)
1607                         p |= AnaP;
1608                 if(ctlr->txcw & TxcwAs)
1609                         p |= AnaAP;
1610                 miiane(ctlr->mii, ~0, p, ~0);
1611                 break;
1612         }
1613         return 0;
1614 }
1615
1616 static int
1617 at93c46io(struct ctlr* ctlr, char* op, int data)
1618 {
1619         char *lp, *p;
1620         int i, loop, eecd, r;
1621
1622         eecd = csr32r(ctlr, Eecd);
1623
1624         r = 0;
1625         loop = -1;
1626         lp = NULL;
1627         for(p = op; *p != '\0'; p++){
1628                 switch(*p){
1629                 default:
1630                         return -1;
1631                 case ' ':
1632                         continue;
1633                 case ':':                       /* start of loop */
1634                         loop = strtol(p+1, &lp, 0)-1;
1635                         lp--;
1636                         if(p == lp)
1637                                 loop = 7;
1638                         p = lp;
1639                         continue;
1640                 case ';':                       /* end of loop */
1641                         if(lp == NULL)
1642                                 return -1;
1643                         loop--;
1644                         if(loop >= 0)
1645                                 p = lp;
1646                         else
1647                                 lp = NULL;
1648                         continue;
1649                 case 'C':                       /* assert clock */
1650                         eecd |= Sk;
1651                         break;
1652                 case 'c':                       /* deassert clock */
1653                         eecd &= ~Sk;
1654                         break;
1655                 case 'D':                       /* next bit in 'data' byte */
1656                         if(loop < 0)
1657                                 return -1;
1658                         if(data & (1<<loop))
1659                                 eecd |= Di;
1660                         else
1661                                 eecd &= ~Di;
1662                         break;
1663                 case 'O':                       /* collect data output */
1664                         i = (csr32r(ctlr, Eecd) & Do) != 0;
1665                         if(loop >= 0)
1666                                 r |= (i<<loop);
1667                         else
1668                                 r = i;
1669                         continue;
1670                 case 'I':                       /* assert data input */
1671                         eecd |= Di;
1672                         break;
1673                 case 'i':                       /* deassert data input */
1674                         eecd &= ~Di;
1675                         break;
1676                 case 'S':                       /* enable chip select */
1677                         eecd |= Cs;
1678                         break;
1679                 case 's':                       /* disable chip select */
1680                         eecd &= ~Cs;
1681                         break;
1682                 }
1683                 csr32w(ctlr, Eecd, eecd);
1684                 udelay(50);
1685         }
1686         if(loop >= 0)
1687                 return -1;
1688         return r;
1689 }
1690
1691 static int
1692 at93c46r(struct ctlr* ctlr)
1693 {
1694         uint16_t sum;
1695         char rop[20];
1696         int addr, areq, bits, data, eecd, i;
1697
1698         eecd = csr32r(ctlr, Eecd);
1699         if(eecd & Spi){
1700                 printd("igbe: SPI EEPROM access not implemented\n");
1701                 return 0;
1702         }
1703         if(eecd & (Eeszaddr|Eesz256))
1704                 bits = 8;
1705         else
1706                 bits = 6;
1707
1708         sum = 0;
1709
1710         switch(ctlr->id){
1711         default:
1712                 areq = 0;
1713                 break;
1714         case i82540em:
1715         case i82540eplp:
1716         case i82541ei:
1717         case i82541gi:
1718         case i82541gi2:
1719         case i82541pi:
1720         case i82545em:
1721         case i82545gmc:
1722         case i82546gb:
1723         case i82546eb:
1724         case i82547ei:
1725         case i82547gi:
1726                 areq = 1;
1727                 csr32w(ctlr, Eecd, eecd|Areq);
1728                 for(i = 0; i < 1000; i++){
1729                         if((eecd = csr32r(ctlr, Eecd)) & Agnt)
1730                                 break;
1731                         udelay(5);
1732                 }
1733                 if(!(eecd & Agnt)){
1734                         printd("igbe: not granted EEPROM access\n");
1735                         goto release;
1736                 }
1737                 break;
1738         }
1739         snprintf(rop, sizeof(rop), "S :%dDCc;", bits+3);
1740
1741         for(addr = 0; addr < 0x40; addr++){
1742                 /*
1743                  * Read a word at address 'addr' from the Atmel AT93C46
1744                  * 3-Wire Serial EEPROM or compatible. The EEPROM access is
1745                  * controlled by 4 bits in Eecd. See the AT93C46 datasheet
1746                  * for protocol details.
1747                  */
1748                 if(at93c46io(ctlr, rop, (0x06<<bits)|addr) != 0){
1749                         printd("igbe: can't set EEPROM address 0x%2.2X\n", addr);
1750                         goto release;
1751                 }
1752                 data = at93c46io(ctlr, ":16COc;", 0);
1753                 at93c46io(ctlr, "sic", 0);
1754                 ctlr->eeprom[addr] = data;
1755                 sum += data;
1756         }
1757
1758 release:
1759         if(areq)
1760                 csr32w(ctlr, Eecd, eecd & ~Areq);
1761         return sum;
1762 }
1763
1764 static int
1765 igbedetach(struct ctlr* ctlr)
1766 {
1767         int r, timeo;
1768
1769         /*
1770          * Perform a device reset to get the chip back to the
1771          * power-on state, followed by an EEPROM reset to read
1772          * the defaults for some internal registers.
1773          */
1774         csr32w(ctlr, Imc, ~0);
1775         csr32w(ctlr, Rctl, 0);
1776         csr32w(ctlr, Tctl, 0);
1777
1778         udelay(10*1000);
1779
1780         csr32w(ctlr, Ctrl, Devrst);
1781         udelay(1*1000);
1782         for(timeo = 0; timeo < 1000; timeo++){
1783                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1784                         break;
1785                 udelay(1*1000);
1786         }
1787         if(csr32r(ctlr, Ctrl) & Devrst)
1788                 return -1;
1789         r = csr32r(ctlr, Ctrlext);
1790         csr32w(ctlr, Ctrlext, r|Eerst);
1791         udelay(1*1000);
1792
1793         for(timeo = 0; timeo < 1000; timeo++){
1794                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1795                         break;
1796                 udelay(1*1000);
1797         }
1798         if(csr32r(ctlr, Ctrlext) & Eerst)
1799                 return -1;
1800
1801         switch(ctlr->id){
1802         default:
1803                 break;
1804         case i82540em:
1805         case i82540eplp:
1806         case i82541gi:
1807         case i82541gi2:
1808         case i82541pi:
1809         case i82545em:
1810         case i82545gmc:
1811         case i82547gi:
1812         case i82546gb:
1813         case i82546eb:
1814                 r = csr32r(ctlr, Manc);
1815                 r &= ~Arpen;
1816                 csr32w(ctlr, Manc, r);
1817                 break;
1818         }
1819
1820         csr32w(ctlr, Imc, ~0);
1821         udelay(1*1000);
1822         for(timeo = 0; timeo < 1000; timeo++){
1823                 if(!csr32r(ctlr, Icr))
1824                         break;
1825                 udelay(1*1000);
1826         }
1827         if(csr32r(ctlr, Icr))
1828                 return -1;
1829
1830         return 0;
1831 }
1832
1833 static void
1834 igbeshutdown(struct ether* ether)
1835 {
1836         igbedetach(ether->ctlr);
1837 }
1838
1839 static int
1840 igbereset(struct ctlr* ctlr)
1841 {
1842         int ctrl, i, pause, r, swdpio, txcw;
1843
1844         if(igbedetach(ctlr))
1845                 return -1;
1846
1847         /*
1848          * Read the EEPROM, validate the checksum
1849          * then get the device back to a power-on state.
1850          */
1851         if((r = at93c46r(ctlr)) != 0xBABA){
1852                 printd("igbe: bad EEPROM checksum - 0x%4.4uX\n", r);
1853                 return -1;
1854         }
1855
1856         /*
1857          * Snarf and set up the receive addresses.
1858          * There are 16 addresses. The first should be the MAC address.
1859          * The others are cleared and not marked valid (MS bit of Rah).
1860          */
1861         if ((ctlr->id == i82546gb || ctlr->id == i82546eb) &&
1862             (pci_config_addr(ctlr->pci->bus, ctlr->pci->dev, 0, 0) ==
1863                  pci_config_addr(0, 1, 0, 0)))
1864                 ctlr->eeprom[Ea+2] += 0x100;            /* second interface */
1865         if(ctlr->id == i82541gi && ctlr->eeprom[Ea] == 0xFFFF)
1866                 ctlr->eeprom[Ea] = 0xD000;
1867         for(i = Ea; i < Eaddrlen/2; i++){
1868                 ctlr->ra[2*i] = ctlr->eeprom[i];
1869                 ctlr->ra[2*i+1] = ctlr->eeprom[i]>>8;
1870         }
1871         /* lan id seems to vary on 82543gc; don't use it */
1872         if (ctlr->id != i82543gc) {
1873                 r = (csr32r(ctlr, Status) & Lanid) >> 2;
1874                 ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1875         }
1876
1877         r = (ctlr->ra[3]<<24)|(ctlr->ra[2]<<16)|(ctlr->ra[1]<<8)|ctlr->ra[0];
1878         csr32w(ctlr, Ral, r);
1879         r = 0x80000000|(ctlr->ra[5]<<8)|ctlr->ra[4];
1880         csr32w(ctlr, Rah, r);
1881         for(i = 1; i < 16; i++){
1882                 csr32w(ctlr, Ral+i*8, 0);
1883                 csr32w(ctlr, Rah+i*8, 0);
1884         }
1885
1886         /*
1887          * Clear the Multicast Table Array.
1888          * It's a 4096 bit vector accessed as 128 32-bit registers.
1889          */
1890         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1891         for(i = 0; i < 128; i++)
1892                 csr32w(ctlr, Mta+i*4, 0);
1893
1894         /*
1895          * Just in case the Eerst didn't load the defaults
1896          * (doesn't appear to fully on the 82543GC), do it manually.
1897          */
1898         if (ctlr->id == i82543gc) {
1899                 txcw = csr32r(ctlr, Txcw);
1900                 txcw &= ~(TxcwAne|TxcwPauseMASK|TxcwFd);
1901                 ctrl = csr32r(ctlr, Ctrl);
1902                 ctrl &= ~(SwdpioloMASK|Frcspd|Ilos|Lrst|Fd);
1903
1904                 if(ctlr->eeprom[Icw1] & 0x0400){
1905                         ctrl |= Fd;
1906                         txcw |= TxcwFd;
1907                 }
1908                 if(ctlr->eeprom[Icw1] & 0x0200)
1909                         ctrl |= Lrst;
1910                 if(ctlr->eeprom[Icw1] & 0x0010)
1911                         ctrl |= Ilos;
1912                 if(ctlr->eeprom[Icw1] & 0x0800)
1913                         ctrl |= Frcspd;
1914                 swdpio = (ctlr->eeprom[Icw1] & 0x01E0)>>5;
1915                 ctrl |= swdpio<<SwdpioloSHIFT;
1916                 csr32w(ctlr, Ctrl, ctrl);
1917
1918                 ctrl = csr32r(ctlr, Ctrlext);
1919                 ctrl &= ~(Ips|SwdpiohiMASK);
1920                 swdpio = (ctlr->eeprom[Icw2] & 0x00F0)>>4;
1921                 if(ctlr->eeprom[Icw1] & 0x1000)
1922                         ctrl |= Ips;
1923                 ctrl |= swdpio<<SwdpiohiSHIFT;
1924                 csr32w(ctlr, Ctrlext, ctrl);
1925
1926                 if(ctlr->eeprom[Icw2] & 0x0800)
1927                         txcw |= TxcwAne;
1928                 pause = (ctlr->eeprom[Icw2] & 0x3000)>>12;
1929                 txcw |= pause<<TxcwPauseSHIFT;
1930                 switch(pause){
1931                 default:
1932                         ctlr->fcrtl = 0x00002000;
1933                         ctlr->fcrth = 0x00004000;
1934                         txcw |= TxcwAs|TxcwPs;
1935                         break;
1936                 case 0:
1937                         ctlr->fcrtl = 0x00002000;
1938                         ctlr->fcrth = 0x00004000;
1939                         break;
1940                 case 2:
1941                         ctlr->fcrtl = 0;
1942                         ctlr->fcrth = 0;
1943                         txcw |= TxcwAs;
1944                         break;
1945                 }
1946                 ctlr->txcw = txcw;
1947                 csr32w(ctlr, Txcw, txcw);
1948         }
1949
1950
1951         /*
1952          * Flow control - values from the datasheet.
1953          */
1954         csr32w(ctlr, Fcal, 0x00C28001);
1955         csr32w(ctlr, Fcah, 0x00000100);
1956         csr32w(ctlr, Fct, 0x00008808);
1957         csr32w(ctlr, Fcttv, 0x00000100);
1958
1959         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1960         csr32w(ctlr, Fcrth, ctlr->fcrth);
1961
1962         /* FYI, igbemii checks status right away too. */
1963         if(!(csr32r(ctlr, Status) & Tbimode) && igbemii(ctlr) < 0) {
1964                 printk("igbemii failed!  igbe failing to reset!\n");
1965                 return -1;
1966         }
1967
1968         return 0;
1969 }
1970
1971 static void
1972 igbepci(void)
1973 {
1974         int cls, id;
1975         struct pci_device *pcidev;
1976         struct ctlr *ctlr;
1977         void *mem;
1978         uintptr_t mmio_paddr;
1979
1980         STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
1981                 /* This checks that pcidev is a Network Controller for Ethernet */
1982                 if (pcidev->class != 0x02 || pcidev->subclass != 0x00)
1983                         continue;
1984                 id = pcidev->dev_id << 16 | pcidev->ven_id;
1985                 switch (id) {
1986                 default:
1987                         continue;
1988                 case i82543gc:
1989                 case i82544ei:
1990                 case i82544eif:
1991                 case i82544gc:
1992                 case i82547ei:
1993                 case i82547gi:
1994                 case i82540em:
1995                 case i82540eplp:
1996                 case i82541ei:
1997                 case i82541gi:
1998                 case i82541gi2:
1999                 case i82541pi:
2000                 case i82545em:
2001                 case i82545gmc:
2002                 case i82546gb:
2003                 case i82546eb:
2004                         break;
2005                 }
2006                 printk("igbe/e1000 driver found 0x%04x:%04x at %02x:%02x.%x\n",
2007                        pcidev->ven_id, pcidev->dev_id,
2008                        pcidev->bus, pcidev->dev, pcidev->func);
2009
2010                 mmio_paddr = pcidev->bar[0].mmio_base32 ? pcidev->bar[0].mmio_base32 : 
2011                                                           pcidev->bar[0].mmio_base64;
2012                 mem = (void*)vmap_pmem(mmio_paddr, pcidev->bar[0].mmio_sz);
2013                 if(mem == NULL){
2014                         printd("igbe: can't map %p\n", pcidev->bar[0].mmio_base32);
2015                         continue;
2016                 }
2017                 cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
2018                 switch(cls){
2019                         default:
2020                                 printd("igbe: unexpected CLS - %d\n", cls*4);
2021                                 break;
2022                         case 0x00:
2023                         case 0xFF:
2024                                 /* bogus value; use a sane default.  cls is set in DWORD (u32)
2025                                  * units. */
2026                                 cls = ARCH_CL_SIZE / sizeof(long);
2027                                 pcidev_write8(pcidev, PCI_CLSZ_REG, cls);
2028                                 break;
2029                         case 0x08:
2030                         case 0x10:
2031                                 break;
2032                 }
2033                 ctlr = kzmalloc(sizeof(struct ctlr), 0);
2034                 if(ctlr == NULL) {
2035                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2036                         error(Enomem);
2037                 }
2038                 spinlock_init_irqsave(&ctlr->imlock);
2039                 spinlock_init_irqsave(&ctlr->tlock);
2040                 qlock_init(&ctlr->alock);
2041                 qlock_init(&ctlr->slock);
2042                 rendez_init(&ctlr->lrendez);
2043                 rendez_init(&ctlr->rrendez);
2044                 /* port seems to be unused, and only used for some comparison with edev.
2045                  * plan9 just used the top of the raw bar, regardless of the type. */
2046                 ctlr->port = pcidev->bar[0].raw_bar & ~0x0f;
2047                 ctlr->pci = pcidev;
2048                 ctlr->id = id;
2049                 ctlr->cls = cls * sizeof(long);
2050                 ctlr->nic = mem;
2051
2052                 if(igbereset(ctlr)){
2053                         kfree(ctlr);
2054                         vunmap_vmem((uintptr_t)mem, pcidev->bar[0].mmio_sz);
2055                         continue;
2056                 }
2057                 pci_set_bus_master(pcidev);
2058
2059                 if(igbectlrhead != NULL)
2060                         igbectlrtail->next = ctlr;
2061                 else
2062                         igbectlrhead = ctlr;
2063                 igbectlrtail = ctlr;
2064         }
2065 }
2066
2067 static int
2068 igbepnp(struct ether* edev)
2069 {
2070         struct ctlr *ctlr;
2071
2072         run_once(igbepci());
2073
2074         /*
2075          * Any adapter matches if no edev->port is supplied,
2076          * otherwise the ports must match.
2077          */
2078         for(ctlr = igbectlrhead; ctlr != NULL; ctlr = ctlr->next){
2079                 if(ctlr->active)
2080                         continue;
2081                 if(edev->port == 0 || edev->port == ctlr->port){
2082                         ctlr->active = 1;
2083                         break;
2084                 }
2085         }
2086         if(ctlr == NULL)
2087                 return -1;
2088
2089         edev->ctlr = ctlr;
2090         edev->port = ctlr->port;
2091         edev->irq = ctlr->pci->irqline;
2092         edev->netif.mbps = 1000;
2093         memmove(edev->ea, ctlr->ra, Eaddrlen);
2094
2095         /*
2096          * Linkage to the generic ethernet driver.
2097          */
2098         edev->tbdf = MKBUS(BusPCI, ctlr->pci->bus, ctlr->pci->dev,
2099                            ctlr->pci->func);
2100         edev->attach = igbeattach;
2101         edev->transmit = igbetransmit;
2102         edev->interrupt = igbeinterrupt;
2103         edev->ifstat = igbeifstat;
2104         edev->ctl = igbectl;
2105         edev->shutdown = igbeshutdown;
2106
2107         edev->netif.arg = edev;
2108         edev->netif.promiscuous = igbepromiscuous;
2109         edev->netif.multicast = igbemulticast;
2110
2111         return 0;
2112 }
2113
2114 linker_func_3(etherigbelink)
2115 {
2116         addethercard("i82543", igbepnp);
2117         addethercard("igbe", igbepnp);
2118 }