442de355764fea80fdbd7a5696ddbb92b8c77648
[akaros.git] / kern / drivers / net / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  *
9  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
10  * Written by: Eliezer Tamir
11  * Based on code from Michael Chan's bnx2 driver
12  * UDP CSUM errata workaround by Arik Gendelman
13  * Slowpath and fastpath rework by Vladislav Zolotarov
14  * Statistics and Link management by Yitchak Gertner
15  *
16  */
17
18 #include "akaros_compat.h"
19
20 #include "bnx2x_cmn.h"
21 #include "bnx2x_init.h"
22 #include "bnx2x_sp.h"
23
24 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
25 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
26 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
27 static void bnx2x_poll(uint32_t srcid, long a0, long a1, long a2);
28
29 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
30 {
31         int i;
32
33         /* Add NAPI objects */
34         for_each_rx_queue_cnic(bp, i) {
35                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
36                                bnx2x_poll, NAPI_POLL_WEIGHT);
37                 napi_hash_add(&bnx2x_fp(bp, i, napi));
38         }
39 }
40
41 static void bnx2x_add_all_napi(struct bnx2x *bp)
42 {
43         int i;
44
45         /* Add NAPI objects */
46         for_each_eth_queue(bp, i) {
47                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
48                                bnx2x_poll, NAPI_POLL_WEIGHT);
49                 napi_hash_add(&bnx2x_fp(bp, i, napi));
50         }
51 }
52
53 static int bnx2x_calc_num_queues(struct bnx2x *bp)
54 {
55         /* default is min(8, num_cpus) in Linux.  we'll set it elsewhere */
56         int nq = bnx2x_num_queues ? : 8;
57
58         /* Reduce memory usage in kdump environment by using only one queue */
59         if (is_kdump_kernel())
60                 nq = 1;
61
62         nq = CLAMP(nq, 1, BNX2X_MAX_QUEUES(bp));
63         return nq;
64 }
65
66 /**
67  * bnx2x_move_fp - move content of the fastpath structure.
68  *
69  * @bp:         driver handle
70  * @from:       source FP index
71  * @to:         destination FP index
72  *
73  * Makes sure the contents of the bp->fp[to].napi is kept
74  * intact. This is done by first copying the napi struct from
75  * the target to the source, and then mem copying the entire
76  * source onto the target. Update txdata pointers and related
77  * content.
78  */
79 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
80 {
81         struct bnx2x_fastpath *from_fp = &bp->fp[from];
82         struct bnx2x_fastpath *to_fp = &bp->fp[to];
83         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
84         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
85         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
86         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
87         int old_max_eth_txqs, new_max_eth_txqs;
88         int old_txdata_index = 0, new_txdata_index = 0;
89         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
90
91         /* Copy the NAPI object as it has been already initialized */
92         from_fp->napi = to_fp->napi;
93
94         /* Move bnx2x_fastpath contents */
95         memcpy(to_fp, from_fp, sizeof(*to_fp));
96         to_fp->index = to;
97
98         /* Retain the tpa_info of the original `to' version as we don't want
99          * 2 FPs to contain the same tpa_info pointer.
100          */
101         to_fp->tpa_info = old_tpa_info;
102
103         /* move sp_objs contents as well, as their indices match fp ones */
104         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
105
106         /* move fp_stats contents as well, as their indices match fp ones */
107         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
108
109         /* Update txdata pointers in fp and move txdata content accordingly:
110          * Each fp consumes 'max_cos' txdata structures, so the index should be
111          * decremented by max_cos x delta.
112          */
113
114         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
115         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
116                                 (bp)->max_cos;
117         if (from == FCOE_IDX(bp)) {
118                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
119                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
120         }
121
122         memcpy(&bp->bnx2x_txq[new_txdata_index],
123                &bp->bnx2x_txq[old_txdata_index],
124                sizeof(struct bnx2x_fp_txdata));
125         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
126 }
127
128 /**
129  * bnx2x_fill_fw_str - Fill buffer with FW version string.
130  *
131  * @bp:        driver handle
132  * @buf:       character buffer to fill with the fw name
133  * @buf_len:   length of the above buffer
134  *
135  */
136 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
137 {
138         if (IS_PF(bp)) {
139                 uint8_t phy_fw_ver[PHY_FW_VER_LEN];
140
141                 phy_fw_ver[0] = '\0';
142                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
143                                              phy_fw_ver, PHY_FW_VER_LEN);
144                 strlcpy(buf, bp->fw_ver, buf_len);
145                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
146                          "bc %d.%d.%d%s%s",
147                          (bp->common.bc_ver & 0xff0000) >> 16,
148                          (bp->common.bc_ver & 0xff00) >> 8,
149                          (bp->common.bc_ver & 0xff),
150                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
151         } else {
152                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
153         }
154 }
155
156 /**
157  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
158  *
159  * @bp: driver handle
160  * @delta:      number of eth queues which were not allocated
161  */
162 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
163 {
164         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
165
166         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
167          * backward along the array could cause memory to be overridden
168          */
169         for (cos = 1; cos < bp->max_cos; cos++) {
170                 for (i = 0; i < old_eth_num - delta; i++) {
171                         struct bnx2x_fastpath *fp = &bp->fp[i];
172                         int new_idx = cos * (old_eth_num - delta) + i;
173
174                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
175                                sizeof(struct bnx2x_fp_txdata));
176                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
177                 }
178         }
179 }
180
181 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
182
183 /* free skb in the packet ring at pos idx
184  * return idx of last bd freed
185  */
186 static uint16_t bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
187                              uint16_t idx, unsigned int *pkts_compl,
188                              unsigned int *bytes_compl)
189 {
190         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
191         struct eth_tx_start_bd *tx_start_bd;
192         struct eth_tx_bd *tx_data_bd;
193         struct block *block = tx_buf->block;
194         uint16_t bd_idx = TX_BD(tx_buf->first_bd), new_cons;
195         int nbd;
196         uint16_t split_bd_len = 0;
197
198         /* prefetch skb end pointer to speedup dev_kfree_skb() */
199         //prefetch(&skb->end); // AKAROS_PORT
200
201         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->block %p\n",
202            txdata->txq_index, idx, tx_buf, block);
203
204         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
205
206         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
207 #ifdef BNX2X_STOP_ON_ERROR
208         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
209                 BNX2X_ERR("BAD nbd!\n");
210                 bnx2x_panic();
211         }
212 #endif
213         new_cons = nbd + tx_buf->first_bd;
214
215         /* Get the next bd */
216         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
217
218         /* Skip a parse bd... */
219         --nbd;
220         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
221
222         if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
223                 /* Skip second parse bd... */
224                 --nbd;
225                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
226         }
227
228         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
229         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
230                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
231                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
232                 --nbd;
233                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
234         }
235
236         /* unmap first bd */
237         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
238                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
239                          DMA_TO_DEVICE);
240
241         /* now free frags */
242         while (nbd > 0) {
243
244                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
245                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
246                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
247                 if (--nbd)
248                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
249         }
250
251         /* release block */
252         warn_on(!block);
253         if (likely(block)) {
254                 (*pkts_compl)++;
255                 (*bytes_compl) += BLEN(block);
256         }
257
258         freeb(block);
259         tx_buf->first_bd = 0;
260         tx_buf->block = NULL;
261
262         return new_cons;
263 }
264
265 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
266 {
267         uint16_t hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
268         unsigned int pkts_compl = 0, bytes_compl = 0;
269
270 #ifdef BNX2X_STOP_ON_ERROR
271         if (unlikely(bp->panic))
272                 return -1;
273 #endif
274
275         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
276         sw_cons = txdata->tx_pkt_cons;
277
278         while (sw_cons != hw_cons) {
279                 uint16_t pkt_cons;
280
281                 pkt_cons = TX_BD(sw_cons);
282
283                 DP(NETIF_MSG_TX_DONE,
284                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
285                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
286
287                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
288                                             &pkts_compl, &bytes_compl);
289
290                 sw_cons++;
291         }
292
293         txdata->tx_pkt_cons = sw_cons;
294         txdata->tx_bd_cons = bd_cons;
295
296         poke(&txdata->poker, txdata);
297         return 0;
298 }
299
300 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
301                                              uint16_t idx)
302 {
303         uint16_t last_max = fp->last_max_sge;
304
305         if (SUB_S16(idx, last_max) > 0)
306                 fp->last_max_sge = idx;
307 }
308
309 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
310                                          uint16_t sge_len,
311                                          struct eth_end_agg_rx_cqe *cqe)
312 {
313         struct bnx2x *bp = fp->bp;
314         uint16_t last_max, last_elem, first_elem;
315         uint16_t delta = 0;
316         uint16_t i;
317
318         if (!sge_len)
319                 return;
320
321         /* First mark all used pages */
322         for (i = 0; i < sge_len; i++)
323                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
324                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
325
326         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
327            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
328
329         /* Here we assume that the last SGE index is the biggest */
330         prefetch((void *)(fp->sge_mask));
331         bnx2x_update_last_max_sge(fp,
332                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
333
334         last_max = RX_SGE(fp->last_max_sge);
335         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
336         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
337
338         /* If ring is not full */
339         if (last_elem + 1 != first_elem)
340                 last_elem++;
341
342         /* Now update the prod */
343         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
344                 if (likely(fp->sge_mask[i]))
345                         break;
346
347                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
348                 delta += BIT_VEC64_ELEM_SZ;
349         }
350
351         if (delta > 0) {
352                 fp->rx_sge_prod += delta;
353                 /* clear page-end entries */
354                 bnx2x_clear_sge_mask_next_elems(fp);
355         }
356
357         DP(NETIF_MSG_RX_STATUS,
358            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
359            fp->last_max_sge, fp->rx_sge_prod);
360 }
361
362 /* Get Toeplitz hash value in the skb using the value from the
363  * CQE (calculated by HW).
364  */
365 static uint32_t bnx2x_get_rxhash(const struct bnx2x *bp,
366                             const struct eth_fast_path_rx_cqe *cqe,
367                             enum pkt_hash_types *rxhash_type)
368 {
369 panic("Not implemented");
370 #if 0 // AKAROS_PORT
371         /* Get Toeplitz hash from CQE */
372         if ((bp->dev->feat & NETIF_F_RXHASH) &&
373             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
374                 enum eth_rss_hash_type htype;
375
376                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
377                 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
378                                 (htype == TCP_IPV6_HASH_TYPE)) ?
379                                PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
380
381                 return le32_to_cpu(cqe->rss_hash_result);
382         }
383         *rxhash_type = PKT_HASH_TYPE_NONE;
384         return 0;
385 #endif
386 }
387
388 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, uint16_t queue,
389                             uint16_t cons, uint16_t prod,
390                             struct eth_fast_path_rx_cqe *cqe)
391 {
392 panic("Not implemented");
393 #if 0 // AKAROS_PORT
394         struct bnx2x *bp = fp->bp;
395         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
396         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
397         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
398         dma_addr_t mapping;
399         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
400         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
401
402         /* print error if current state != stop */
403         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
404                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
405
406         /* Try to map an empty data buffer from the aggregation info  */
407         mapping = dma_map_single(&bp->pdev->dev,
408                                  first_buf->data + NET_SKB_PAD,
409                                  fp->rx_buf_size, DMA_FROM_DEVICE);
410         /*
411          *  ...if it fails - move the skb from the consumer to the producer
412          *  and set the current aggregation state as ERROR to drop it
413          *  when TPA_STOP arrives.
414          */
415
416         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
417                 /* Move the BD from the consumer to the producer */
418                 bnx2x_reuse_rx_data(fp, cons, prod);
419                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
420                 return;
421         }
422
423         /* move empty data from pool to prod */
424         prod_rx_buf->data = first_buf->data;
425         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
426         /* point prod_bd to new data */
427         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
428         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
429
430         /* move partial skb from cons to pool (don't unmap yet) */
431         *first_buf = *cons_rx_buf;
432
433         /* mark bin state as START */
434         tpa_info->parsing_flags =
435                 le16_to_cpu(cqe->pars_flags.flags);
436         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
437         tpa_info->tpa_state = BNX2X_TPA_START;
438         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
439         tpa_info->placement_offset = cqe->placement_offset;
440         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
441         if (fp->mode == TPA_MODE_GRO) {
442                 uint16_t gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
443                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
444                 tpa_info->gro_size = gro_size;
445         }
446
447 #ifdef BNX2X_STOP_ON_ERROR
448         fp->tpa_queue_used |= (1 << queue);
449         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
450            fp->tpa_queue_used);
451 #endif
452 #endif
453 }
454
455 /* Timestamp option length allowed for TPA aggregation:
456  *
457  *              nop nop kind length echo val
458  */
459 #define TPA_TSTAMP_OPT_LEN      12
460 /**
461  * bnx2x_set_gro_params - compute GRO values
462  *
463  * @skb:                packet skb
464  * @parsing_flags:      parsing flags from the START CQE
465  * @len_on_bd:          total length of the first packet for the
466  *                      aggregation.
467  * @pkt_len:            length of all segments
468  *
469  * Approximate value of the MSS for this aggregation calculated using
470  * the first packet of it.
471  * Compute number of aggregated segments, and gso_type.
472  */
473 static void bnx2x_set_gro_params(struct sk_buff *skb, uint16_t parsing_flags,
474                                  uint16_t len_on_bd, unsigned int pkt_len,
475                                  uint16_t num_of_coalesced_segs)
476 {
477 panic("Not implemented");
478 #if 0 // AKAROS_PORT
479         /* TPA aggregation won't have either IP options or TCP options
480          * other than timestamp or IPv6 extension headers.
481          */
482         uint16_t hdrs_len = ETHERHDRSIZE + sizeof(struct tcphdr);
483
484         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
485             PRS_FLAG_OVERETH_IPV6) {
486                 hdrs_len += sizeof(struct ipv6hdr);
487                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
488         } else {
489                 hdrs_len += sizeof(struct iphdr);
490                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
491         }
492
493         /* Check if there was a TCP timestamp, if there is it's will
494          * always be 12 bytes length: nop nop kind length echo val.
495          *
496          * Otherwise FW would close the aggregation.
497          */
498         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
499                 hdrs_len += TPA_TSTAMP_OPT_LEN;
500
501         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
502
503         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
504          * to skb_shinfo(skb)->gso_segs
505          */
506         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
507 #endif
508 }
509
510 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
511                               uint16_t index, gfp_t gfp_mask)
512 {
513         /* AKAROS_PORT: our get_cont_pages returns KVAs, not struct page * */
514         struct page *page = kva2page(get_cont_pages(PAGES_PER_SGE_SHIFT, gfp_mask));
515         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
516         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
517         dma_addr_t mapping;
518
519         if (unlikely(page == NULL)) {
520                 BNX2X_ERR("Can't alloc sge\n");
521                 return -ENOMEM;
522         }
523
524         mapping = dma_map_page(&bp->pdev->dev, page, 0,
525                                SGE_PAGES, DMA_FROM_DEVICE);
526         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
527                 free_cont_pages(page2kva(page), PAGES_PER_SGE_SHIFT);
528                 BNX2X_ERR("Can't map sge\n");
529                 return -ENOMEM;
530         }
531
532         sw_buf->page = page;
533         dma_unmap_addr_set(sw_buf, mapping, mapping);
534
535         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
536         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
537
538         return 0;
539 }
540
541 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
542                                struct bnx2x_agg_info *tpa_info,
543                                uint16_t pages,
544                                struct sk_buff *skb,
545                                struct eth_end_agg_rx_cqe *cqe,
546                                uint16_t cqe_idx)
547 {
548 panic("Not implemented");
549 #if 0 // AKAROS_PORT
550         struct sw_rx_page *rx_pg, old_rx_pg;
551         uint32_t i, frag_len, frag_size;
552         int err, j, frag_id = 0;
553         uint16_t len_on_bd = tpa_info->len_on_bd;
554         uint16_t full_page = 0, gro_size = 0;
555
556         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
557
558         if (fp->mode == TPA_MODE_GRO) {
559                 gro_size = tpa_info->gro_size;
560                 full_page = tpa_info->full_page;
561         }
562
563         /* This is needed in order to enable forwarding support */
564         if (frag_size)
565                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
566                                      le16_to_cpu(cqe->pkt_len),
567                                      le16_to_cpu(cqe->num_of_coalesced_segs));
568
569 #ifdef BNX2X_STOP_ON_ERROR
570         if (pages > MIN_T(uint32_t, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
571                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
572                           pages, cqe_idx);
573                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
574                 bnx2x_panic();
575                 return -EINVAL;
576         }
577 #endif
578
579         /* Run through the SGL and compose the fragmented skb */
580         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
581                 uint16_t sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
582
583                 /* FW gives the indices of the SGE as if the ring is an array
584                    (meaning that "next" element will consume 2 indices) */
585                 if (fp->mode == TPA_MODE_GRO)
586                         frag_len = MIN_T(uint32_t, frag_size,
587                                          (uint32_t)full_page);
588                 else /* LRO */
589                         frag_len = MIN_T(uint32_t, frag_size,
590                                          (uint32_t)SGE_PAGES);
591
592                 rx_pg = &fp->rx_page_ring[sge_idx];
593                 old_rx_pg = *rx_pg;
594
595                 /* If we fail to allocate a substitute page, we simply stop
596                    where we are and drop the whole packet */
597                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, 0);
598                 if (unlikely(err)) {
599                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
600                         return err;
601                 }
602
603                 /* Unmap the page as we're going to pass it to the stack */
604                 dma_unmap_page(&bp->pdev->dev,
605                                dma_unmap_addr(&old_rx_pg, mapping),
606                                SGE_PAGES, DMA_FROM_DEVICE);
607                 /* Add one frag and update the appropriate fields in the skb */
608                 if (fp->mode == TPA_MODE_LRO)
609                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
610                 else { /* GRO */
611                         int rem;
612                         int offset = 0;
613                         for (rem = frag_len; rem > 0; rem -= gro_size) {
614                                 int len = rem > gro_size ? gro_size : rem;
615                                 skb_fill_page_desc(skb, frag_id++,
616                                                    old_rx_pg.page, offset, len);
617                                 if (offset)
618                                         page_incref(old_rx_pg.page);
619                                 offset += len;
620                         }
621                 }
622
623                 skb->data_len += frag_len;
624                 skb->truesize += SGE_PAGES;
625                 skb->len += frag_len;
626
627                 frag_size -= frag_len;
628         }
629
630         return 0;
631 #endif
632 }
633
634 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
635 {
636         if (fp->rx_frag_size)
637                 page_decref(kva2page(data));
638         else
639                 kfree(data);
640 }
641
642 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
643 {
644         if (fp->rx_frag_size) {
645                 /* GFP_KERNEL allocations are used only during initialization */
646                 if (unlikely(gfp_mask & KMALLOC_WAIT))
647                         return (void *)kpage_alloc_addr();
648
649 #if 0 // AKAROS_PORT
650                 return netdev_alloc_frag(fp->rx_frag_size);
651 #else
652                 return (void *)kpage_alloc_addr();
653 #endif
654         }
655
656         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
657 }
658
659 #ifdef CONFIG_INET
660 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
661 {
662         const struct iphdr *iph = ip_hdr(skb);
663         struct tcphdr *th;
664
665         skb_set_transport_header(skb, sizeof(struct iphdr));
666         th = tcp_hdr(skb);
667
668         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
669                                   iph->saddr, iph->daddr, 0);
670 }
671
672 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
673 {
674         struct ipv6hdr *iph = ipv6_hdr(skb);
675         struct tcphdr *th;
676
677         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
678         th = tcp_hdr(skb);
679
680         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
681                                   &iph->saddr, &iph->daddr, 0);
682 }
683
684 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
685                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
686 {
687         skb_set_network_header(skb, 0);
688         gro_func(bp, skb);
689         tcp_gro_complete(skb);
690 }
691 #endif
692
693 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
694                                struct sk_buff *skb)
695 {
696 panic("Not implemented");
697 #if 0 // AKAROS_PORT
698 #ifdef CONFIG_INET
699         if (skb_shinfo(skb)->gso_size) {
700                 switch (be16_to_cpu(skb->protocol)) {
701                 case ETH_P_IP:
702                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
703                         break;
704                 case ETH_P_IPV6:
705                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
706                         break;
707                 default:
708                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
709                                   be16_to_cpu(skb->protocol));
710                 }
711         }
712 #endif
713         skb_record_rx_queue(skb, fp->rx_queue);
714         napi_gro_receive(&fp->napi, skb);
715 #endif
716 }
717
718 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
719                            struct bnx2x_agg_info *tpa_info,
720                            uint16_t pages,
721                            struct eth_end_agg_rx_cqe *cqe,
722                            uint16_t cqe_idx)
723 {
724 panic("Not implemented");
725 #if 0 // AKAROS_PORT
726         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
727         uint8_t pad = tpa_info->placement_offset;
728         uint16_t len = tpa_info->len_on_bd;
729         struct sk_buff *skb = NULL;
730         uint8_t *new_data, *data = rx_buf->data;
731         uint8_t old_tpa_state = tpa_info->tpa_state;
732
733         tpa_info->tpa_state = BNX2X_TPA_STOP;
734
735         /* If we there was an error during the handling of the TPA_START -
736          * drop this aggregation.
737          */
738         if (old_tpa_state == BNX2X_TPA_ERROR)
739                 goto drop;
740
741         /* Try to allocate the new data */
742         new_data = bnx2x_frag_alloc(fp, 0);
743         /* Unmap skb in the pool anyway, as we are going to change
744            pool entry status to BNX2X_TPA_STOP even if new skb allocation
745            fails. */
746         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
747                          fp->rx_buf_size, DMA_FROM_DEVICE);
748         if (likely(new_data))
749                 skb = build_skb(data, fp->rx_frag_size);
750
751         if (likely(skb)) {
752 #ifdef BNX2X_STOP_ON_ERROR
753                 if (pad + len > fp->rx_buf_size) {
754                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
755                                   pad, len, fp->rx_buf_size);
756                         bnx2x_panic();
757                         return;
758                 }
759 #endif
760
761                 skb_reserve(skb, pad + NET_SKB_PAD);
762                 skb_put(skb, len);
763                 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
764
765                 skb->protocol = eth_type_trans(skb, bp->dev);
766                 skb->ip_summed = CHECKSUM_UNNECESSARY;
767
768                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
769                                          skb, cqe, cqe_idx)) {
770                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
771                                 __vlan_hwaccel_put_tag(skb,
772                                                        cpu_to_be16(ETH_P_8021Q),
773                                                        tpa_info->vlan_tag);
774                         bnx2x_gro_receive(bp, fp, skb);
775                 } else {
776                         DP(NETIF_MSG_RX_STATUS,
777                            "Failed to allocate new pages - dropping packet!\n");
778                         dev_kfree_skb_any(skb);
779                 }
780
781                 /* put new data in bin */
782                 rx_buf->data = new_data;
783
784                 return;
785         }
786         if (new_data)
787                 bnx2x_frag_free(fp, new_data);
788 drop:
789         /* drop the packet and keep the buffer in the bin */
790         DP(NETIF_MSG_RX_STATUS,
791            "Failed to allocate or map a new skb - dropping packet!\n");
792         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
793 #endif
794 }
795
796 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
797                                uint16_t index, gfp_t gfp_mask)
798 {
799         uint8_t *data;
800         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
801         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
802         dma_addr_t mapping;
803
804         data = bnx2x_frag_alloc(fp, gfp_mask);
805         if (unlikely(data == NULL))
806                 return -ENOMEM;
807
808         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
809                                  fp->rx_buf_size,
810                                  DMA_FROM_DEVICE);
811         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
812                 bnx2x_frag_free(fp, data);
813                 BNX2X_ERR("Can't map rx data\n");
814                 return -ENOMEM;
815         }
816
817         rx_buf->data = data;
818         dma_unmap_addr_set(rx_buf, mapping, mapping);
819
820         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
821         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
822
823         return 0;
824 }
825
826 static
827 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
828                                  struct bnx2x_fastpath *fp,
829                                  struct bnx2x_eth_q_stats *qstats)
830 {
831 panic("Not implemented");
832 #if 0 // AKAROS_PORT
833         /* Do nothing if no L4 csum validation was done.
834          * We do not check whether IP csum was validated. For IPv4 we assume
835          * that if the card got as far as validating the L4 csum, it also
836          * validated the IP csum. IPv6 has no IP csum.
837          */
838         if (cqe->fast_path_cqe.status_flags &
839             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
840                 return;
841
842         /* If L4 validation was done, check if an error was found. */
843
844         if (cqe->fast_path_cqe.type_error_flags &
845             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
846              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
847                 qstats->hw_csum_err++;
848         else
849                 skb->ip_summed = CHECKSUM_UNNECESSARY;
850 #endif
851 }
852
853 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
854 {
855         struct bnx2x *bp = fp->bp;
856         uint16_t bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
857         uint16_t sw_comp_cons, sw_comp_prod;
858         int rx_pkt = 0;
859         union eth_rx_cqe *cqe;
860         struct eth_fast_path_rx_cqe *cqe_fp;
861
862         struct block *block;
863
864 #ifdef BNX2X_STOP_ON_ERROR
865         if (unlikely(bp->panic))
866                 return 0;
867 #endif
868         if (budget <= 0)
869                 return rx_pkt;
870
871         bd_cons = fp->rx_bd_cons;
872         bd_prod = fp->rx_bd_prod;
873         bd_prod_fw = bd_prod;
874         sw_comp_cons = fp->rx_comp_cons;
875         sw_comp_prod = fp->rx_comp_prod;
876
877         comp_ring_cons = RCQ_BD(sw_comp_cons);
878         cqe = &fp->rx_comp_ring[comp_ring_cons];
879         cqe_fp = &cqe->fast_path_cqe;
880
881         DP(NETIF_MSG_RX_STATUS,
882            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
883
884         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
885                 struct sw_rx_bd *rx_buf = NULL;
886                 uint8_t cqe_fp_flags;
887                 enum eth_rx_cqe_type cqe_fp_type;
888                 uint16_t len, pad, queue;
889                 uint8_t *data;
890                 uint32_t rxhash;
891
892 #ifdef BNX2X_STOP_ON_ERROR
893                 if (unlikely(bp->panic))
894                         return 0;
895 #endif
896
897                 bd_prod = RX_BD(bd_prod);
898                 bd_cons = RX_BD(bd_cons);
899
900                 /* A rmb() is required to ensure that the CQE is not read
901                  * before it is written by the adapter DMA.  PCI ordering
902                  * rules will make sure the other fields are written before
903                  * the marker at the end of struct eth_fast_path_rx_cqe
904                  * but without rmb() a weakly ordered processor can process
905                  * stale data.  Without the barrier TPA state-machine might
906                  * enter inconsistent state and kernel stack might be
907                  * provided with incorrect packet description - these lead
908                  * to various kernel crashed.
909                  */
910                 rmb();
911
912                 cqe_fp_flags = cqe_fp->type_error_flags;
913                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
914
915                 DP(NETIF_MSG_RX_STATUS,
916                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
917                    CQE_TYPE(cqe_fp_flags),
918                    cqe_fp_flags, cqe_fp->status_flags,
919                    le32_to_cpu(cqe_fp->rss_hash_result),
920                    le16_to_cpu(cqe_fp->vlan_tag),
921                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
922
923                 /* is this a slowpath msg? */
924                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
925                         bnx2x_sp_event(fp, cqe);
926                         goto next_cqe;
927                 }
928
929                 rx_buf = &fp->rx_buf_ring[bd_cons];
930                 data = rx_buf->data;
931
932                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
933                         struct bnx2x_agg_info *tpa_info;
934                         uint16_t frag_size, pages;
935 #ifdef BNX2X_STOP_ON_ERROR
936                         /* sanity check */
937                         if (fp->disable_tpa &&
938                             (CQE_TYPE_START(cqe_fp_type) ||
939                              CQE_TYPE_STOP(cqe_fp_type)))
940                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
941                                           CQE_TYPE(cqe_fp_type));
942 #endif
943
944                         if (CQE_TYPE_START(cqe_fp_type)) {
945                                 uint16_t queue = cqe_fp->queue_index;
946                                 DP(NETIF_MSG_RX_STATUS,
947                                    "calling tpa_start on queue %d\n",
948                                    queue);
949
950                                 bnx2x_tpa_start(fp, queue,
951                                                 bd_cons, bd_prod,
952                                                 cqe_fp);
953
954                                 goto next_rx;
955                         }
956                         queue = cqe->end_agg_cqe.queue_index;
957                         tpa_info = &fp->tpa_info[queue];
958                         DP(NETIF_MSG_RX_STATUS,
959                            "calling tpa_stop on queue %d\n",
960                            queue);
961
962                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
963                                     tpa_info->len_on_bd;
964
965                         if (fp->mode == TPA_MODE_GRO)
966                                 pages = (frag_size + tpa_info->full_page - 1) /
967                                          tpa_info->full_page;
968                         else
969                                 pages = SGE_PAGE_ALIGN(frag_size) >>
970                                         SGE_PAGE_SHIFT;
971
972                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
973                                        &cqe->end_agg_cqe, comp_ring_cons);
974 #ifdef BNX2X_STOP_ON_ERROR
975                         if (bp->panic)
976                                 return 0;
977 #endif
978
979                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
980                         goto next_cqe;
981                 }
982                 /* non TPA */
983                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
984                 pad = cqe_fp->placement_offset;
985                 dma_sync_single_for_cpu(&bp->pdev->dev,
986                                         dma_unmap_addr(rx_buf, mapping),
987                                         pad + RX_COPY_THRESH,
988                                         DMA_FROM_DEVICE);
989                 pad += NET_SKB_PAD;
990                 prefetch(data + pad); /* speedup eth_type_trans() */
991                 /* is this an error packet? */
992                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
993                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
994                            "ERROR  flags %x  rx packet %u\n",
995                            cqe_fp_flags, sw_comp_cons);
996                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
997                         goto reuse_rx;
998                 }
999
1000                 /* Since we don't have a jumbo ring
1001                  * copy small packets if mtu > 1500
1002                  */
1003                 /* TODO: AKAROS_PORT always copy out the packet for now. */
1004                 if (1) {
1005 //              if ((bp->dev->maxmtu > ETH_MAX_PACKET_SIZE) &&
1006 //                  (len <= RX_COPY_THRESH)) {
1007                         block = iallocb(len);
1008                         if (block == NULL) {
1009                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1010                                    "ERROR  packet dropped because of alloc failure\n");
1011                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1012                                 goto reuse_rx;
1013                         }
1014                         memcpy(block->wp, data + pad, len);
1015                         block->wp += len;
1016                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1017                 } else {
1018                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1019                                                        0) == 0)) {
1020                                 dma_unmap_single(&bp->pdev->dev,
1021                                                  dma_unmap_addr(rx_buf, mapping),
1022                                                  fp->rx_buf_size,
1023                                                  DMA_FROM_DEVICE);
1024                                 /* TODO: block extra data here */
1025                                 panic("Extra-data not implemented");
1026                                 #if 0 // AKAROS_PORT
1027                                 skb = build_skb(data, fp->rx_frag_size);
1028                                 if (unlikely(!skb)) {
1029                                         bnx2x_frag_free(fp, data);
1030                                         bnx2x_fp_qstats(bp, fp)->
1031                                                         rx_skb_alloc_failed++;
1032                                         goto next_rx;
1033                                 }
1034                                 skb_reserve(skb, pad);
1035                                 #endif
1036                         } else {
1037                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1038                                    "ERROR  packet dropped because of alloc failure\n");
1039                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1040 reuse_rx:
1041                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1042                                 goto next_rx;
1043                         }
1044                 }
1045
1046                 // AKAROS_PORT TODO: set hash and checksum stuff
1047 #if 0
1048                 skb_put(skb, len);
1049                 skb->protocol = eth_type_trans(skb, bp->dev);
1050
1051                 /* Set Toeplitz hash for a none-LRO skb */
1052                 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1053                 skb_set_hash(skb, rxhash, rxhash_type);
1054
1055                 skb_checksum_none_assert(skb);
1056
1057                 if (bp->dev->feat & NETIF_F_RXCSUM)
1058                         bnx2x_csum_validate(skb, cqe, fp,
1059                                             bnx2x_fp_qstats(bp, fp));
1060
1061                 skb_record_rx_queue(skb, fp->rx_queue);
1062
1063                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1064                     PARSING_FLAGS_VLAN)
1065                         __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
1066                                                le16_to_cpu(cqe_fp->vlan_tag));
1067
1068                 skb_mark_napi_id(skb, &fp->napi);
1069
1070                 if (bnx2x_fp_ll_polling(fp))
1071                         netif_receive_skb(skb);
1072                 else
1073                         napi_gro_receive(&fp->napi, skb);
1074 #endif
1075                 etheriq(bp->edev, block, TRUE);
1076 next_rx:
1077                 rx_buf->data = NULL;
1078
1079                 bd_cons = NEXT_RX_IDX(bd_cons);
1080                 bd_prod = NEXT_RX_IDX(bd_prod);
1081                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1082                 rx_pkt++;
1083 next_cqe:
1084                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1085                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1086
1087                 /* mark CQE as free */
1088                 BNX2X_SEED_CQE(cqe_fp);
1089
1090                 if (rx_pkt == budget)
1091                         break;
1092
1093                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1094                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1095                 cqe_fp = &cqe->fast_path_cqe;
1096         } /* while */
1097
1098         fp->rx_bd_cons = bd_cons;
1099         fp->rx_bd_prod = bd_prod_fw;
1100         fp->rx_comp_cons = sw_comp_cons;
1101         fp->rx_comp_prod = sw_comp_prod;
1102
1103         /* Update producers */
1104         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1105                              fp->rx_sge_prod);
1106
1107         fp->rx_pkt += rx_pkt;
1108         fp->rx_calls++;
1109
1110         return rx_pkt;
1111 }
1112
1113 static void bnx2x_msix_fp_int(struct hw_trapframe *hw_tf, void *fp_cookie)
1114 {
1115         struct bnx2x_fastpath *fp = fp_cookie;
1116         struct bnx2x *bp = fp->bp;
1117         uint8_t cos;
1118
1119         DP(NETIF_MSG_INTR,
1120            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1121            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1122
1123         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1124
1125 #ifdef BNX2X_STOP_ON_ERROR
1126         if (unlikely(bp->panic))
1127                 return;
1128 #endif
1129
1130         /* Handle Rx and Tx according to MSI-X vector */
1131         for_each_cos_in_tx_queue(fp, cos)
1132                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1133
1134         prefetch(&fp->sb_running_index[SM_RX_ID]);
1135         // AKAROS_PORT
1136         send_kernel_message(core_id(), bnx2x_poll, (long)fp, 0, 0, KMSG_ROUTINE);
1137         napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1138
1139         return;
1140 }
1141
1142 /* HW Lock for shared dual port PHYs */
1143 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1144 {
1145         qlock(&bp->port.phy_mutex);
1146
1147         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1148 }
1149
1150 void bnx2x_release_phy_lock(struct bnx2x *bp)
1151 {
1152         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1153
1154         qunlock(&bp->port.phy_mutex);
1155 }
1156
1157 /* calculates MF speed according to current linespeed and MF configuration */
1158 uint16_t bnx2x_get_mf_speed(struct bnx2x *bp)
1159 {
1160         uint16_t line_speed = bp->link_vars.line_speed;
1161         if (IS_MF(bp)) {
1162                 uint16_t maxCfg = bnx2x_extract_max_cfg(bp,
1163                                                    bp->mf_config[BP_VN(bp)]);
1164
1165                 /* Calculate the current MAX line speed limit for the MF
1166                  * devices
1167                  */
1168                 if (IS_MF_SI(bp))
1169                         line_speed = (line_speed * maxCfg) / 100;
1170                 else { /* SD mode */
1171                         uint16_t vn_max_rate = maxCfg * 100;
1172
1173                         if (vn_max_rate < line_speed)
1174                                 line_speed = vn_max_rate;
1175                 }
1176         }
1177
1178         return line_speed;
1179 }
1180
1181 /**
1182  * bnx2x_fill_report_data - fill link report data to report
1183  *
1184  * @bp:         driver handle
1185  * @data:       link state to update
1186  *
1187  * It uses a none-atomic bit operations because is called under the mutex.
1188  */
1189 static void bnx2x_fill_report_data(struct bnx2x *bp,
1190                                    struct bnx2x_link_report_data *data)
1191 {
1192         memset(data, 0, sizeof(*data));
1193
1194         if (IS_PF(bp)) {
1195                 /* Fill the report data: effective line speed */
1196                 data->line_speed = bnx2x_get_mf_speed(bp);
1197
1198                 /* Link is down */
1199                 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1200                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1201                                   &data->link_report_flags);
1202
1203                 if (!BNX2X_NUM_ETH_QUEUES(bp))
1204                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1205                                   &data->link_report_flags);
1206
1207                 /* Full DUPLEX */
1208                 if (bp->link_vars.duplex == DUPLEX_FULL)
1209                         __set_bit(BNX2X_LINK_REPORT_FD,
1210                                   &data->link_report_flags);
1211
1212                 /* Rx Flow Control is ON */
1213                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1214                         __set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1215                                   &data->link_report_flags);
1216
1217                 /* Tx Flow Control is ON */
1218                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1219                         __set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1220                                   &data->link_report_flags);
1221         } else { /* VF */
1222                 *data = bp->vf_link_vars;
1223         }
1224 }
1225
1226 /**
1227  * bnx2x_link_report - report link status to OS.
1228  *
1229  * @bp:         driver handle
1230  *
1231  * Calls the __bnx2x_link_report() under the same locking scheme
1232  * as a link/PHY state managing code to ensure a consistent link
1233  * reporting.
1234  */
1235
1236 void bnx2x_link_report(struct bnx2x *bp)
1237 {
1238         bnx2x_acquire_phy_lock(bp);
1239         __bnx2x_link_report(bp);
1240         bnx2x_release_phy_lock(bp);
1241 }
1242
1243 /**
1244  * __bnx2x_link_report - report link status to OS.
1245  *
1246  * @bp:         driver handle
1247  *
1248  * None atomic implementation.
1249  * Should be called under the phy_lock.
1250  */
1251 void __bnx2x_link_report(struct bnx2x *bp)
1252 {
1253         struct bnx2x_link_report_data cur_data;
1254
1255         /* reread mf_cfg */
1256         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1257                 bnx2x_read_mf_cfg(bp);
1258
1259         /* Read the current link report info */
1260         bnx2x_fill_report_data(bp, &cur_data);
1261
1262         /* Don't report link down or exactly the same link status twice */
1263         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1264             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1265                       &bp->last_reported_link.link_report_flags) &&
1266              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1267                       &cur_data.link_report_flags)))
1268                 return;
1269
1270         bp->link_cnt++;
1271
1272         /* We are going to report a new link parameters now -
1273          * remember the current data for the next time.
1274          */
1275         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1276
1277         /* propagate status to VFs */
1278         if (IS_PF(bp))
1279                 bnx2x_iov_link_update(bp);
1280
1281         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1282                      &cur_data.link_report_flags)) {
1283                 netif_carrier_off(bp->dev);
1284                 netdev_err(bp->dev, "NIC Link is Down\n");
1285                 return;
1286         } else {
1287                 const char *duplex;
1288                 const char *flow;
1289
1290                 netif_carrier_on(bp->dev);
1291
1292                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1293                                        &cur_data.link_report_flags))
1294                         duplex = "full";
1295                 else
1296                         duplex = "half";
1297
1298                 /* Handle the FC at the end so that only these flags would be
1299                  * possibly set. This way we may easily check if there is no FC
1300                  * enabled.
1301                  */
1302                 if (cur_data.link_report_flags) {
1303                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1304                                      &cur_data.link_report_flags)) {
1305                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1306                                      &cur_data.link_report_flags))
1307                                         flow = "ON - receive & transmit";
1308                                 else
1309                                         flow = "ON - receive";
1310                         } else {
1311                                 flow = "ON - transmit";
1312                         }
1313                 } else {
1314                         flow = "none";
1315                 }
1316                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1317                             cur_data.line_speed, duplex, flow);
1318         }
1319 }
1320
1321 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1322 {
1323         int i;
1324
1325         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1326                 struct eth_rx_sge *sge;
1327
1328                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1329                 sge->addr_hi =
1330                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1331                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1332
1333                 sge->addr_lo =
1334                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1335                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1336         }
1337 }
1338
1339 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1340                                 struct bnx2x_fastpath *fp, int last)
1341 {
1342         int i;
1343
1344         for (i = 0; i < last; i++) {
1345                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1346                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1347                 uint8_t *data = first_buf->data;
1348
1349                 if (data == NULL) {
1350                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1351                         continue;
1352                 }
1353                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1354                         dma_unmap_single(&bp->pdev->dev,
1355                                          dma_unmap_addr(first_buf, mapping),
1356                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1357                 bnx2x_frag_free(fp, data);
1358                 first_buf->data = NULL;
1359         }
1360 }
1361
1362 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1363 {
1364         int j;
1365
1366         for_each_rx_queue_cnic(bp, j) {
1367                 struct bnx2x_fastpath *fp = &bp->fp[j];
1368
1369                 fp->rx_bd_cons = 0;
1370
1371                 /* Activate BD ring */
1372                 /* Warning!
1373                  * this will generate an interrupt (to the TSTORM)
1374                  * must only be done after chip is initialized
1375                  */
1376                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1377                                      fp->rx_sge_prod);
1378         }
1379 }
1380
1381 void bnx2x_init_rx_rings(struct bnx2x *bp)
1382 {
1383         int func = BP_FUNC(bp);
1384         uint16_t ring_prod;
1385         int i, j;
1386
1387         /* Allocate TPA resources */
1388         for_each_eth_queue(bp, j) {
1389                 struct bnx2x_fastpath *fp = &bp->fp[j];
1390
1391                 DP(NETIF_MSG_IFUP,
1392                    "mtu %d  rx_buf_size %d\n", bp->dev->maxmtu, fp->rx_buf_size);
1393
1394                 if (!fp->disable_tpa) {
1395                         /* Fill the per-aggregation pool */
1396                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1397                                 struct bnx2x_agg_info *tpa_info =
1398                                         &fp->tpa_info[i];
1399                                 struct sw_rx_bd *first_buf =
1400                                         &tpa_info->first_buf;
1401
1402                                 first_buf->data =
1403                                         bnx2x_frag_alloc(fp, KMALLOC_WAIT);
1404                                 if (!first_buf->data) {
1405                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1406                                                   j);
1407                                         bnx2x_free_tpa_pool(bp, fp, i);
1408                                         fp->disable_tpa = 1;
1409                                         break;
1410                                 }
1411                                 dma_unmap_addr_set(first_buf, mapping, 0);
1412                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1413                         }
1414
1415                         /* "next page" elements initialization */
1416                         bnx2x_set_next_page_sgl(fp);
1417
1418                         /* set SGEs bit mask */
1419                         bnx2x_init_sge_ring_bit_mask(fp);
1420
1421                         /* Allocate SGEs and initialize the ring elements */
1422                         for (i = 0, ring_prod = 0;
1423                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1424
1425                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1426                                                        KMALLOC_WAIT) < 0) {
1427                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1428                                                   i);
1429                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1430                                                   j);
1431                                         /* Cleanup already allocated elements */
1432                                         bnx2x_free_rx_sge_range(bp, fp,
1433                                                                 ring_prod);
1434                                         bnx2x_free_tpa_pool(bp, fp,
1435                                                             MAX_AGG_QS(bp));
1436                                         fp->disable_tpa = 1;
1437                                         ring_prod = 0;
1438                                         break;
1439                                 }
1440                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1441                         }
1442
1443                         fp->rx_sge_prod = ring_prod;
1444                 }
1445         }
1446
1447         for_each_eth_queue(bp, j) {
1448                 struct bnx2x_fastpath *fp = &bp->fp[j];
1449
1450                 fp->rx_bd_cons = 0;
1451
1452                 /* Activate BD ring */
1453                 /* Warning!
1454                  * this will generate an interrupt (to the TSTORM)
1455                  * must only be done after chip is initialized
1456                  */
1457                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1458                                      fp->rx_sge_prod);
1459
1460                 if (j != 0)
1461                         continue;
1462
1463                 if (CHIP_IS_E1(bp)) {
1464                         REG_WR(bp, BAR_USTRORM_INTMEM +
1465                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1466                                U64_LO(fp->rx_comp_mapping));
1467                         REG_WR(bp, BAR_USTRORM_INTMEM +
1468                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1469                                U64_HI(fp->rx_comp_mapping));
1470                 }
1471         }
1472 }
1473
1474 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1475 {
1476 panic("Not implemented");
1477 #if 0 // AKAROS_PORT
1478         uint8_t cos;
1479         struct bnx2x *bp = fp->bp;
1480
1481         for_each_cos_in_tx_queue(fp, cos) {
1482                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1483                 unsigned pkts_compl = 0, bytes_compl = 0;
1484
1485                 uint16_t sw_prod = txdata->tx_pkt_prod;
1486                 uint16_t sw_cons = txdata->tx_pkt_cons;
1487
1488                 while (sw_cons != sw_prod) {
1489                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1490                                           &pkts_compl, &bytes_compl);
1491                         sw_cons++;
1492                 }
1493
1494                 netdev_tx_reset_queue(
1495                         netdev_get_tx_queue(bp->dev,
1496                                             txdata->txq_index));
1497         }
1498 #endif
1499 }
1500
1501 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1502 {
1503         int i;
1504
1505         for_each_tx_queue_cnic(bp, i) {
1506                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1507         }
1508 }
1509
1510 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1511 {
1512         int i;
1513
1514         for_each_eth_queue(bp, i) {
1515                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1516         }
1517 }
1518
1519 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1520 {
1521         struct bnx2x *bp = fp->bp;
1522         int i;
1523
1524         /* ring wasn't allocated */
1525         if (fp->rx_buf_ring == NULL)
1526                 return;
1527
1528         for (i = 0; i < NUM_RX_BD; i++) {
1529                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1530                 uint8_t *data = rx_buf->data;
1531
1532                 if (data == NULL)
1533                         continue;
1534                 dma_unmap_single(&bp->pdev->dev,
1535                                  dma_unmap_addr(rx_buf, mapping),
1536                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1537
1538                 rx_buf->data = NULL;
1539                 bnx2x_frag_free(fp, data);
1540         }
1541 }
1542
1543 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1544 {
1545         int j;
1546
1547         for_each_rx_queue_cnic(bp, j) {
1548                 bnx2x_free_rx_bds(&bp->fp[j]);
1549         }
1550 }
1551
1552 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1553 {
1554         int j;
1555
1556         for_each_eth_queue(bp, j) {
1557                 struct bnx2x_fastpath *fp = &bp->fp[j];
1558
1559                 bnx2x_free_rx_bds(fp);
1560
1561                 if (!fp->disable_tpa)
1562                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1563         }
1564 }
1565
1566 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1567 {
1568         bnx2x_free_tx_skbs_cnic(bp);
1569         bnx2x_free_rx_skbs_cnic(bp);
1570 }
1571
1572 void bnx2x_free_skbs(struct bnx2x *bp)
1573 {
1574         bnx2x_free_tx_skbs(bp);
1575         bnx2x_free_rx_skbs(bp);
1576 }
1577
1578 void bnx2x_update_max_mf_config(struct bnx2x *bp, uint32_t value)
1579 {
1580         /* load old values */
1581         uint32_t mf_cfg = bp->mf_config[BP_VN(bp)];
1582
1583         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1584                 /* leave all but MAX value */
1585                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1586
1587                 /* set new MAX value */
1588                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1589                                 & FUNC_MF_CFG_MAX_BW_MASK;
1590
1591                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1592         }
1593 }
1594
1595 /**
1596  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1597  *
1598  * @bp:         driver handle
1599  * @nvecs:      number of vectors to be released
1600  */
1601 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1602 {
1603 panic("Not implemented");
1604 #if 0 // AKAROS_PORT
1605         int i, offset = 0;
1606
1607         if (nvecs == offset)
1608                 return;
1609
1610         /* VFs don't have a default SB */
1611         if (IS_PF(bp)) {
1612                 free_irq(bp->msix_table[offset].vector, bp->dev);
1613                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1614                    bp->msix_table[offset].vector);
1615                 offset++;
1616         }
1617
1618         if (CNIC_SUPPORT(bp)) {
1619                 if (nvecs == offset)
1620                         return;
1621                 offset++;
1622         }
1623
1624         for_each_eth_queue(bp, i) {
1625                 if (nvecs == offset)
1626                         return;
1627                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1628                    i, bp->msix_table[offset].vector);
1629
1630                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1631         }
1632 #endif
1633 }
1634
1635 void bnx2x_free_irq(struct bnx2x *bp)
1636 {
1637 panic("Not implemented");
1638 #if 0 // AKAROS_PORT
1639         if (bp->flags & USING_MSIX_FLAG &&
1640             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1641                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1642
1643                 /* vfs don't have a default status block */
1644                 if (IS_PF(bp))
1645                         nvecs++;
1646
1647                 bnx2x_free_msix_irqs(bp, nvecs);
1648         } else {
1649                 free_irq(bp->dev->irq, bp->dev);
1650         }
1651 #endif
1652 }
1653
1654 int bnx2x_enable_msix(struct bnx2x *bp)
1655 {
1656         int msix_vec = 0, i, rc;
1657 panic("Not implemented");
1658 #if 0 // AKAROS_PORT
1659         /* VFs don't have a default status block */
1660         if (IS_PF(bp)) {
1661                 bp->msix_table[msix_vec].entry = msix_vec;
1662                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1663                                bp->msix_table[0].entry);
1664                 msix_vec++;
1665         }
1666
1667         /* Cnic requires an msix vector for itself */
1668         if (CNIC_SUPPORT(bp)) {
1669                 bp->msix_table[msix_vec].entry = msix_vec;
1670                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1671                                msix_vec, bp->msix_table[msix_vec].entry);
1672                 msix_vec++;
1673         }
1674
1675         /* We need separate vectors for ETH queues only (not FCoE) */
1676         for_each_eth_queue(bp, i) {
1677                 bp->msix_table[msix_vec].entry = msix_vec;
1678                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1679                                msix_vec, msix_vec, i);
1680                 msix_vec++;
1681         }
1682
1683         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1684            msix_vec);
1685
1686         rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1687                                    BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1688         /*
1689          * reconfigure number of tx/rx queues according to available
1690          * MSI-X vectors
1691          */
1692         if (rc == -ENOSPC) {
1693                 /* Get by with single vector */
1694                 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1695                 if (rc < 0) {
1696                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1697                                        rc);
1698                         goto no_msix;
1699                 }
1700
1701                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1702                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1703
1704                 BNX2X_DEV_INFO("set number of queues to 1\n");
1705                 bp->num_ethernet_queues = 1;
1706                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1707         } else if (rc < 0) {
1708                 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1709                 goto no_msix;
1710         } else if (rc < msix_vec) {
1711                 /* how less vectors we will have? */
1712                 int diff = msix_vec - rc;
1713
1714                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1715
1716                 /*
1717                  * decrease number of queues by number of unallocated entries
1718                  */
1719                 bp->num_ethernet_queues -= diff;
1720                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1721
1722                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1723                                bp->num_queues);
1724         }
1725
1726         bp->flags |= USING_MSIX_FLAG;
1727
1728         return 0;
1729
1730 no_msix:
1731         /* fall to INTx if not enough memory */
1732         if (rc == -ENOMEM)
1733                 bp->flags |= DISABLE_MSI_FLAG;
1734
1735         return rc;
1736 #endif
1737 }
1738
1739 static void bullshit_handler(struct hw_trapframe *hw_tf, void *cnic_turd)
1740 {
1741         printk("bnx2x CNIC IRQ fired.  Probably a bug!\n");
1742 }
1743
1744 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1745 {
1746         int i, rc, offset = 0;
1747
1748         /* no default status block for vf */
1749         if (IS_PF(bp)) {
1750                 rc = register_irq(bp->msix_table[offset++].vector,
1751                                   bnx2x_msix_sp_int, bp->dev,
1752                                   pci_to_tbdf(bp->pdev));
1753                 if (rc) {
1754                         BNX2X_ERR("request sp irq failed\n");
1755                         return -EBUSY;
1756                 }
1757         }
1758
1759         if (CNIC_SUPPORT(bp)) {
1760                 offset++;
1761                 // AKAROS_PORT
1762                 rc = register_irq(0, bullshit_handler, 0, pci_to_tbdf(bp->pdev));
1763                 if (rc) {
1764                         BNX2X_ERR("Fucked up getting a CNIC MSIX vector!");
1765                         return -EBUSY;
1766                 }
1767         }
1768
1769         for_each_eth_queue(bp, i) {
1770                 struct bnx2x_fastpath *fp = &bp->fp[i];
1771                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1772                          bp->dev->name, i);
1773
1774                 rc = register_irq(bp->msix_table[offset].vector,
1775                                   bnx2x_msix_fp_int, fp, pci_to_tbdf(bp->pdev));
1776                 if (rc) {
1777                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1778                               bp->msix_table[offset].vector, rc);
1779                         bnx2x_free_msix_irqs(bp, offset);
1780                         return -EBUSY;
1781                 }
1782
1783                 offset++;
1784         }
1785
1786         i = BNX2X_NUM_ETH_QUEUES(bp);
1787         if (IS_PF(bp)) {
1788                 offset = 1 + CNIC_SUPPORT(bp);
1789                 netdev_info(bp->dev,
1790                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1791                             bp->msix_table[0].vector,
1792                             0, bp->msix_table[offset].vector,
1793                             i - 1, bp->msix_table[offset + i - 1].vector);
1794         } else {
1795                 offset = CNIC_SUPPORT(bp);
1796                 netdev_info(bp->dev,
1797                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1798                             0, bp->msix_table[offset].vector,
1799                             i - 1, bp->msix_table[offset + i - 1].vector);
1800         }
1801         return 0;
1802 }
1803
1804 int bnx2x_enable_msi(struct bnx2x *bp)
1805 {
1806 panic("Not implemented");
1807 #if 0 // AKAROS_PORT
1808         int rc;
1809
1810         rc = pci_enable_msi(bp->pdev);
1811         if (rc) {
1812                 BNX2X_DEV_INFO("MSI is not attainable\n");
1813                 return -1;
1814         }
1815         bp->flags |= USING_MSI_FLAG;
1816
1817         return 0;
1818 #endif
1819 }
1820
1821 static int bnx2x_req_irq(struct bnx2x *bp)
1822 {
1823         unsigned long flags;
1824 panic("Not implemented");
1825 #if 0 // AKAROS_PORT
1826         unsigned int irq;
1827
1828         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1829                 flags = 0;
1830         else
1831                 flags = IRQF_SHARED;
1832
1833         if (bp->flags & USING_MSIX_FLAG)
1834                 irq = bp->msix_table[0].vector;
1835         else
1836                 irq = bp->pdev->irq;
1837
1838         return register_irq(irq, bnx2x_interrupt, bp->dev,
1839                             pci_to_tbdf(bp->pdev));
1840 #endif
1841 }
1842
1843 static int bnx2x_setup_irqs(struct bnx2x *bp)
1844 {
1845         return bnx2x_req_msix_irqs(bp);
1846 #if 0 // AKAROS_PORT we just register_irq
1847         if (bp->flags & USING_MSIX_FLAG &&
1848             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1849                 rc = bnx2x_req_msix_irqs(bp);
1850                 if (rc)
1851                         return rc;
1852         } else {
1853                 rc = bnx2x_req_irq(bp);
1854                 if (rc) {
1855                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1856                         return rc;
1857                 }
1858                 if (bp->flags & USING_MSI_FLAG) {
1859                         bp->dev->irq = bp->pdev->irq;
1860                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1861                                     bp->dev->irq);
1862                 }
1863                 if (bp->flags & USING_MSIX_FLAG) {
1864                         bp->dev->irq = bp->msix_table[0].vector;
1865                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1866                                     bp->dev->irq);
1867                 }
1868         }
1869
1870         return 0;
1871 #endif
1872 }
1873
1874 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1875 {
1876         int i;
1877
1878         for_each_rx_queue_cnic(bp, i) {
1879                 bnx2x_fp_init_lock(&bp->fp[i]);
1880                 napi_enable(&bnx2x_fp(bp, i, napi));
1881         }
1882 }
1883
1884 static void bnx2x_napi_enable(struct bnx2x *bp)
1885 {
1886         int i;
1887
1888         for_each_eth_queue(bp, i) {
1889                 bnx2x_fp_init_lock(&bp->fp[i]);
1890                 napi_enable(&bnx2x_fp(bp, i, napi));
1891         }
1892 }
1893
1894 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1895 {
1896         int i;
1897
1898         for_each_rx_queue_cnic(bp, i) {
1899                 napi_disable(&bnx2x_fp(bp, i, napi));
1900                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1901                         kthread_usleep(1000);
1902         }
1903 }
1904
1905 static void bnx2x_napi_disable(struct bnx2x *bp)
1906 {
1907         int i;
1908
1909         for_each_eth_queue(bp, i) {
1910                 napi_disable(&bnx2x_fp(bp, i, napi));
1911                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1912                         kthread_usleep(1000);
1913         }
1914 }
1915
1916 void bnx2x_netif_start(struct bnx2x *bp)
1917 {
1918 panic("Not implemented");
1919 #if 0 // AKAROS_PORT
1920         if (netif_running(bp->dev)) {
1921                 bnx2x_napi_enable(bp);
1922                 if (CNIC_LOADED(bp))
1923                         bnx2x_napi_enable_cnic(bp);
1924                 bnx2x_int_enable(bp);
1925                 if (bp->state == BNX2X_STATE_OPEN)
1926                         netif_tx_wake_all_queues(bp->dev);
1927         }
1928 #endif
1929 }
1930
1931 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1932 {
1933         bnx2x_int_disable_sync(bp, disable_hw);
1934         bnx2x_napi_disable(bp);
1935         if (CNIC_LOADED(bp))
1936                 bnx2x_napi_disable_cnic(bp);
1937 }
1938
1939 uint16_t bnx2x_select_queue(struct ether *dev, struct sk_buff *skb,
1940                        void *accel_priv, select_queue_fallback_t fallback)
1941 {
1942 panic("Not implemented");
1943 #if 0 // AKAROS_PORT
1944         struct bnx2x *bp = netdev_priv(dev);
1945
1946         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1947                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1948                 uint16_t ether_type = be16_to_cpu(hdr->h_proto);
1949
1950                 /* Skip VLAN tag if present */
1951                 if (ether_type == ETH_P_8021Q) {
1952                         struct vlan_ethhdr *vhdr =
1953                                 (struct vlan_ethhdr *)skb->data;
1954
1955                         ether_type = be16_to_cpu(vhdr->h_vlan_encapsulated_proto);
1956                 }
1957
1958                 /* If ethertype is FCoE or FIP - use FCoE ring */
1959                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1960                         return bnx2x_fcoe_tx(bp, txq_index);
1961         }
1962
1963         /* select a non-FCoE queue */
1964         return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1965 #endif
1966 }
1967
1968 void bnx2x_set_num_queues(struct bnx2x *bp)
1969 {
1970         /* RSS queues */
1971         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1972
1973         /* override in STORAGE SD modes */
1974         if (IS_MF_STORAGE_ONLY(bp))
1975                 bp->num_ethernet_queues = 1;
1976
1977         /* Add special queues */
1978         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1979         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1980
1981         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1982 }
1983
1984 /**
1985  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1986  *
1987  * @bp:         Driver handle
1988  *
1989  * We currently support for at most 16 Tx queues for each CoS thus we will
1990  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1991  * bp->max_cos.
1992  *
1993  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1994  * index after all ETH L2 indices.
1995  *
1996  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1997  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1998  * 16..31,...) with indices that are not coupled with any real Tx queue.
1999  *
2000  * The proper configuration of skb->queue_mapping is handled by
2001  * bnx2x_select_queue() and __skb_tx_hash().
2002  *
2003  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
2004  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
2005  */
2006 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
2007 {
2008         int rc, tx, rx;
2009
2010         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
2011         rx = BNX2X_NUM_ETH_QUEUES(bp);
2012
2013 /* account for fcoe queue */
2014         if (include_cnic && !NO_FCOE(bp)) {
2015                 rx++;
2016                 tx++;
2017         }
2018
2019 #if 0 // AKAROS_PORT XME: set queues in ether
2020         rc = netif_set_real_num_tx_queues(bp->dev, tx);
2021         if (rc) {
2022                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
2023                 return rc;
2024         }
2025         rc = netif_set_real_num_rx_queues(bp->dev, rx);
2026         if (rc) {
2027                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
2028                 return rc;
2029         }
2030 #else
2031         rc = 0;
2032 #endif
2033
2034         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
2035                           tx, rx);
2036
2037         return rc;
2038 }
2039
2040 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2041 {
2042         int i;
2043
2044         for_each_queue(bp, i) {
2045                 struct bnx2x_fastpath *fp = &bp->fp[i];
2046                 uint32_t mtu;
2047
2048                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
2049                 if (IS_FCOE_IDX(i))
2050                         /*
2051                          * Although there are no IP frames expected to arrive to
2052                          * this ring we still want to add an
2053                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2054                          * overrun attack.
2055                          */
2056                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2057                 else
2058                         mtu = bp->dev->maxmtu;
2059                 /* AKAROS_PORT XME struct block alignment and size issues? */
2060                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2061                                   IP_HEADER_ALIGNMENT_PADDING +
2062                                   ETH_OVREHEAD +
2063                                   mtu +
2064                                   BNX2X_FW_RX_ALIGN_END;
2065                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2066                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2067                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2068                 else
2069                         fp->rx_frag_size = 0;
2070         }
2071 }
2072
2073 static int bnx2x_init_rss(struct bnx2x *bp)
2074 {
2075         int i;
2076         uint8_t num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2077
2078         /* Prepare the initial contents for the indirection table if RSS is
2079          * enabled
2080          */
2081         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2082                 bp->rss_conf_obj.ind_table[i] =
2083                         bp->fp->cl_id +
2084                         ethtool_rxfh_indir_default(i, num_eth_queues);
2085
2086         /*
2087          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2088          * per-port, so if explicit configuration is needed , do it only
2089          * for a PMF.
2090          *
2091          * For 57712 and newer on the other hand it's a per-function
2092          * configuration.
2093          */
2094         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2095 }
2096
2097 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2098               bool config_hash, bool enable)
2099 {
2100         struct bnx2x_config_rss_params params = {NULL};
2101
2102         /* Although RSS is meaningless when there is a single HW queue we
2103          * still need it enabled in order to have HW Rx hash generated.
2104          *
2105          * if (!is_eth_multi(bp))
2106          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2107          */
2108
2109         params.rss_obj = rss_obj;
2110
2111         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2112
2113         if (enable) {
2114                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2115
2116                 /* RSS configuration */
2117                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2118                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2119                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2120                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2121                 if (rss_obj->udp_rss_v4)
2122                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2123                 if (rss_obj->udp_rss_v6)
2124                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2125
2126                 if (!CHIP_IS_E1x(bp))
2127                         /* valid only for TUNN_MODE_GRE tunnel mode */
2128                         __set_bit(BNX2X_RSS_GRE_INNER_HDRS, &params.rss_flags);
2129         } else {
2130                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2131         }
2132
2133         /* Hash bits */
2134         params.rss_result_mask = MULTI_MASK;
2135
2136         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2137
2138         if (config_hash) {
2139                 /* RSS keys */
2140                 #if 0 // AKAROS_PORT
2141                 netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2142                 #else
2143                 /* linux picks a random, once, then uses it here.  it could be 5a! */
2144                 memset(params.rss_key, 0x5a, T_ETH_RSS_KEY * 4);
2145                 #endif
2146                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2147         }
2148
2149         if (IS_PF(bp))
2150                 return bnx2x_config_rss(bp, &params);
2151         else
2152                 return bnx2x_vfpf_config_rss(bp, &params);
2153 }
2154
2155 static int bnx2x_init_hw(struct bnx2x *bp, uint32_t load_code)
2156 {
2157         struct bnx2x_func_state_params func_params = {NULL};
2158
2159         /* Prepare parameters for function state transitions */
2160         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2161
2162         func_params.f_obj = &bp->func_obj;
2163         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2164
2165         func_params.params.hw_init.load_phase = load_code;
2166
2167         return bnx2x_func_state_change(bp, &func_params);
2168 }
2169
2170 /*
2171  * Cleans the object that have internal lists without sending
2172  * ramrods. Should be run when interrupts are disabled.
2173  */
2174 void bnx2x_squeeze_objects(struct bnx2x *bp)
2175 {
2176         int rc;
2177         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2178         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2179         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2180
2181         /***************** Cleanup MACs' object first *************************/
2182
2183         /* Wait for completion of requested */
2184         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2185         /* Perform a dry cleanup */
2186         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2187
2188         /* Clean ETH primary MAC */
2189         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2190         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2191                                  &ramrod_flags);
2192         if (rc != 0)
2193                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2194
2195         /* Cleanup UC list */
2196         vlan_mac_flags = 0;
2197         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2198         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2199                                  &ramrod_flags);
2200         if (rc != 0)
2201                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2202
2203         /***************** Now clean mcast object *****************************/
2204         rparam.mcast_obj = &bp->mcast_obj;
2205         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2206
2207         /* Add a DEL command... - Since we're doing a driver cleanup only,
2208          * we take a lock surrounding both the initial send and the CONTs,
2209          * as we don't want a true completion to disrupt us in the middle.
2210          */
2211         qlock(&bp->dev->qlock);
2212         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2213         if (rc < 0)
2214                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2215                           rc);
2216
2217         /* ...and wait until all pending commands are cleared */
2218         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2219         while (rc != 0) {
2220                 if (rc < 0) {
2221                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2222                                   rc);
2223                         qunlock(&bp->dev->qlock);
2224                         return;
2225                 }
2226
2227                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2228         }
2229         qunlock(&bp->dev->qlock);
2230 }
2231
2232 #ifndef BNX2X_STOP_ON_ERROR
2233 #define LOAD_ERROR_EXIT(bp, label) \
2234         do { \
2235                 (bp)->state = BNX2X_STATE_ERROR; \
2236                 goto label; \
2237         } while (0)
2238
2239 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2240         do { \
2241                 bp->cnic_loaded = false; \
2242                 goto label; \
2243         } while (0)
2244 #else /*BNX2X_STOP_ON_ERROR*/
2245 #define LOAD_ERROR_EXIT(bp, label) \
2246         do { \
2247                 (bp)->state = BNX2X_STATE_ERROR; \
2248                 (bp)->panic = 1; \
2249                 return -EBUSY; \
2250         } while (0)
2251 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2252         do { \
2253                 bp->cnic_loaded = false; \
2254                 (bp)->panic = 1; \
2255                 return -EBUSY; \
2256         } while (0)
2257 #endif /*BNX2X_STOP_ON_ERROR*/
2258
2259 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2260 {
2261         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2262                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2263         return;
2264 }
2265
2266 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2267 {
2268         int num_groups, vf_headroom = 0;
2269         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2270
2271         /* number of queues for statistics is number of eth queues + FCoE */
2272         uint8_t num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2273
2274         /* Total number of FW statistics requests =
2275          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2276          * and fcoe l2 queue) stats + num of queues (which includes another 1
2277          * for fcoe l2 queue if applicable)
2278          */
2279         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2280
2281         /* vf stats appear in the request list, but their data is allocated by
2282          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2283          * it is used to determine where to place the vf stats queries in the
2284          * request struct
2285          */
2286         if (IS_SRIOV(bp))
2287                 vf_headroom = bnx2x_vf_headroom(bp);
2288
2289         /* Request is built from stats_query_header and an array of
2290          * stats_query_cmd_group each of which contains
2291          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2292          * configured in the stats_query_header.
2293          */
2294         num_groups =
2295                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2296                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2297                  1 : 0));
2298
2299         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2300            bp->fw_stats_num, vf_headroom, num_groups);
2301         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2302                 num_groups * sizeof(struct stats_query_cmd_group);
2303
2304         /* Data for statistics requests + stats_counter
2305          * stats_counter holds per-STORM counters that are incremented
2306          * when STORM has finished with the current request.
2307          * memory for FCoE offloaded statistics are counted anyway,
2308          * even if they will not be sent.
2309          * VF stats are not accounted for here as the data of VF stats is stored
2310          * in memory allocated by the VF, not here.
2311          */
2312         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2313                 sizeof(struct per_pf_stats) +
2314                 sizeof(struct fcoe_statistics_params) +
2315                 sizeof(struct per_queue_stats) * num_queue_stats +
2316                 sizeof(struct stats_counter);
2317
2318         bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2319                                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2320         if (!bp->fw_stats)
2321                 goto alloc_mem_err;
2322
2323         /* Set shortcuts */
2324         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2325         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2326         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2327                 ((uint8_t *)bp->fw_stats + bp->fw_stats_req_sz);
2328         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2329                 bp->fw_stats_req_sz;
2330
2331         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2332            U64_HI(bp->fw_stats_req_mapping),
2333            U64_LO(bp->fw_stats_req_mapping));
2334         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2335            U64_HI(bp->fw_stats_data_mapping),
2336            U64_LO(bp->fw_stats_data_mapping));
2337         return 0;
2338
2339 alloc_mem_err:
2340         bnx2x_free_fw_stats_mem(bp);
2341         BNX2X_ERR("Can't allocate FW stats memory\n");
2342         return -ENOMEM;
2343 }
2344
2345 /* send load request to mcp and analyze response */
2346 static int bnx2x_nic_load_request(struct bnx2x *bp, uint32_t *load_code)
2347 {
2348         uint32_t param;
2349
2350         /* init fw_seq */
2351         bp->fw_seq =
2352                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2353                  DRV_MSG_SEQ_NUMBER_MASK);
2354         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2355
2356         /* Get current FW pulse sequence */
2357         bp->fw_drv_pulse_wr_seq =
2358                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2359                  DRV_PULSE_SEQ_MASK);
2360         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2361
2362         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2363
2364         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2365                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2366
2367         /* load request */
2368         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2369
2370         /* if mcp fails to respond we must abort */
2371         if (!(*load_code)) {
2372                 BNX2X_ERR("MCP response failure, aborting\n");
2373                 return -EBUSY;
2374         }
2375
2376         /* If mcp refused (e.g. other port is in diagnostic mode) we
2377          * must abort
2378          */
2379         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2380                 BNX2X_ERR("MCP refused load request, aborting\n");
2381                 return -EBUSY;
2382         }
2383         return 0;
2384 }
2385
2386 /* check whether another PF has already loaded FW to chip. In
2387  * virtualized environments a pf from another VM may have already
2388  * initialized the device including loading FW
2389  */
2390 int bnx2x_compare_fw_ver(struct bnx2x *bp, uint32_t load_code,
2391                          bool print_err)
2392 {
2393         /* is another pf loaded on this engine? */
2394         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2395             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2396                 /* build my FW version dword */
2397                 uint32_t my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2398                         (BCM_5710_FW_MINOR_VERSION << 8) +
2399                         (BCM_5710_FW_REVISION_VERSION << 16) +
2400                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2401
2402                 /* read loaded FW from chip */
2403                 uint32_t loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2404
2405                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2406                    loaded_fw, my_fw);
2407
2408                 /* abort nic load if version mismatch */
2409                 if (my_fw != loaded_fw) {
2410                         if (print_err)
2411                                 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2412                                           loaded_fw, my_fw);
2413                         else
2414                                 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2415                                                loaded_fw, my_fw);
2416                         return -EBUSY;
2417                 }
2418         }
2419         return 0;
2420 }
2421
2422 /* returns the "mcp load_code" according to global load_count array */
2423 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2424 {
2425         int path = BP_PATH(bp);
2426
2427         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2428            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2429            bnx2x_load_count[path][2]);
2430         bnx2x_load_count[path][0]++;
2431         bnx2x_load_count[path][1 + port]++;
2432         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2433            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2434            bnx2x_load_count[path][2]);
2435         if (bnx2x_load_count[path][0] == 1)
2436                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2437         else if (bnx2x_load_count[path][1 + port] == 1)
2438                 return FW_MSG_CODE_DRV_LOAD_PORT;
2439         else
2440                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2441 }
2442
2443 /* mark PMF if applicable */
2444 static void bnx2x_nic_load_pmf(struct bnx2x *bp, uint32_t load_code)
2445 {
2446         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2447             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2448             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2449                 bp->port.pmf = 1;
2450                 /* We need the barrier to ensure the ordering between the
2451                  * writing to bp->port.pmf here and reading it from the
2452                  * bnx2x_periodic_task().
2453                  */
2454                 mb();
2455         } else {
2456                 bp->port.pmf = 0;
2457         }
2458
2459         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2460 }
2461
2462 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2463 {
2464         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2465              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2466             (bp->common.shmem2_base)) {
2467                 if (SHMEM2_HAS(bp, dcc_support))
2468                         SHMEM2_WR(bp, dcc_support,
2469                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2470                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2471                 if (SHMEM2_HAS(bp, afex_driver_support))
2472                         SHMEM2_WR(bp, afex_driver_support,
2473                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2474         }
2475
2476         /* Set AFEX default VLAN tag to an invalid value */
2477         bp->afex_def_vlan_tag = -1;
2478 }
2479
2480 /**
2481  * bnx2x_bz_fp - zero content of the fastpath structure.
2482  *
2483  * @bp:         driver handle
2484  * @index:      fastpath index to be zeroed
2485  *
2486  * Makes sure the contents of the bp->fp[index].napi is kept
2487  * intact.
2488  */
2489 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2490 {
2491         struct bnx2x_fastpath *fp = &bp->fp[index];
2492         int cos;
2493         struct napi_struct orig_napi = fp->napi;
2494         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2495
2496         /* bzero bnx2x_fastpath contents */
2497         if (fp->tpa_info)
2498                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2499                        sizeof(struct bnx2x_agg_info));
2500         memset(fp, 0, sizeof(*fp));
2501
2502         /* AKAROS_PORT: let the code set up whatever fake napi stuff it needs */
2503         /* Restore the NAPI object as it has been already initialized */
2504         fp->napi = orig_napi;
2505         fp->tpa_info = orig_tpa_info;
2506         fp->bp = bp;
2507         fp->index = index;
2508         if (IS_ETH_FP(fp))
2509                 fp->max_cos = bp->max_cos;
2510         else
2511                 /* Special queues support only one CoS */
2512                 fp->max_cos = 1;
2513
2514         /* Init txdata pointers */
2515         if (IS_FCOE_FP(fp))
2516                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2517         if (IS_ETH_FP(fp))
2518                 for_each_cos_in_tx_queue(fp, cos)
2519                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2520                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2521
2522         /* set the tpa flag for each queue. The tpa flag determines the queue
2523          * minimal size so it must be set prior to queue memory allocation
2524          */
2525         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2526                                   (bp->flags & GRO_ENABLE_FLAG &&
2527                                    bnx2x_mtu_allows_gro(bp->dev->maxmtu)));
2528         if (bp->flags & TPA_ENABLE_FLAG)
2529                 fp->mode = TPA_MODE_LRO;
2530         else if (bp->flags & GRO_ENABLE_FLAG)
2531                 fp->mode = TPA_MODE_GRO;
2532
2533         /* We don't want TPA on an FCoE L2 ring */
2534         if (IS_FCOE_FP(fp))
2535                 fp->disable_tpa = 1;
2536 }
2537
2538 int bnx2x_load_cnic(struct bnx2x *bp)
2539 {
2540         int i, rc, port = BP_PORT(bp);
2541
2542         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2543
2544         qlock_init(&bp->cnic_mutex);
2545
2546         if (IS_PF(bp)) {
2547                 rc = bnx2x_alloc_mem_cnic(bp);
2548                 if (rc) {
2549                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2550                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2551                 }
2552         }
2553
2554         rc = bnx2x_alloc_fp_mem_cnic(bp);
2555         if (rc) {
2556                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2557                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2558         }
2559
2560         /* Update the number of queues with the cnic queues */
2561         rc = bnx2x_set_real_num_queues(bp, 1);
2562         if (rc) {
2563                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2564                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2565         }
2566
2567         /* Add all CNIC NAPI objects */
2568         bnx2x_add_all_napi_cnic(bp);
2569         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2570         bnx2x_napi_enable_cnic(bp);
2571
2572         rc = bnx2x_init_hw_func_cnic(bp);
2573         if (rc)
2574                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2575
2576         bnx2x_nic_init_cnic(bp);
2577
2578         if (IS_PF(bp)) {
2579                 /* Enable Timer scan */
2580                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2581
2582                 /* setup cnic queues */
2583                 for_each_cnic_queue(bp, i) {
2584                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2585                         if (rc) {
2586                                 BNX2X_ERR("Queue setup failed\n");
2587                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2588                         }
2589                 }
2590         }
2591
2592         /* Initialize Rx filter. */
2593         bnx2x_set_rx_mode_inner(bp);
2594
2595         /* re-read iscsi info */
2596         bnx2x_get_iscsi_info(bp);
2597         bnx2x_setup_cnic_irq_info(bp);
2598         bnx2x_setup_cnic_info(bp);
2599         bp->cnic_loaded = true;
2600         if (bp->state == BNX2X_STATE_OPEN)
2601                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2602
2603         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2604
2605         return 0;
2606
2607 #ifndef BNX2X_STOP_ON_ERROR
2608 load_error_cnic2:
2609         /* Disable Timer scan */
2610         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2611
2612 load_error_cnic1:
2613         bnx2x_napi_disable_cnic(bp);
2614         /* Update the number of queues without the cnic queues */
2615         if (bnx2x_set_real_num_queues(bp, 0))
2616                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2617 load_error_cnic0:
2618         BNX2X_ERR("CNIC-related load failed\n");
2619         bnx2x_free_fp_mem_cnic(bp);
2620         bnx2x_free_mem_cnic(bp);
2621         return rc;
2622 #endif /* ! BNX2X_STOP_ON_ERROR */
2623 }
2624
2625 /* must be called with rtnl_lock */
2626 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2627 {
2628         int port = BP_PORT(bp);
2629         int i, rc = 0;
2630         uint32_t load_code = 0;
2631
2632         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2633         DP(NETIF_MSG_IFUP,
2634            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2635
2636 #ifdef BNX2X_STOP_ON_ERROR
2637         if (unlikely(bp->panic)) {
2638                 BNX2X_ERR("Can't load NIC when there is panic\n");
2639                 return -EPERM;
2640         }
2641 #endif
2642
2643         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2644
2645         /* zero the structure w/o any lock, before SP handler is initialized */
2646         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2647         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2648                 &bp->last_reported_link.link_report_flags);
2649
2650         if (IS_PF(bp))
2651                 /* must be called before memory allocation and HW init */
2652                 bnx2x_ilt_set_info(bp);
2653
2654         /*
2655          * Zero fastpath structures preserving invariants like napi, which are
2656          * allocated only once, fp index, max_cos, bp pointer.
2657          * Also set fp->disable_tpa and txdata_ptr.
2658          */
2659         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2660         for_each_queue(bp, i)
2661                 bnx2x_bz_fp(bp, i);
2662         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2663                                   bp->num_cnic_queues) *
2664                                   sizeof(struct bnx2x_fp_txdata));
2665
2666         bp->fcoe_init = false;
2667
2668         /* Set the receive queues buffer size */
2669         bnx2x_set_rx_buf_size(bp);
2670
2671         if (IS_PF(bp)) {
2672                 rc = bnx2x_alloc_mem(bp);
2673                 if (rc) {
2674                         BNX2X_ERR("Unable to allocate bp memory\n");
2675                         return rc;
2676                 }
2677         }
2678
2679         /* need to be done after alloc mem, since it's self adjusting to amount
2680          * of memory available for RSS queues
2681          */
2682         rc = bnx2x_alloc_fp_mem(bp);
2683         if (rc) {
2684                 BNX2X_ERR("Unable to allocate memory for fps\n");
2685                 LOAD_ERROR_EXIT(bp, load_error0);
2686         }
2687
2688         /* Allocated memory for FW statistics  */
2689         if (bnx2x_alloc_fw_stats_mem(bp))
2690                 LOAD_ERROR_EXIT(bp, load_error0);
2691
2692         /* request pf to initialize status blocks */
2693         if (IS_VF(bp)) {
2694                 rc = bnx2x_vfpf_init(bp);
2695                 if (rc)
2696                         LOAD_ERROR_EXIT(bp, load_error0);
2697         }
2698
2699         /* As long as bnx2x_alloc_mem() may possibly update
2700          * bp->num_queues, bnx2x_set_real_num_queues() should always
2701          * come after it. At this stage cnic queues are not counted.
2702          */
2703         rc = bnx2x_set_real_num_queues(bp, 0);
2704         if (rc) {
2705                 BNX2X_ERR("Unable to set real_num_queues\n");
2706                 LOAD_ERROR_EXIT(bp, load_error0);
2707         }
2708
2709         /* configure multi cos mappings in kernel.
2710          * this configuration may be overridden by a multi class queue
2711          * discipline or by a dcbx negotiation result.
2712          */
2713         bnx2x_setup_tc(bp->dev, bp->max_cos);
2714
2715         /* Add all NAPI objects */
2716         bnx2x_add_all_napi(bp);
2717         DP(NETIF_MSG_IFUP, "napi added\n");
2718         bnx2x_napi_enable(bp);
2719
2720         if (IS_PF(bp)) {
2721                 /* set pf load just before approaching the MCP */
2722                 bnx2x_set_pf_load(bp);
2723
2724                 /* if mcp exists send load request and analyze response */
2725                 if (!BP_NOMCP(bp)) {
2726                         /* attempt to load pf */
2727                         rc = bnx2x_nic_load_request(bp, &load_code);
2728                         if (rc)
2729                                 LOAD_ERROR_EXIT(bp, load_error1);
2730
2731                         /* what did mcp say? */
2732                         rc = bnx2x_compare_fw_ver(bp, load_code, true);
2733                         if (rc) {
2734                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2735                                 LOAD_ERROR_EXIT(bp, load_error2);
2736                         }
2737                 } else {
2738                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2739                 }
2740
2741                 /* mark pmf if applicable */
2742                 bnx2x_nic_load_pmf(bp, load_code);
2743
2744                 /* Init Function state controlling object */
2745                 bnx2x__init_func_obj(bp);
2746
2747                 /* Initialize HW */
2748                 rc = bnx2x_init_hw(bp, load_code);
2749                 if (rc) {
2750                         BNX2X_ERR("HW init failed, aborting\n");
2751                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2752                         LOAD_ERROR_EXIT(bp, load_error2);
2753                 }
2754         }
2755
2756         bnx2x_pre_irq_nic_init(bp);
2757
2758         /* Connect to IRQs */
2759         rc = bnx2x_setup_irqs(bp);
2760         if (rc) {
2761                 BNX2X_ERR("setup irqs failed\n");
2762                 if (IS_PF(bp))
2763                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2764                 LOAD_ERROR_EXIT(bp, load_error2);
2765         }
2766
2767         /* Init per-function objects */
2768         if (IS_PF(bp)) {
2769                 /* Setup NIC internals and enable interrupts */
2770                 bnx2x_post_irq_nic_init(bp, load_code);
2771
2772                 bnx2x_init_bp_objs(bp);
2773                 bnx2x_iov_nic_init(bp);
2774
2775                 /* Set AFEX default VLAN tag to an invalid value */
2776                 bp->afex_def_vlan_tag = -1;
2777                 bnx2x_nic_load_afex_dcc(bp, load_code);
2778                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2779                 rc = bnx2x_func_start(bp);
2780                 if (rc) {
2781                         BNX2X_ERR("Function start failed!\n");
2782                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2783
2784                         LOAD_ERROR_EXIT(bp, load_error3);
2785                 }
2786
2787                 /* Send LOAD_DONE command to MCP */
2788                 if (!BP_NOMCP(bp)) {
2789                         load_code = bnx2x_fw_command(bp,
2790                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2791                         if (!load_code) {
2792                                 BNX2X_ERR("MCP response failure, aborting\n");
2793                                 rc = -EBUSY;
2794                                 LOAD_ERROR_EXIT(bp, load_error3);
2795                         }
2796                 }
2797
2798                 /* initialize FW coalescing state machines in RAM */
2799                 bnx2x_update_coalesce(bp);
2800         }
2801
2802         /* setup the leading queue */
2803         rc = bnx2x_setup_leading(bp);
2804         if (rc) {
2805                 BNX2X_ERR("Setup leading failed!\n");
2806                 LOAD_ERROR_EXIT(bp, load_error3);
2807         }
2808
2809         /* set up the rest of the queues */
2810         for_each_nondefault_eth_queue(bp, i) {
2811                 if (IS_PF(bp))
2812                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2813                 else /* VF */
2814                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2815                 if (rc) {
2816                         BNX2X_ERR("Queue %d setup failed\n", i);
2817                         LOAD_ERROR_EXIT(bp, load_error3);
2818                 }
2819         }
2820
2821         /* setup rss */
2822         rc = bnx2x_init_rss(bp);
2823         if (rc) {
2824                 BNX2X_ERR("PF RSS init failed\n");
2825                 LOAD_ERROR_EXIT(bp, load_error3);
2826         }
2827
2828         /* Now when Clients are configured we are ready to work */
2829         bp->state = BNX2X_STATE_OPEN;
2830
2831         /* Configure a ucast MAC */
2832         if (IS_PF(bp))
2833                 rc = bnx2x_set_eth_mac(bp, true);
2834         else /* vf */
2835                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->ea, bp->fp->index,
2836                                            true);
2837         if (rc) {
2838                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2839                 LOAD_ERROR_EXIT(bp, load_error3);
2840         }
2841
2842         if (IS_PF(bp) && bp->pending_max) {
2843                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2844                 bp->pending_max = 0;
2845         }
2846
2847         if (bp->port.pmf) {
2848                 rc = bnx2x_initial_phy_init(bp, load_mode);
2849                 if (rc)
2850                         LOAD_ERROR_EXIT(bp, load_error3);
2851         }
2852         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2853
2854         /* Start fast path */
2855
2856         /* Initialize Rx filter. */
2857         bnx2x_set_rx_mode_inner(bp);
2858
2859         /* Start Tx */
2860         switch (load_mode) {
2861         case LOAD_NORMAL:
2862                 /* Tx queue should be only re-enabled */
2863                 netif_tx_wake_all_queues(bp->dev);
2864                 break;
2865
2866         case LOAD_OPEN:
2867                 netif_tx_start_all_queues(bp->dev);
2868                 cmb();
2869                 break;
2870
2871         case LOAD_DIAG:
2872         case LOAD_LOOPBACK_EXT:
2873                 bp->state = BNX2X_STATE_DIAG;
2874                 break;
2875
2876         default:
2877                 break;
2878         }
2879
2880         if (bp->port.pmf)
2881                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2882         else
2883                 bnx2x__link_status_update(bp);
2884
2885         /* start the timer */
2886         set_awaiter_rel(&bp->timer, bp->current_interval * 1000); // fudge
2887         set_alarm(&per_cpu_info[0].tchain, &bp->timer);
2888
2889         if (CNIC_ENABLED(bp))
2890                 bnx2x_load_cnic(bp);
2891
2892         if (IS_PF(bp))
2893                 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2894
2895         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2896                 /* mark driver is loaded in shmem2 */
2897                 uint32_t val;
2898                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2899                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2900                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2901                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2902         }
2903
2904         /* Wait for all pending SP commands to complete */
2905         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2906                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2907                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2908                 return -EBUSY;
2909         }
2910
2911         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2912         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2913                 bnx2x_dcbx_init(bp, false);
2914
2915         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2916
2917         return 0;
2918
2919 #ifndef BNX2X_STOP_ON_ERROR
2920 load_error3:
2921         if (IS_PF(bp)) {
2922                 bnx2x_int_disable_sync(bp, 1);
2923
2924                 /* Clean queueable objects */
2925                 bnx2x_squeeze_objects(bp);
2926         }
2927
2928         /* Free SKBs, SGEs, TPA pool and driver internals */
2929         bnx2x_free_skbs(bp);
2930         for_each_rx_queue(bp, i)
2931                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2932
2933         /* Release IRQs */
2934         bnx2x_free_irq(bp);
2935 load_error2:
2936         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2937                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2938                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2939         }
2940
2941         bp->port.pmf = 0;
2942 load_error1:
2943         bnx2x_napi_disable(bp);
2944         bnx2x_del_all_napi(bp);
2945
2946         /* clear pf_load status, as it was already set */
2947         if (IS_PF(bp))
2948                 bnx2x_clear_pf_load(bp);
2949 load_error0:
2950         bnx2x_free_fw_stats_mem(bp);
2951         bnx2x_free_fp_mem(bp);
2952         bnx2x_free_mem(bp);
2953
2954         return rc;
2955 #endif /* ! BNX2X_STOP_ON_ERROR */
2956 }
2957
2958 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2959 {
2960         uint8_t rc = 0, cos, i;
2961
2962         /* Wait until tx fastpath tasks complete */
2963         for_each_tx_queue(bp, i) {
2964                 struct bnx2x_fastpath *fp = &bp->fp[i];
2965
2966                 for_each_cos_in_tx_queue(fp, cos)
2967                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2968                 if (rc)
2969                         return rc;
2970         }
2971         return 0;
2972 }
2973
2974 /* must be called with rtnl_lock */
2975 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2976 {
2977 panic("Not implemented");
2978 #if 0 // AKAROS_PORT
2979         int i;
2980         bool global = false;
2981
2982         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2983
2984         /* mark driver is unloaded in shmem2 */
2985         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2986                 uint32_t val;
2987                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2988                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2989                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2990         }
2991
2992         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2993             (bp->state == BNX2X_STATE_CLOSED ||
2994              bp->state == BNX2X_STATE_ERROR)) {
2995                 /* We can get here if the driver has been unloaded
2996                  * during parity error recovery and is either waiting for a
2997                  * leader to complete or for other functions to unload and
2998                  * then ifdown has been issued. In this case we want to
2999                  * unload and let other functions to complete a recovery
3000                  * process.
3001                  */
3002                 bp->recovery_state = BNX2X_RECOVERY_DONE;
3003                 bp->is_leader = 0;
3004                 bnx2x_release_leader_lock(bp);
3005                 mb();
3006
3007                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
3008                 BNX2X_ERR("Can't unload in closed or error state\n");
3009                 return -EINVAL;
3010         }
3011
3012         /* Nothing to do during unload if previous bnx2x_nic_load()
3013          * have not completed successfully - all resources are released.
3014          *
3015          * we can get here only after unsuccessful ndo_* callback, during which
3016          * dev->IFF_UP flag is still on.
3017          */
3018         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
3019                 return 0;
3020
3021         /* It's important to set the bp->state to the value different from
3022          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3023          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3024          */
3025         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3026         mb();
3027
3028         /* indicate to VFs that the PF is going down */
3029         bnx2x_iov_channel_down(bp);
3030
3031         if (CNIC_LOADED(bp))
3032                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3033
3034         /* Stop Tx */
3035         bnx2x_tx_disable(bp);
3036         netdev_reset_tc(bp->dev);
3037
3038         bp->rx_mode = BNX2X_RX_MODE_NONE;
3039
3040         del_timer_sync(&bp->timer);
3041
3042         if (IS_PF(bp)) {
3043                 /* Set ALWAYS_ALIVE bit in shmem */
3044                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3045                 bnx2x_drv_pulse(bp);
3046                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3047                 bnx2x_save_statistics(bp);
3048         }
3049
3050         /* wait till consumers catch up with producers in all queues */
3051         bnx2x_drain_tx_queues(bp);
3052
3053         /* if VF indicate to PF this function is going down (PF will delete sp
3054          * elements and clear initializations
3055          */
3056         if (IS_VF(bp))
3057                 bnx2x_vfpf_close_vf(bp);
3058         else if (unload_mode != UNLOAD_RECOVERY)
3059                 /* if this is a normal/close unload need to clean up chip*/
3060                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3061         else {
3062                 /* Send the UNLOAD_REQUEST to the MCP */
3063                 bnx2x_send_unload_req(bp, unload_mode);
3064
3065                 /* Prevent transactions to host from the functions on the
3066                  * engine that doesn't reset global blocks in case of global
3067                  * attention once global blocks are reset and gates are opened
3068                  * (the engine which leader will perform the recovery
3069                  * last).
3070                  */
3071                 if (!CHIP_IS_E1x(bp))
3072                         bnx2x_pf_disable(bp);
3073
3074                 /* Disable HW interrupts, NAPI */
3075                 bnx2x_netif_stop(bp, 1);
3076                 /* Delete all NAPI objects */
3077                 bnx2x_del_all_napi(bp);
3078                 if (CNIC_LOADED(bp))
3079                         bnx2x_del_all_napi_cnic(bp);
3080                 /* Release IRQs */
3081                 bnx2x_free_irq(bp);
3082
3083                 /* Report UNLOAD_DONE to MCP */
3084                 bnx2x_send_unload_done(bp, false);
3085         }
3086
3087         /*
3088          * At this stage no more interrupts will arrive so we may safely clean
3089          * the queueable objects here in case they failed to get cleaned so far.
3090          */
3091         if (IS_PF(bp))
3092                 bnx2x_squeeze_objects(bp);
3093
3094         /* There should be no more pending SP commands at this stage */
3095         bp->sp_state = 0;
3096
3097         bp->port.pmf = 0;
3098
3099         /* clear pending work in rtnl task */
3100         bp->sp_rtnl_state = 0;
3101         mb();
3102
3103         /* Free SKBs, SGEs, TPA pool and driver internals */
3104         bnx2x_free_skbs(bp);
3105         if (CNIC_LOADED(bp))
3106                 bnx2x_free_skbs_cnic(bp);
3107         for_each_rx_queue(bp, i)
3108                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3109
3110         bnx2x_free_fp_mem(bp);
3111         if (CNIC_LOADED(bp))
3112                 bnx2x_free_fp_mem_cnic(bp);
3113
3114         if (IS_PF(bp)) {
3115                 if (CNIC_LOADED(bp))
3116                         bnx2x_free_mem_cnic(bp);
3117         }
3118         bnx2x_free_mem(bp);
3119
3120         bp->state = BNX2X_STATE_CLOSED;
3121         bp->cnic_loaded = false;
3122
3123         /* Clear driver version indication in shmem */
3124         if (IS_PF(bp))
3125                 bnx2x_update_mng_version(bp);
3126
3127         /* Check if there are pending parity attentions. If there are - set
3128          * RECOVERY_IN_PROGRESS.
3129          */
3130         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3131                 bnx2x_set_reset_in_progress(bp);
3132
3133                 /* Set RESET_IS_GLOBAL if needed */
3134                 if (global)
3135                         bnx2x_set_reset_global(bp);
3136         }
3137
3138         /* The last driver must disable a "close the gate" if there is no
3139          * parity attention or "process kill" pending.
3140          */
3141         if (IS_PF(bp) &&
3142             !bnx2x_clear_pf_load(bp) &&
3143             bnx2x_reset_is_done(bp, BP_PATH(bp)))
3144                 bnx2x_disable_close_the_gate(bp);
3145
3146         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3147
3148         return 0;
3149 #endif
3150 }
3151
3152 #if 0 // AKAROS_PORT
3153 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3154 {
3155         uint16_t pmcsr;
3156
3157         /* If there is no power capability, silently succeed */
3158         if (!bp->pdev->pm_cap) {
3159                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3160                 return 0;
3161         }
3162
3163         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3164
3165         switch (state) {
3166         case PCI_D0:
3167                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3168                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3169                                        PCI_PM_CTRL_PME_STATUS));
3170
3171                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3172                         /* delay required during transition out of D3hot */
3173                         kthread_usleep(1000 * 20);
3174                 break;
3175
3176         case PCI_D3hot:
3177                 /* If there are other clients above don't
3178                    shut down the power */
3179                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3180                         return 0;
3181                 /* Don't shut down the power for emulation and FPGA */
3182                 if (CHIP_REV_IS_SLOW(bp))
3183                         return 0;
3184
3185                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3186                 pmcsr |= 3;
3187
3188                 if (bp->wol)
3189                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3190
3191                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3192                                       pmcsr);
3193
3194                 /* No more memory access after this point until
3195                 * device is brought back to D0.
3196                 */
3197                 break;
3198
3199         default:
3200                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3201                 return -EINVAL;
3202         }
3203         return 0;
3204 }
3205 #endif
3206
3207 /*
3208  * net_device service functions
3209  */
3210 static void bnx2x_poll(uint32_t srcid, long a0, long a1, long a2)
3211 {
3212         struct bnx2x_fastpath *fp = (struct bnx2x_fastpath*)a0;
3213         int work_done = 0;
3214         int budget = INT32_MAX; // AKAROS_PORT  comes from napi; just let it run
3215         uint8_t cos;
3216         struct bnx2x *bp = fp->bp;
3217
3218         while (1) {
3219 #ifdef BNX2X_STOP_ON_ERROR
3220                 if (unlikely(bp->panic)) {
3221                         napi_complete(napi);
3222                         return 0;
3223                 }
3224 #endif
3225                 if (!bnx2x_fp_lock_napi(fp))
3226                         return;
3227
3228                 for_each_cos_in_tx_queue(fp, cos)
3229                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3230                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3231
3232                 if (bnx2x_has_rx_work(fp)) {
3233                         work_done += bnx2x_rx_int(fp, budget - work_done);
3234
3235                         /* must not complete if we consumed full budget */
3236                         if (work_done >= budget) {
3237                                 bnx2x_fp_unlock_napi(fp);
3238                                 break;
3239                         }
3240                 }
3241
3242                 /* Fall out from the NAPI loop if needed */
3243                 if (!bnx2x_fp_unlock_napi(fp) &&
3244                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3245
3246                         /* No need to update SB for FCoE L2 ring as long as
3247                          * it's connected to the default SB and the SB
3248                          * has been updated when NAPI was scheduled.
3249                          */
3250                         if (IS_FCOE_FP(fp)) {
3251                                 napi_complete(napi);
3252                                 break;
3253                         }
3254                         bnx2x_update_fpsb_idx(fp);
3255                         /* bnx2x_has_rx_work() reads the status block,
3256                          * thus we need to ensure that status block indices
3257                          * have been actually read (bnx2x_update_fpsb_idx)
3258                          * prior to this check (bnx2x_has_rx_work) so that
3259                          * we won't write the "newer" value of the status block
3260                          * to IGU (if there was a DMA right after
3261                          * bnx2x_has_rx_work and if there is no rmb, the memory
3262                          * reading (bnx2x_update_fpsb_idx) may be postponed
3263                          * to right before bnx2x_ack_sb). In this case there
3264                          * will never be another interrupt until there is
3265                          * another update of the status block, while there
3266                          * is still unhandled work.
3267                          */
3268                         rmb();
3269
3270                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3271                                 napi_complete(napi);
3272                                 /* Re-enable interrupts */
3273                                 DP(NETIF_MSG_RX_STATUS,
3274                                    "Update index to %d\n", fp->fp_hc_idx);
3275                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3276                                              le16_to_cpu(fp->fp_hc_idx),
3277                                              IGU_INT_ENABLE, 1);
3278                                 break;
3279                         }
3280                 }
3281         }
3282 }
3283
3284 #ifdef CONFIG_NET_RX_BUSY_POLL
3285 /* must be called with local_bh_disable()d */
3286 int bnx2x_low_latency_recv(struct napi_struct *napi)
3287 {
3288         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3289                                                  napi);
3290         struct bnx2x *bp = fp->bp;
3291         int found = 0;
3292
3293         if ((bp->state == BNX2X_STATE_CLOSED) ||
3294             (bp->state == BNX2X_STATE_ERROR) ||
3295             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3296                 return LL_FLUSH_FAILED;
3297
3298         if (!bnx2x_fp_lock_poll(fp))
3299                 return LL_FLUSH_BUSY;
3300
3301         if (bnx2x_has_rx_work(fp))
3302                 found = bnx2x_rx_int(fp, 4);
3303
3304         bnx2x_fp_unlock_poll(fp);
3305
3306         return found;
3307 }
3308 #endif
3309
3310 /* we split the first BD into headers and data BDs
3311  * to ease the pain of our fellow microcode engineers
3312  * we use one mapping for both BDs
3313  */
3314 static uint16_t bnx2x_tx_split(struct bnx2x *bp,
3315                           struct bnx2x_fp_txdata *txdata,
3316                           struct sw_tx_bd *tx_buf,
3317                           struct eth_tx_start_bd **tx_bd, uint16_t hlen,
3318                           uint16_t bd_prod)
3319 {
3320         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3321         struct eth_tx_bd *d_tx_bd;
3322         dma_addr_t mapping;
3323         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3324
3325         /* first fix first BD */
3326         h_tx_bd->nbytes = cpu_to_le16(hlen);
3327
3328         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3329            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3330
3331         /* now get a new data BD
3332          * (after the pbd) and fill it */
3333         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3334         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3335
3336         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3337                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3338
3339         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3340         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3341         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3342
3343         /* this marks the BD as one that has no individual mapping */
3344         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3345
3346         DP(NETIF_MSG_TX_QUEUED,
3347            "TSO split data size is %d (%x:%x)\n",
3348            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3349
3350         /* update tx_bd */
3351         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3352
3353         return bd_prod;
3354 }
3355
3356 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3357 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3358 static __le16 bnx2x_csum_fix(unsigned char *t_header, uint16_t csum,
3359                              int8_t fix)
3360 {
3361 panic("Not implemented");
3362 #if 0 // AKAROS_PORT
3363         __sum16 tsum = (__force __sum16) csum;
3364
3365         if (fix > 0)
3366                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3367                                   csum_partial(t_header - fix, fix, 0)));
3368
3369         else if (fix < 0)
3370                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3371                                   csum_partial(t_header, -fix, 0)));
3372
3373         return bswab16(tsum);
3374 #endif
3375 }
3376
3377 static uint32_t bnx2x_xmit_type(struct bnx2x *bp, struct block *block)
3378 {
3379         uint32_t rc;
3380         __u8 prot = 0;
3381         __be16 protocol;
3382
3383         /* TODO: AKAROS_PORT ask for checksums */
3384         return XMIT_PLAIN;
3385
3386 #if 0 // AKAROS_PORT
3387         if (skb->ip_summed != CHECKSUM_PARTIAL)
3388                 return XMIT_PLAIN;
3389
3390         protocol = vlan_get_protocol(skb);
3391         if (protocol == cpu_to_be16(ETH_P_IPV6)) {
3392                 rc = XMIT_CSUM_V6;
3393                 prot = ipv6_hdr(skb)->nexthdr;
3394         } else {
3395                 rc = XMIT_CSUM_V4;
3396                 prot = ip_hdr(skb)->protocol;
3397         }
3398
3399         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3400                 if (inner_ip_hdr(skb)->version == 6) {
3401                         rc |= XMIT_CSUM_ENC_V6;
3402                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3403                                 rc |= XMIT_CSUM_TCP;
3404                 } else {
3405                         rc |= XMIT_CSUM_ENC_V4;
3406                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3407                                 rc |= XMIT_CSUM_TCP;
3408                 }
3409         }
3410         if (prot == IPPROTO_TCP)
3411                 rc |= XMIT_CSUM_TCP;
3412
3413         if (skb_is_gso(skb)) {
3414                 if (skb_is_gso_v6(skb)) {
3415                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3416                         if (rc & XMIT_CSUM_ENC)
3417                                 rc |= XMIT_GSO_ENC_V6;
3418                 } else {
3419                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3420                         if (rc & XMIT_CSUM_ENC)
3421                                 rc |= XMIT_GSO_ENC_V4;
3422                 }
3423         }
3424
3425         return rc;
3426 #endif
3427 }
3428
3429 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3430 /* check if packet requires linearization (packet is too fragmented)
3431    no need to check fragmentation if page size > 8K (there will be no
3432    violation to FW restrictions) */
3433 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3434                              uint32_t xmit_type)
3435 {
3436 panic("Not implemented");
3437 #if 0 // AKAROS_PORT
3438         int to_copy = 0;
3439         int hlen = 0;
3440         int first_bd_sz = 0;
3441
3442         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3443         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3444
3445                 if (xmit_type & XMIT_GSO) {
3446                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3447                         /* Check if LSO packet needs to be copied:
3448                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3449                         int wnd_size = MAX_FETCH_BD - 3;
3450                         /* Number of windows to check */
3451                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3452                         int wnd_idx = 0;
3453                         int frag_idx = 0;
3454                         uint32_t wnd_sum = 0;
3455
3456                         /* Headers length */
3457                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3458                                 tcp_hdrlen(skb);
3459
3460                         /* Amount of data (w/o headers) on linear part of SKB*/
3461                         first_bd_sz = skb_headlen(skb) - hlen;
3462
3463                         wnd_sum  = first_bd_sz;
3464
3465                         /* Calculate the first sum - it's special */
3466                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3467                                 wnd_sum +=
3468                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3469
3470                         /* If there was data on linear skb data - check it */
3471                         if (first_bd_sz > 0) {
3472                                 if (unlikely(wnd_sum < lso_mss)) {
3473                                         to_copy = 1;
3474                                         goto exit_lbl;
3475                                 }
3476
3477                                 wnd_sum -= first_bd_sz;
3478                         }
3479
3480                         /* Others are easier: run through the frag list and
3481                            check all windows */
3482                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3483                                 wnd_sum +=
3484                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3485
3486                                 if (unlikely(wnd_sum < lso_mss)) {
3487                                         to_copy = 1;
3488                                         break;
3489                                 }
3490                                 wnd_sum -=
3491                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3492                         }
3493                 } else {
3494                         /* in non-LSO too fragmented packet should always
3495                            be linearized */
3496                         to_copy = 1;
3497                 }
3498         }
3499
3500 exit_lbl:
3501         if (unlikely(to_copy))
3502                 DP(NETIF_MSG_TX_QUEUED,
3503                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3504                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3505                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3506
3507         return to_copy;
3508 #endif
3509 }
3510 #endif
3511
3512 /**
3513  * bnx2x_set_pbd_gso - update PBD in GSO case.
3514  *
3515  * @skb:        packet skb
3516  * @pbd:        parse BD
3517  * @xmit_type:  xmit flags
3518  */
3519 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3520                               struct eth_tx_parse_bd_e1x *pbd,
3521                               uint32_t xmit_type)
3522 {
3523 panic("Not implemented");
3524 #if 0 // AKAROS_PORT
3525         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3526         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3527         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3528
3529         if (xmit_type & XMIT_GSO_V4) {
3530                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3531                 pbd->tcp_pseudo_csum =
3532                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3533                                                    ip_hdr(skb)->daddr,
3534                                                    0, IPPROTO_TCP, 0));
3535         } else {
3536                 pbd->tcp_pseudo_csum =
3537                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3538                                                  &ipv6_hdr(skb)->daddr,
3539                                                  0, IPPROTO_TCP, 0));
3540         }
3541
3542         pbd->global_data |=
3543                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3544 #endif
3545 }
3546
3547 /**
3548  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3549  *
3550  * @bp:                 driver handle
3551  * @skb:                packet skb
3552  * @parsing_data:       data to be updated
3553  * @xmit_type:          xmit flags
3554  *
3555  * 57712/578xx related, when skb has encapsulation
3556  */
3557 static uint8_t bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3558                                  uint32_t *parsing_data, uint32_t xmit_type)
3559 {
3560 panic("Not implemented");
3561 #if 0 // AKAROS_PORT
3562         *parsing_data |=
3563                 ((((uint8_t *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3564                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3565                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3566
3567         if (xmit_type & XMIT_CSUM_TCP) {
3568                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3569                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3570                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3571
3572                 return skb_inner_transport_header(skb) +
3573                         inner_tcp_hdrlen(skb) - skb->data;
3574         }
3575
3576         /* We support checksum offload for TCP and UDP only.
3577          * No need to pass the UDP header length - it's a constant.
3578          */
3579         return skb_inner_transport_header(skb) +
3580                 sizeof(struct udphdr) - skb->data;
3581 #endif
3582 }
3583
3584 /**
3585  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3586  *
3587  * @bp:                 driver handle
3588  * @skb:                packet skb
3589  * @parsing_data:       data to be updated
3590  * @xmit_type:          xmit flags
3591  *
3592  * 57712/578xx related
3593  */
3594 static uint8_t bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3595                                 uint32_t *parsing_data, uint32_t xmit_type)
3596 {
3597 panic("Not implemented");
3598 #if 0 // AKAROS_PORT
3599         *parsing_data |=
3600                 ((((uint8_t *)skb_transport_header(skb) - skb->data) >> 1) <<
3601                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3602                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3603
3604         if (xmit_type & XMIT_CSUM_TCP) {
3605                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3606                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3607                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3608
3609                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3610         }
3611         /* We support checksum offload for TCP and UDP only.
3612          * No need to pass the UDP header length - it's a constant.
3613          */
3614         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3615 #endif
3616 }
3617
3618 /* set FW indication according to inner or outer protocols if tunneled */
3619 static void bnx2x_set_sbd_csum(struct bnx2x *bp, void *ignored_skb,
3620                                struct eth_tx_start_bd *tx_start_bd,
3621                                uint32_t xmit_type)
3622 {
3623         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3624
3625         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3626                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3627
3628         if (!(xmit_type & XMIT_CSUM_TCP))
3629                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3630 }
3631
3632 /**
3633  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3634  *
3635  * @bp:         driver handle
3636  * @skb:        packet skb
3637  * @pbd:        parse BD to be updated
3638  * @xmit_type:  xmit flags
3639  */
3640 static uint8_t bnx2x_set_pbd_csum(struct bnx2x *bp, struct block *block,
3641                              struct eth_tx_parse_bd_e1x *pbd,
3642                              uint32_t xmit_type)
3643 {
3644 panic("Not implemented");
3645 #if 0 // AKAROS_PORT
3646         uint8_t hlen = (skb_network_header(skb) - skb->data) >> 1;
3647
3648         /* for now NS flag is not used in Linux */
3649         pbd->global_data =
3650                 cpu_to_le16(hlen |
3651                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3652                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3653
3654         pbd->ip_hlen_w = (skb_transport_header(skb) -
3655                         skb_network_header(skb)) >> 1;
3656
3657         hlen += pbd->ip_hlen_w;
3658
3659         /* We support checksum offload for TCP and UDP only */
3660         if (xmit_type & XMIT_CSUM_TCP)
3661                 hlen += tcp_hdrlen(skb) / 2;
3662         else
3663                 hlen += sizeof(struct udphdr) / 2;
3664
3665         pbd->total_hlen_w = cpu_to_le16(hlen);
3666         hlen = hlen*2;
3667
3668         if (xmit_type & XMIT_CSUM_TCP) {
3669                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);